Add m-esr52 at 52.6.0

1 files changed, 1153 insertions, 0 deletions

diff --git a/netwerk/cache2/CacheIndex.h b/netwerk/cache2/CacheIndex.h
new file mode 100644
index 000000000..dc72c346f
--- /dev/null
+++ b/netwerk/cache2/CacheIndex.h
@@ -0,0 +1,1153 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef CacheIndex__h__
+#define CacheIndex__h__
+
+#include "CacheLog.h"
+#include "CacheFileIOManager.h"
+#include "nsIRunnable.h"
+#include "CacheHashUtils.h"
+#include "nsICacheStorageService.h"
+#include "nsICacheEntry.h"
+#include "nsILoadContextInfo.h"
+#include "nsTHashtable.h"
+#include "nsThreadUtils.h"
+#include "nsWeakReference.h"
+#include "mozilla/SHA1.h"
+#include "mozilla/StaticMutex.h"
+#include "mozilla/StaticPtr.h"
+#include "mozilla/EndianUtils.h"
+#include "mozilla/TimeStamp.h"
+
+class nsIFile;
+class nsIDirectoryEnumerator;
+class nsITimer;
+
+
+#ifdef DEBUG
+#define DEBUG_STATS 1
+#endif
+
+namespace mozilla {
+namespace net {
+
+class CacheFileMetadata;
+class FileOpenHelper;
+class CacheIndexIterator;
+
+typedef struct {
+  // Version of the index. The index must be ignored and deleted when the file
+  // on disk was written with a newer version.
+  uint32_t mVersion;
+
+  // Timestamp of time when the last successful write of the index started.
+  // During update process we use this timestamp for a quick validation of entry
+  // files. If last modified time of the file is lower than this timestamp, we
+  // skip parsing of such file since the information in index should be up to
+  // date.
+  uint32_t mTimeStamp;
+
+  // We set this flag as soon as possible after parsing index during startup
+  // and clean it after we write journal to disk during shutdown. We ignore the
+  // journal and start update process whenever this flag is set during index
+  // parsing.
+  uint32_t mIsDirty;
+} CacheIndexHeader;
+
+static_assert(
+  sizeof(CacheIndexHeader::mVersion) + sizeof(CacheIndexHeader::mTimeStamp) +
+  sizeof(CacheIndexHeader::mIsDirty) == sizeof(CacheIndexHeader),
+  "Unexpected sizeof(CacheIndexHeader)!");
+
+struct CacheIndexRecord {
+  SHA1Sum::Hash   mHash;
+  uint32_t        mFrecency;
+  OriginAttrsHash mOriginAttrsHash;
+  uint32_t        mExpirationTime;
+
+  /*
+   *    1000 0000 0000 0000 0000 0000 0000 0000 : initialized
+   *    0100 0000 0000 0000 0000 0000 0000 0000 : anonymous
+   *    0010 0000 0000 0000 0000 0000 0000 0000 : removed
+   *    0001 0000 0000 0000 0000 0000 0000 0000 : dirty
+   *    0000 1000 0000 0000 0000 0000 0000 0000 : fresh
+   *    0000 0100 0000 0000 0000 0000 0000 0000 : pinned
+   *    0000 0011 0000 0000 0000 0000 0000 0000 : reserved
+   *    0000 0000 1111 1111 1111 1111 1111 1111 : file size (in kB)
+   */
+  uint32_t        mFlags;
+
+  CacheIndexRecord()
+    : mFrecency(0)
+    , mOriginAttrsHash(0)
+    , mExpirationTime(nsICacheEntry::NO_EXPIRATION_TIME)
+    , mFlags(0)
+  {}
+};
+
+static_assert(
+  sizeof(CacheIndexRecord::mHash) + sizeof(CacheIndexRecord::mFrecency) +
+  sizeof(CacheIndexRecord::mOriginAttrsHash) + sizeof(CacheIndexRecord::mExpirationTime) +
+  sizeof(CacheIndexRecord::mFlags) == sizeof(CacheIndexRecord),
+  "Unexpected sizeof(CacheIndexRecord)!");
+
+class CacheIndexEntry : public PLDHashEntryHdr
+{
+public:
+  typedef const SHA1Sum::Hash& KeyType;
+  typedef const SHA1Sum::Hash* KeyTypePointer;
+
+  explicit CacheIndexEntry(KeyTypePointer aKey)
+  {
+    MOZ_COUNT_CTOR(CacheIndexEntry);
+    mRec = new CacheIndexRecord();
+    LOG(("CacheIndexEntry::CacheIndexEntry() - Created record [rec=%p]", mRec.get()));
+    memcpy(&mRec->mHash, aKey, sizeof(SHA1Sum::Hash));
+  }
+  CacheIndexEntry(const CacheIndexEntry& aOther)
+  {
+    NS_NOTREACHED("CacheIndexEntry copy constructor is forbidden!");
+  }
+  ~CacheIndexEntry()
+  {
+    MOZ_COUNT_DTOR(CacheIndexEntry);
+    LOG(("CacheIndexEntry::~CacheIndexEntry() - Deleting record [rec=%p]",
+         mRec.get()));
+  }
+
+  // KeyEquals(): does this entry match this key?
+  bool KeyEquals(KeyTypePointer aKey) const
+  {
+    return memcmp(&mRec->mHash, aKey, sizeof(SHA1Sum::Hash)) == 0;
+  }
+
+  // KeyToPointer(): Convert KeyType to KeyTypePointer
+  static KeyTypePointer KeyToPointer(KeyType aKey) { return &aKey; }
+
+  // HashKey(): calculate the hash number
+  static PLDHashNumber HashKey(KeyTypePointer aKey)
+  {
+    return (reinterpret_cast<const uint32_t *>(aKey))[0];
+  }
+
+  // ALLOW_MEMMOVE can we move this class with memmove(), or do we have
+  // to use the copy constructor?
+  enum { ALLOW_MEMMOVE = true };
+
+  bool operator==(const CacheIndexEntry& aOther) const
+  {
+    return KeyEquals(&aOther.mRec->mHash);
+  }
+
+  CacheIndexEntry& operator=(const CacheIndexEntry& aOther)
+  {
+    MOZ_ASSERT(memcmp(&mRec->mHash, &aOther.mRec->mHash,
+               sizeof(SHA1Sum::Hash)) == 0);
+    mRec->mFrecency = aOther.mRec->mFrecency;
+    mRec->mExpirationTime = aOther.mRec->mExpirationTime;
+    mRec->mOriginAttrsHash = aOther.mRec->mOriginAttrsHash;
+    mRec->mFlags = aOther.mRec->mFlags;
+    return *this;
+  }
+
+  void InitNew()
+  {
+    mRec->mFrecency = 0;
+    mRec->mExpirationTime = nsICacheEntry::NO_EXPIRATION_TIME;
+    mRec->mOriginAttrsHash = 0;
+    mRec->mFlags = 0;
+  }
+
+  void Init(OriginAttrsHash aOriginAttrsHash, bool aAnonymous, bool aPinned)
+  {
+    MOZ_ASSERT(mRec->mFrecency == 0);
+    MOZ_ASSERT(mRec->mExpirationTime == nsICacheEntry::NO_EXPIRATION_TIME);
+    MOZ_ASSERT(mRec->mOriginAttrsHash == 0);
+    // When we init the entry it must be fresh and may be dirty
+    MOZ_ASSERT((mRec->mFlags & ~kDirtyMask) == kFreshMask);
+
+    mRec->mOriginAttrsHash = aOriginAttrsHash;
+    mRec->mFlags |= kInitializedMask;
+    if (aAnonymous) {
+      mRec->mFlags |= kAnonymousMask;
+    }
+    if (aPinned) {
+      mRec->mFlags |= kPinnedMask;
+    }
+  }
+
+  const SHA1Sum::Hash * Hash() const { return &mRec->mHash; }
+
+  bool IsInitialized() const { return !!(mRec->mFlags & kInitializedMask); }
+
+  mozilla::net::OriginAttrsHash OriginAttrsHash() const { return mRec->mOriginAttrsHash; }
+
+  bool Anonymous() const { return !!(mRec->mFlags & kAnonymousMask); }
+
+  bool IsRemoved() const { return !!(mRec->mFlags & kRemovedMask); }
+  void MarkRemoved() { mRec->mFlags |= kRemovedMask; }
+
+  bool IsDirty() const { return !!(mRec->mFlags & kDirtyMask); }
+  void MarkDirty() { mRec->mFlags |= kDirtyMask; }
+  void ClearDirty() { mRec->mFlags &= ~kDirtyMask; }
+
+  bool IsFresh() const { return !!(mRec->mFlags & kFreshMask); }
+  void MarkFresh() { mRec->mFlags |= kFreshMask; }
+
+  bool IsPinned() const { return !!(mRec->mFlags & kPinnedMask); }
+
+  void     SetFrecency(uint32_t aFrecency) { mRec->mFrecency = aFrecency; }
+  uint32_t GetFrecency() const { return mRec->mFrecency; }
+
+  void     SetExpirationTime(uint32_t aExpirationTime)
+  {
+    mRec->mExpirationTime = aExpirationTime;
+  }
+  uint32_t GetExpirationTime() const { return mRec->mExpirationTime; }
+
+  // Sets filesize in kilobytes.
+  void     SetFileSize(uint32_t aFileSize)
+  {
+    if (aFileSize > kFileSizeMask) {
+      LOG(("CacheIndexEntry::SetFileSize() - FileSize is too large, "
+           "truncating to %u", kFileSizeMask));
+      aFileSize = kFileSizeMask;
+    }
+    mRec->mFlags &= ~kFileSizeMask;
+    mRec->mFlags |= aFileSize;
+  }
+  // Returns filesize in kilobytes.
+  uint32_t GetFileSize() const { return GetFileSize(mRec); }
+  static uint32_t GetFileSize(CacheIndexRecord *aRec)
+  {
+    return aRec->mFlags & kFileSizeMask;
+  }
+  static uint32_t IsPinned(CacheIndexRecord *aRec)
+  {
+    return aRec->mFlags & kPinnedMask;
+  }
+  bool     IsFileEmpty() const { return GetFileSize() == 0; }
+
+  void WriteToBuf(void *aBuf)
+  {
+    uint8_t* ptr = static_cast<uint8_t*>(aBuf);
+    memcpy(ptr, mRec->mHash, sizeof(SHA1Sum::Hash)); ptr += sizeof(SHA1Sum::Hash);
+    NetworkEndian::writeUint32(ptr, mRec->mFrecency); ptr += sizeof(uint32_t);
+    NetworkEndian::writeUint64(ptr, mRec->mOriginAttrsHash); ptr += sizeof(uint64_t);
+    NetworkEndian::writeUint32(ptr, mRec->mExpirationTime); ptr += sizeof(uint32_t);
+    // Dirty and fresh flags should never go to disk, since they make sense only
+    // during current session.
+    NetworkEndian::writeUint32(ptr, mRec->mFlags & ~(kDirtyMask | kFreshMask));
+  }
+
+  void ReadFromBuf(void *aBuf)
+  {
+    const uint8_t* ptr = static_cast<const uint8_t*>(aBuf);
+    MOZ_ASSERT(memcmp(&mRec->mHash, ptr, sizeof(SHA1Sum::Hash)) == 0); ptr += sizeof(SHA1Sum::Hash);
+    mRec->mFrecency = NetworkEndian::readUint32(ptr); ptr += sizeof(uint32_t);
+    mRec->mOriginAttrsHash = NetworkEndian::readUint64(ptr); ptr += sizeof(uint64_t);
+    mRec->mExpirationTime = NetworkEndian::readUint32(ptr); ptr += sizeof(uint32_t);
+    mRec->mFlags = NetworkEndian::readUint32(ptr);
+  }
+
+  void Log() const {
+    LOG(("CacheIndexEntry::Log() [this=%p, hash=%08x%08x%08x%08x%08x, fresh=%u,"
+         " initialized=%u, removed=%u, dirty=%u, anonymous=%u, "
+         "originAttrsHash=%llx, frecency=%u, expirationTime=%u, size=%u]",
+         this, LOGSHA1(mRec->mHash), IsFresh(), IsInitialized(), IsRemoved(),
+         IsDirty(), Anonymous(), OriginAttrsHash(), GetFrecency(),
+         GetExpirationTime(), GetFileSize()));
+  }
+
+  static bool RecordMatchesLoadContextInfo(CacheIndexRecord *aRec,
+                                           nsILoadContextInfo *aInfo)
+  {
+    if (!aInfo->IsPrivate() &&
+        GetOriginAttrsHash(*aInfo->OriginAttributesPtr()) == aRec->mOriginAttrsHash &&
+        aInfo->IsAnonymous() == !!(aRec->mFlags & kAnonymousMask)) {
+      return true;
+    }
+
+    return false;
+  }
+
+  // Memory reporting
+  size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const
+  {
+    return mallocSizeOf(mRec.get());
+  }
+
+  size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const
+  {
+    return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf);
+  }
+
+private:
+  friend class CacheIndexEntryUpdate;
+  friend class CacheIndex;
+  friend class CacheIndexEntryAutoManage;
+
+  static const uint32_t kInitializedMask = 0x80000000;
+  static const uint32_t kAnonymousMask   = 0x40000000;
+
+  // This flag is set when the entry was removed. We need to keep this
+  // information in memory until we write the index file.
+  static const uint32_t kRemovedMask     = 0x20000000;
+
+  // This flag is set when the information in memory is not in sync with the
+  // information in index file on disk.
+  static const uint32_t kDirtyMask       = 0x10000000;
+
+  // This flag is set when the information about the entry is fresh, i.e.
+  // we've created or opened this entry during this session, or we've seen
+  // this entry during update or build process.
+  static const uint32_t kFreshMask       = 0x08000000;
+
+  // Indicates a pinned entry.
+  static const uint32_t kPinnedMask      = 0x04000000;
+
+  static const uint32_t kReservedMask    = 0x03000000;
+
+  // FileSize in kilobytes
+  static const uint32_t kFileSizeMask    = 0x00FFFFFF;
+
+  nsAutoPtr<CacheIndexRecord> mRec;
+};
+
+class CacheIndexEntryUpdate : public CacheIndexEntry
+{
+public:
+  explicit CacheIndexEntryUpdate(CacheIndexEntry::KeyTypePointer aKey)
+    : CacheIndexEntry(aKey)
+    , mUpdateFlags(0)
+  {
+    MOZ_COUNT_CTOR(CacheIndexEntryUpdate);
+    LOG(("CacheIndexEntryUpdate::CacheIndexEntryUpdate()"));
+  }
+  ~CacheIndexEntryUpdate()
+  {
+    MOZ_COUNT_DTOR(CacheIndexEntryUpdate);
+    LOG(("CacheIndexEntryUpdate::~CacheIndexEntryUpdate()"));
+  }
+
+  CacheIndexEntryUpdate& operator=(const CacheIndexEntry& aOther)
+  {
+    MOZ_ASSERT(memcmp(&mRec->mHash, &aOther.mRec->mHash,
+               sizeof(SHA1Sum::Hash)) == 0);
+    mUpdateFlags = 0;
+    *(static_cast<CacheIndexEntry *>(this)) = aOther;
+    return *this;
+  }
+
+  void InitNew()
+  {
+    mUpdateFlags = kFrecencyUpdatedMask | kExpirationUpdatedMask |
+                   kFileSizeUpdatedMask;
+    CacheIndexEntry::InitNew();
+  }
+
+  void SetFrecency(uint32_t aFrecency)
+  {
+    mUpdateFlags |= kFrecencyUpdatedMask;
+    CacheIndexEntry::SetFrecency(aFrecency);
+  }
+
+  void SetExpirationTime(uint32_t aExpirationTime)
+  {
+    mUpdateFlags |= kExpirationUpdatedMask;
+    CacheIndexEntry::SetExpirationTime(aExpirationTime);
+  }
+
+  void SetFileSize(uint32_t aFileSize)
+  {
+    mUpdateFlags |= kFileSizeUpdatedMask;
+    CacheIndexEntry::SetFileSize(aFileSize);
+  }
+
+  void ApplyUpdate(CacheIndexEntry *aDst) {
+    MOZ_ASSERT(memcmp(&mRec->mHash, &aDst->mRec->mHash,
+               sizeof(SHA1Sum::Hash)) == 0);
+    if (mUpdateFlags & kFrecencyUpdatedMask) {
+      aDst->mRec->mFrecency = mRec->mFrecency;
+    }
+    if (mUpdateFlags & kExpirationUpdatedMask) {
+      aDst->mRec->mExpirationTime = mRec->mExpirationTime;
+    }
+    aDst->mRec->mOriginAttrsHash = mRec->mOriginAttrsHash;
+    if (mUpdateFlags & kFileSizeUpdatedMask) {
+      aDst->mRec->mFlags = mRec->mFlags;
+    } else {
+      // Copy all flags except file size.
+      aDst->mRec->mFlags &= kFileSizeMask;
+      aDst->mRec->mFlags |= (mRec->mFlags & ~kFileSizeMask);
+    }
+  }
+
+private:
+  static const uint32_t kFrecencyUpdatedMask = 0x00000001;
+  static const uint32_t kExpirationUpdatedMask = 0x00000002;
+  static const uint32_t kFileSizeUpdatedMask = 0x00000004;
+
+  uint32_t mUpdateFlags;
+};
+
+class CacheIndexStats
+{
+public:
+  CacheIndexStats()
+    : mCount(0)
+    , mNotInitialized(0)
+    , mRemoved(0)
+    , mDirty(0)
+    , mFresh(0)
+    , mEmpty(0)
+    , mSize(0)
+#ifdef DEBUG
+    , mStateLogged(false)
+    , mDisableLogging(false)
+#endif
+  {
+  }
+
+  bool operator==(const CacheIndexStats& aOther) const
+  {
+    return
+#ifdef DEBUG
+           aOther.mStateLogged == mStateLogged &&
+#endif
+           aOther.mCount == mCount &&
+           aOther.mNotInitialized == mNotInitialized &&
+           aOther.mRemoved == mRemoved &&
+           aOther.mDirty == mDirty &&
+           aOther.mFresh == mFresh &&
+           aOther.mEmpty == mEmpty &&
+           aOther.mSize == mSize;
+  }
+
+#ifdef DEBUG
+  void DisableLogging() {
+    mDisableLogging = true;
+  }
+#endif
+
+  void Log() {
+    LOG(("CacheIndexStats::Log() [count=%u, notInitialized=%u, removed=%u, "
+         "dirty=%u, fresh=%u, empty=%u, size=%u]", mCount, mNotInitialized,
+         mRemoved, mDirty, mFresh, mEmpty, mSize));
+  }
+
+  void Clear() {
+    MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Clear() - state logged!");
+
+    mCount = 0;
+    mNotInitialized = 0;
+    mRemoved = 0;
+    mDirty = 0;
+    mFresh = 0;
+    mEmpty = 0;
+    mSize = 0;
+  }
+
+#ifdef DEBUG
+  bool StateLogged() {
+    return mStateLogged;
+  }
+#endif
+
+  uint32_t Count() {
+    MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Count() - state logged!");
+    return mCount;
+  }
+
+  uint32_t Dirty() {
+    MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Dirty() - state logged!");
+    return mDirty;
+  }
+
+  uint32_t Fresh() {
+    MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Fresh() - state logged!");
+    return mFresh;
+  }
+
+  uint32_t ActiveEntriesCount() {
+    MOZ_ASSERT(!mStateLogged, "CacheIndexStats::ActiveEntriesCount() - state "
+               "logged!");
+    return mCount - mRemoved - mNotInitialized - mEmpty;
+  }
+
+  uint32_t Size() {
+    MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Size() - state logged!");
+    return mSize;
+  }
+
+  void BeforeChange(const CacheIndexEntry *aEntry) {
+#ifdef DEBUG_STATS
+    if (!mDisableLogging) {
+      LOG(("CacheIndexStats::BeforeChange()"));
+      Log();
+    }
+#endif
+
+    MOZ_ASSERT(!mStateLogged, "CacheIndexStats::BeforeChange() - state "
+               "logged!");
+#ifdef DEBUG
+    mStateLogged = true;
+#endif
+    if (aEntry) {
+      MOZ_ASSERT(mCount);
+      mCount--;
+      if (aEntry->IsDirty()) {
+        MOZ_ASSERT(mDirty);
+        mDirty--;
+      }
+      if (aEntry->IsFresh()) {
+        MOZ_ASSERT(mFresh);
+        mFresh--;
+      }
+      if (aEntry->IsRemoved()) {
+        MOZ_ASSERT(mRemoved);
+        mRemoved--;
+      } else {
+        if (!aEntry->IsInitialized()) {
+          MOZ_ASSERT(mNotInitialized);
+          mNotInitialized--;
+        } else {
+          if (aEntry->IsFileEmpty()) {
+            MOZ_ASSERT(mEmpty);
+            mEmpty--;
+          } else {
+            MOZ_ASSERT(mSize >= aEntry->GetFileSize());
+            mSize -= aEntry->GetFileSize();
+          }
+        }
+      }
+    }
+  }
+
+  void AfterChange(const CacheIndexEntry *aEntry) {
+    MOZ_ASSERT(mStateLogged, "CacheIndexStats::AfterChange() - state not "
+               "logged!");
+#ifdef DEBUG
+    mStateLogged = false;
+#endif
+    if (aEntry) {
+      ++mCount;
+      if (aEntry->IsDirty()) {
+        mDirty++;
+      }
+      if (aEntry->IsFresh()) {
+        mFresh++;
+      }
+      if (aEntry->IsRemoved()) {
+        mRemoved++;
+      } else {
+        if (!aEntry->IsInitialized()) {
+          mNotInitialized++;
+        } else {
+          if (aEntry->IsFileEmpty()) {
+            mEmpty++;
+          } else {
+            mSize += aEntry->GetFileSize();
+          }
+        }
+      }
+    }
+
+#ifdef DEBUG_STATS
+    if (!mDisableLogging) {
+      LOG(("CacheIndexStats::AfterChange()"));
+      Log();
+    }
+#endif
+  }
+
+private:
+  uint32_t mCount;
+  uint32_t mNotInitialized;
+  uint32_t mRemoved;
+  uint32_t mDirty;
+  uint32_t mFresh;
+  uint32_t mEmpty;
+  uint32_t mSize;
+#ifdef DEBUG
+  // We completely remove the data about an entry from the stats in
+  // BeforeChange() and set this flag to true. The entry is then modified,
+  // deleted or created and the data is again put into the stats and this flag
+  // set to false. Statistics must not be read during this time since the
+  // information is not correct.
+  bool     mStateLogged;
+
+  // Disables logging in this instance of CacheIndexStats
+  bool     mDisableLogging;
+#endif
+};
+
+class CacheIndex : public CacheFileIOListener
+                 , public nsIRunnable
+{
+public:
+  NS_DECL_THREADSAFE_ISUPPORTS
+  NS_DECL_NSIRUNNABLE
+
+  CacheIndex();
+
+  static nsresult Init(nsIFile *aCacheDirectory);
+  static nsresult PreShutdown();
+  static nsresult Shutdown();
+
+  // Following methods can be called only on IO thread.
+
+  // Add entry to the index. The entry shouldn't be present in index. This
+  // method is called whenever a new handle for a new entry file is created. The
+  // newly created entry is not initialized and it must be either initialized
+  // with InitEntry() or removed with RemoveEntry().
+  static nsresult AddEntry(const SHA1Sum::Hash *aHash);
+
+  // Inform index about an existing entry that should be present in index. This
+  // method is called whenever a new handle for an existing entry file is
+  // created. Like in case of AddEntry(), either InitEntry() or RemoveEntry()
+  // must be called on the entry, since the entry is not initizlized if the
+  // index is outdated.
+  static nsresult EnsureEntryExists(const SHA1Sum::Hash *aHash);
+
+  // Initialize the entry. It MUST be present in index. Call to AddEntry() or
+  // EnsureEntryExists() must precede the call to this method.
+  static nsresult InitEntry(const SHA1Sum::Hash *aHash,
+                            OriginAttrsHash      aOriginAttrsHash,
+                            bool                 aAnonymous,
+                            bool                 aPinned);
+
+  // Remove entry from index. The entry should be present in index.
+  static nsresult RemoveEntry(const SHA1Sum::Hash *aHash);
+
+  // Update some information in entry. The entry MUST be present in index and
+  // MUST be initialized. Call to AddEntry() or EnsureEntryExists() and to
+  // InitEntry() must precede the call to this method.
+  // Pass nullptr if the value didn't change.
+  static nsresult UpdateEntry(const SHA1Sum::Hash *aHash,
+                              const uint32_t      *aFrecency,
+                              const uint32_t      *aExpirationTime,
+                              const uint32_t      *aSize);
+
+  // Remove all entries from the index. Called when clearing the whole cache.
+  static nsresult RemoveAll();
+
+  enum EntryStatus {
+    EXISTS         = 0,
+    DOES_NOT_EXIST = 1,
+    DO_NOT_KNOW    = 2
+  };
+
+  // Returns status of the entry in index for the given key. It can be called
+  // on any thread.
+  // If _pinned is non-null, it's filled with pinning status of the entry.
+  static nsresult HasEntry(const nsACString &aKey, EntryStatus *_retval,
+                           bool *_pinned = nullptr);
+  static nsresult HasEntry(const SHA1Sum::Hash &hash, EntryStatus *_retval,
+                           bool *_pinned = nullptr);
+
+  // Returns a hash of the least important entry that should be evicted if the
+  // cache size is over limit and also returns a total number of all entries in
+  // the index minus the number of forced valid entries and unpinned entries
+  // that we encounter when searching (see below)
+  static nsresult GetEntryForEviction(bool aIgnoreEmptyEntries, SHA1Sum::Hash *aHash, uint32_t *aCnt);
+
+  // Checks if a cache entry is currently forced valid. Used to prevent an entry
+  // (that has been forced valid) from being evicted when the cache size reaches
+  // its limit.
+  static bool IsForcedValidEntry(const SHA1Sum::Hash *aHash);
+
+  // Returns cache size in kB.
+  static nsresult GetCacheSize(uint32_t *_retval);
+
+  // Returns number of entry files in the cache
+  static nsresult GetEntryFileCount(uint32_t *_retval);
+
+  // Synchronously returns the disk occupation and number of entries per-context.
+  // Callable on any thread.
+  static nsresult GetCacheStats(nsILoadContextInfo *aInfo, uint32_t *aSize, uint32_t *aCount);
+
+  // Asynchronously gets the disk cache size, used for display in the UI.
+  static nsresult AsyncGetDiskConsumption(nsICacheStorageConsumptionObserver* aObserver);
+
+  // Returns an iterator that returns entries matching a given context that were
+  // present in the index at the time this method was called. If aAddNew is true
+  // then the iterator will also return entries created after this call.
+  // NOTE: When some entry is removed from index it is removed also from the
+  // iterator regardless what aAddNew was passed.
+  static nsresult GetIterator(nsILoadContextInfo *aInfo, bool aAddNew,
+                              CacheIndexIterator **_retval);
+
+  // Returns true if we _think_ that the index is up to date. I.e. the state is
+  // READY or WRITING and mIndexNeedsUpdate as well as mShuttingDown is false.
+  static nsresult IsUpToDate(bool *_retval);
+
+  // Called from CacheStorageService::Clear() and CacheFileContextEvictor::EvictEntries(),
+  // sets a flag that blocks notification to AsyncGetDiskConsumption.
+  static void OnAsyncEviction(bool aEvicting);
+
+  // Memory reporting
+  static size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf);
+  static size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf);
+
+private:
+  friend class CacheIndexEntryAutoManage;
+  friend class FileOpenHelper;
+  friend class CacheIndexIterator;
+
+  virtual ~CacheIndex();
+
+  NS_IMETHOD OnFileOpened(CacheFileHandle *aHandle, nsresult aResult) override;
+  nsresult   OnFileOpenedInternal(FileOpenHelper *aOpener,
+                                  CacheFileHandle *aHandle, nsresult aResult);
+  NS_IMETHOD OnDataWritten(CacheFileHandle *aHandle, const char *aBuf,
+                           nsresult aResult) override;
+  NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult) override;
+  NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult) override;
+  NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult) override;
+  NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult) override;
+
+  nsresult InitInternal(nsIFile *aCacheDirectory);
+  void     PreShutdownInternal();
+
+  // This method returns false when index is not initialized or is shut down.
+  bool IsIndexUsable();
+
+  // This method checks whether the entry has the same values of
+  // originAttributes and isAnonymous. We don't expect to find a collision
+  // since these values are part of the key that we hash and we use a strong
+  // hash function.
+  static bool IsCollision(CacheIndexEntry *aEntry,
+                          OriginAttrsHash  aOriginAttrsHash,
+                          bool             aAnonymous);
+
+  // Checks whether any of the information about the entry has changed.
+  static bool HasEntryChanged(CacheIndexEntry *aEntry,
+                              const uint32_t  *aFrecency,
+                              const uint32_t  *aExpirationTime,
+                              const uint32_t  *aSize);
+
+  // Merge all pending operations from mPendingUpdates into mIndex.
+  void ProcessPendingOperations();
+
+  // Following methods perform writing of the index file.
+  //
+  // The index is written periodically, but not earlier than once in
+  // kMinDumpInterval and there must be at least kMinUnwrittenChanges
+  // differences between index on disk and in memory. Index is always first
+  // written to a temporary file and the old index file is replaced when the
+  // writing process succeeds.
+  //
+  // Starts writing of index when both limits (minimal delay between writes and
+  // minimum number of changes in index) were exceeded.
+  bool WriteIndexToDiskIfNeeded();
+  // Starts writing of index file.
+  void WriteIndexToDisk();
+  // Serializes part of mIndex hashtable to the write buffer a writes the buffer
+  // to the file.
+  void WriteRecords();
+  // Finalizes writing process.
+  void FinishWrite(bool aSucceeded);
+
+  // Following methods perform writing of the journal during shutdown. All these
+  // methods must be called only during shutdown since they write/delete files
+  // directly on the main thread instead of using CacheFileIOManager that does
+  // it asynchronously on IO thread. Journal contains only entries that are
+  // dirty, i.e. changes that are not present in the index file on the disk.
+  // When the log is written successfully, the dirty flag in index file is
+  // cleared.
+  nsresult GetFile(const nsACString &aName, nsIFile **_retval);
+  nsresult RemoveFile(const nsACString &aName);
+  void     RemoveAllIndexFiles();
+  void     RemoveJournalAndTempFile();
+  // Writes journal to the disk and clears dirty flag in index header.
+  nsresult WriteLogToDisk();
+
+  // Following methods perform reading of the index from the disk.
+  //
+  // Index is read at startup just after initializing the CacheIndex. There are
+  // 3 files used when manipulating with index: index file, journal file and
+  // a temporary file. All files contain the hash of the data, so we can check
+  // whether the content is valid and complete. Index file contains also a dirty
+  // flag in the index header which is unset on a clean shutdown. During opening
+  // and reading of the files we determine the status of the whole index from
+  // the states of the separate files. Following table shows all possible
+  // combinations:
+  //
+  // index, journal, tmpfile
+  // M      *        *       - index is missing    -> BUILD
+  // I      *        *       - index is invalid    -> BUILD
+  // D      *        *       - index is dirty      -> UPDATE
+  // C      M        *       - index is dirty      -> UPDATE
+  // C      I        *       - unexpected state    -> UPDATE
+  // C      V        E       - unexpected state    -> UPDATE
+  // C      V        M       - index is up to date -> READY
+  //
+  // where the letters mean:
+  //   * - any state
+  //   E - file exists
+  //   M - file is missing
+  //   I - data is invalid (parsing failed or hash didn't match)
+  //   D - dirty (data in index file is correct, but dirty flag is set)
+  //   C - clean (index file is clean)
+  //   V - valid (data in journal file is correct)
+  //
+  // Note: We accept the data from journal only when the index is up to date as
+  // a whole (i.e. C,V,M state).
+  //
+  // We rename the journal file to the temporary file as soon as possible after
+  // initial test to ensure that we start update process on the next startup if
+  // FF crashes during parsing of the index.
+  //
+  // Initiates reading index from disk.
+  void ReadIndexFromDisk();
+  // Starts reading data from index file.
+  void StartReadingIndex();
+  // Parses data read from index file.
+  void ParseRecords();
+  // Starts reading data from journal file.
+  void StartReadingJournal();
+  // Parses data read from journal file.
+  void ParseJournal();
+  // Merges entries from journal into mIndex.
+  void MergeJournal();
+  // In debug build this method checks that we have no fresh entry in mIndex
+  // after we finish reading index and before we process pending operations.
+  void EnsureNoFreshEntry();
+  // In debug build this method is called after processing pending operations
+  // to make sure mIndexStats contains correct information.
+  void EnsureCorrectStats();
+  // Finalizes reading process.
+  void FinishRead(bool aSucceeded);
+
+  // Following methods perform updating and building of the index.
+  // Timer callback that starts update or build process.
+  static void DelayedUpdate(nsITimer *aTimer, void *aClosure);
+  void DelayedUpdateLocked();
+  // Posts timer event that start update or build process.
+  nsresult ScheduleUpdateTimer(uint32_t aDelay);
+  nsresult SetupDirectoryEnumerator();
+  void InitEntryFromDiskData(CacheIndexEntry *aEntry,
+                             CacheFileMetadata *aMetaData,
+                             int64_t aFileSize);
+  // Returns true when either a timer is scheduled or event is posted.
+  bool IsUpdatePending();
+  // Iterates through all files in entries directory that we didn't create/open
+  // during this session, parses them and adds the entries to the index.
+  void BuildIndex();
+
+  bool StartUpdatingIndexIfNeeded(bool aSwitchingToReadyState = false);
+  // Starts update or build process or fires a timer when it is too early after
+  // startup.
+  void StartUpdatingIndex(bool aRebuild);
+  // Iterates through all files in entries directory that we didn't create/open
+  // during this session and theirs last modified time is newer than timestamp
+  // in the index header. Parses the files and adds the entries to the index.
+  void UpdateIndex();
+  // Finalizes update or build process.
+  void FinishUpdate(bool aSucceeded);
+
+  void RemoveNonFreshEntries();
+
+  enum EState {
+    // Initial state in which the index is not usable
+    // Possible transitions:
+    //  -> READING
+    INITIAL  = 0,
+
+    // Index is being read from the disk.
+    // Possible transitions:
+    //  -> INITIAL  - We failed to dispatch a read event.
+    //  -> BUILDING - No or corrupted index file was found.
+    //  -> UPDATING - No or corrupted journal file was found.
+    //              - Dirty flag was set in index header.
+    //  -> READY    - Index was read successfully or was interrupted by
+    //                pre-shutdown.
+    //  -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
+    READING  = 1,
+
+    // Index is being written to the disk.
+    // Possible transitions:
+    //  -> READY    - Writing of index finished or was interrupted by
+    //                pre-shutdown..
+    //  -> UPDATING - Writing of index finished, but index was found outdated
+    //                during writing.
+    //  -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
+    WRITING  = 2,
+
+    // Index is being build.
+    // Possible transitions:
+    //  -> READY    - Building of index finished or was interrupted by
+    //                pre-shutdown.
+    //  -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
+    BUILDING = 3,
+
+    // Index is being updated.
+    // Possible transitions:
+    //  -> READY    - Updating of index finished or was interrupted by
+    //                pre-shutdown.
+    //  -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
+    UPDATING = 4,
+
+    // Index is ready.
+    // Possible transitions:
+    //  -> UPDATING - Index was found outdated.
+    //  -> SHUTDOWN - Index is shutting down.
+    READY    = 5,
+
+    // Index is shutting down.
+    SHUTDOWN = 6
+  };
+
+  static char const * StateString(EState aState);
+  void ChangeState(EState aNewState);
+  void NotifyAsyncGetDiskConsumptionCallbacks();
+
+  // Allocates and releases buffer used for reading and writing index.
+  void AllocBuffer();
+  void ReleaseBuffer();
+
+  // Methods used by CacheIndexEntryAutoManage to keep the iterators up to date.
+  void AddRecordToIterators(CacheIndexRecord *aRecord);
+  void RemoveRecordFromIterators(CacheIndexRecord *aRecord);
+  void ReplaceRecordInIterators(CacheIndexRecord *aOldRecord,
+                                CacheIndexRecord *aNewRecord);
+
+  // Memory reporting (private part)
+  size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const;
+
+  void ReportHashStats();
+
+  static mozilla::StaticRefPtr<CacheIndex> gInstance;
+  static StaticMutex sLock;
+
+  nsCOMPtr<nsIFile> mCacheDirectory;
+
+  EState         mState;
+  // Timestamp of time when the index was initialized. We use it to delay
+  // initial update or build of index.
+  TimeStamp      mStartTime;
+  // Set to true in PreShutdown(), it is checked on variaous places to prevent
+  // starting any process (write, update, etc.) during shutdown.
+  bool           mShuttingDown;
+  // When set to true, update process should start as soon as possible. This
+  // flag is set whenever we find some inconsistency which would be fixed by
+  // update process. The flag is checked always when switching to READY state.
+  // To make sure we start the update process as soon as possible, methods that
+  // set this flag should also call StartUpdatingIndexIfNeeded() to cover the
+  // case when we are currently in READY state.
+  bool           mIndexNeedsUpdate;
+  // Set at the beginning of RemoveAll() which clears the whole index. When
+  // removing all entries we must stop any pending reading, writing, updating or
+  // building operation. This flag is checked at various places and it prevents
+  // we won't start another operation (e.g. canceling reading of the index would
+  // normally start update or build process)
+  bool           mRemovingAll;
+  // Whether the index file on disk exists and is valid.
+  bool           mIndexOnDiskIsValid;
+  // When something goes wrong during updating or building process, we don't
+  // mark index clean (and also don't write journal) to ensure that update or
+  // build will be initiated on the next start.
+  bool           mDontMarkIndexClean;
+  // Timestamp value from index file. It is used during update process to skip
+  // entries that were last modified before this timestamp.
+  uint32_t       mIndexTimeStamp;
+  // Timestamp of last time the index was dumped to disk.
+  // NOTE: The index might not be necessarily dumped at this time. The value
+  // is used to schedule next dump of the index.
+  TimeStamp      mLastDumpTime;
+
+  // Timer of delayed update/build.
+  nsCOMPtr<nsITimer> mUpdateTimer;
+  // True when build or update event is posted
+  bool               mUpdateEventPending;
+
+  // Helper members used when reading/writing index from/to disk.
+  // Contains number of entries that should be skipped:
+  //  - in hashtable when writing index because they were already written
+  //  - in index file when reading index because they were already read
+  uint32_t                  mSkipEntries;
+  // Number of entries that should be written to disk. This is number of entries
+  // in hashtable that are initialized and are not marked as removed when writing
+  // begins.
+  uint32_t                  mProcessEntries;
+  char                     *mRWBuf;
+  uint32_t                  mRWBufSize;
+  uint32_t                  mRWBufPos;
+  RefPtr<CacheHash>         mRWHash;
+
+  // True if read or write operation is pending. It is used to ensure that
+  // mRWBuf is not freed until OnDataRead or OnDataWritten is called.
+  bool                      mRWPending;
+
+  // Reading of journal succeeded if true.
+  bool                      mJournalReadSuccessfully;
+
+  // Handle used for writing and reading index file.
+  RefPtr<CacheFileHandle> mIndexHandle;
+  // Handle used for reading journal file.
+  RefPtr<CacheFileHandle> mJournalHandle;
+  // Used to check the existence of the file during reading process.
+  RefPtr<CacheFileHandle> mTmpHandle;
+
+  RefPtr<FileOpenHelper>    mIndexFileOpener;
+  RefPtr<FileOpenHelper>    mJournalFileOpener;
+  RefPtr<FileOpenHelper>    mTmpFileOpener;
+
+  // Directory enumerator used when building and updating index.
+  nsCOMPtr<nsIDirectoryEnumerator> mDirEnumerator;
+
+  // Main index hashtable.
+  nsTHashtable<CacheIndexEntry> mIndex;
+
+  // We cannot add, remove or change any entry in mIndex in states READING and
+  // WRITING. We track all changes in mPendingUpdates during these states.
+  nsTHashtable<CacheIndexEntryUpdate> mPendingUpdates;
+
+  // Contains information statistics for mIndex + mPendingUpdates.
+  CacheIndexStats               mIndexStats;
+
+  // When reading journal, we must first parse the whole file and apply the
+  // changes iff the journal was read successfully. mTmpJournal is used to store
+  // entries from the journal file. We throw away all these entries if parsing
+  // of the journal fails or the hash does not match.
+  nsTHashtable<CacheIndexEntry> mTmpJournal;
+
+  // FrecencyArray maintains order of entry records for eviction. Ideally, the
+  // records would be ordered by frecency all the time, but since this would be
+  // quite expensive, we allow certain amount of entries to be out of order.
+  // When the frecency is updated the new value is always bigger than the old
+  // one. Instead of keeping updated entries at the same position, we move them
+  // at the end of the array. This protects recently updated entries from
+  // eviction. The array is sorted once we hit the limit of maximum unsorted
+  // entries.
+  class FrecencyArray
+  {
+    class Iterator
+    {
+    public:
+      explicit Iterator(nsTArray<CacheIndexRecord *> *aRecs)
+        : mRecs(aRecs)
+        , mIdx(0)
+      {
+        while (!Done() && !(*mRecs)[mIdx]) {
+          mIdx++;
+        }
+      }
+
+      bool Done() const { return mIdx == mRecs->Length(); }
+
+      CacheIndexRecord* Get() const
+      {
+        MOZ_ASSERT(!Done());
+        return (*mRecs)[mIdx];
+      }
+
+      void Next()
+      {
+        MOZ_ASSERT(!Done());
+        ++mIdx;
+        while (!Done() && !(*mRecs)[mIdx]) {
+          mIdx++;
+        }
+      }
+
+    private:
+      nsTArray<CacheIndexRecord *> *mRecs;
+      uint32_t mIdx;
+    };
+
+  public:
+    Iterator Iter() { return Iterator(&mRecs); }
+
+    FrecencyArray() : mUnsortedElements(0)
+                    , mRemovedElements(0) {}
+
+    // Methods used by CacheIndexEntryAutoManage to keep the array up to date.
+    void AppendRecord(CacheIndexRecord *aRecord);
+    void RemoveRecord(CacheIndexRecord *aRecord);
+    void ReplaceRecord(CacheIndexRecord *aOldRecord,
+                       CacheIndexRecord *aNewRecord);
+    void SortIfNeeded();
+
+    size_t Length() const { return mRecs.Length() - mRemovedElements; }
+    void Clear() { mRecs.Clear(); }
+
+  private:
+    friend class CacheIndex;
+
+    nsTArray<CacheIndexRecord *> mRecs;
+    uint32_t                     mUnsortedElements;
+    // Instead of removing elements from the array immediately, we null them out
+    // and the iterator skips them when accessing the array. The null pointers
+    // are placed at the end during sorting and we strip them out all at once.
+    // This saves moving a lot of memory in nsTArray::RemoveElementsAt.
+    uint32_t                     mRemovedElements;
+  };
+
+  FrecencyArray mFrecencyArray;
+
+  nsTArray<CacheIndexIterator *> mIterators;
+
+  // This flag is true iff we are between CacheStorageService:Clear() and processing
+  // all contexts to be evicted.  It will make UI to show "calculating" instead of
+  // any intermediate cache size.
+  bool mAsyncGetDiskConsumptionBlocked;
+
+  class DiskConsumptionObserver : public Runnable
+  {
+  public:
+    static DiskConsumptionObserver* Init(nsICacheStorageConsumptionObserver* aObserver)
+    {
+      nsWeakPtr observer = do_GetWeakReference(aObserver);
+      if (!observer)
+        return nullptr;
+
+      return new DiskConsumptionObserver(observer);
+    }
+
+    void OnDiskConsumption(int64_t aSize)
+    {
+      mSize = aSize;
+      NS_DispatchToMainThread(this);
+    }
+
+  private:
+    explicit DiskConsumptionObserver(nsWeakPtr const &aWeakObserver)
+      : mObserver(aWeakObserver) { }
+    virtual ~DiskConsumptionObserver() {
+      if (mObserver && !NS_IsMainThread()) {
+        NS_ReleaseOnMainThread(mObserver.forget());
+      }
+    }
+
+    NS_IMETHOD Run() override
+    {
+      MOZ_ASSERT(NS_IsMainThread());
+
+      nsCOMPtr<nsICacheStorageConsumptionObserver> observer =
+        do_QueryReferent(mObserver);
+
+      mObserver = nullptr;
+
+      if (observer) {
+        observer->OnNetworkCacheDiskConsumption(mSize);
+      }
+
+      return NS_OK;
+    }
+
+    nsWeakPtr mObserver;
+    int64_t mSize;
+  };
+
+  // List of async observers that want to get disk consumption information
+  nsTArray<RefPtr<DiskConsumptionObserver> > mDiskConsumptionObservers;
+};
+
+} // namespace net
+} // namespace mozilla
+
+#endif