diff options
Diffstat (limited to 'toolkit/components/url-classifier')
148 files changed, 41435 insertions, 0 deletions
diff --git a/toolkit/components/url-classifier/ChunkSet.cpp b/toolkit/components/url-classifier/ChunkSet.cpp new file mode 100644 index 000000000..162a74260 --- /dev/null +++ b/toolkit/components/url-classifier/ChunkSet.cpp @@ -0,0 +1,278 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ChunkSet.h" + +namespace mozilla { +namespace safebrowsing { + +const size_t ChunkSet::IO_BUFFER_SIZE; + +nsresult +ChunkSet::Serialize(nsACString& aChunkStr) +{ + aChunkStr.Truncate(); + for (const Range* range = mRanges.begin(); range != mRanges.end(); range++) { + if (range != mRanges.begin()) { + aChunkStr.Append(','); + } + + aChunkStr.AppendInt((int32_t)range->Begin()); + if (range->Begin() != range->End()) { + aChunkStr.Append('-'); + aChunkStr.AppendInt((int32_t)range->End()); + } + } + + return NS_OK; +} + +nsresult +ChunkSet::Set(uint32_t aChunk) +{ + if (!Has(aChunk)) { + Range chunkRange(aChunk, aChunk); + + if (mRanges.Length() == 0) { + if (!mRanges.AppendElement(chunkRange, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + return NS_OK; + } + + if (mRanges.LastElement().Precedes(chunkRange)) { + mRanges.LastElement().End(aChunk); + } else if (chunkRange.Precedes(mRanges[0])) { + mRanges[0].Begin(aChunk); + } else { + ChunkSet tmp; + if (!tmp.mRanges.AppendElement(chunkRange, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + return Merge(tmp); + } + } + + return NS_OK; +} + +bool +ChunkSet::Has(uint32_t aChunk) const +{ + size_t idx; + return BinarySearchIf(mRanges, 0, mRanges.Length(), + Range::IntersectionComparator(Range(aChunk, aChunk)), + &idx); + // IntersectionComparator works because we create a + // single-chunk range. +} + +nsresult +ChunkSet::Merge(const ChunkSet& aOther) +{ + size_t oldLen = mRanges.Length(); + + for (const Range* mergeRange = aOther.mRanges.begin(); + mergeRange != aOther.mRanges.end(); mergeRange++) { + if (!HasSubrange(*mergeRange)) { + if (!mRanges.InsertElementSorted(*mergeRange, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + } + } + + if (oldLen < mRanges.Length()) { + for (size_t i = 1; i < mRanges.Length(); i++) { + while (mRanges[i - 1].FoldLeft(mRanges[i])) { + mRanges.RemoveElementAt(i); + + if (i == mRanges.Length()) { + return NS_OK; + } + } + } + } + + return NS_OK; +} + +uint32_t +ChunkSet::Length() const +{ + uint32_t len = 0; + for (const Range* range = mRanges.begin(); range != mRanges.end(); range++) { + len += range->Length(); + } + + return len; +} + +nsresult +ChunkSet::Remove(const ChunkSet& aOther) +{ + for (const Range* removalRange = aOther.mRanges.begin(); + removalRange != aOther.mRanges.end(); removalRange++) { + + if (mRanges.Length() == 0) { + return NS_OK; + } + + if (mRanges.LastElement().End() < removalRange->Begin() || + aOther.mRanges.LastElement().End() < mRanges[0].Begin()) { + return NS_OK; + } + + size_t intersectionIdx; + while (BinarySearchIf(mRanges, 0, mRanges.Length(), + Range::IntersectionComparator(*removalRange), &intersectionIdx)) { + + ChunkSet remains; + nsresult rv = mRanges[intersectionIdx].Remove(*removalRange, remains); + + if (NS_FAILED(rv)) { + return rv; + } + + mRanges.RemoveElementAt(intersectionIdx); + if (!mRanges.InsertElementsAt(intersectionIdx, remains.mRanges, + fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + } + } + + return NS_OK; +} + +void +ChunkSet::Clear() +{ + mRanges.Clear(); +} + +nsresult +ChunkSet::Write(nsIOutputStream* aOut) +{ + nsTArray<uint32_t> chunks(IO_BUFFER_SIZE); + + for (const Range* range = mRanges.begin(); range != mRanges.end(); range++) { + for (uint32_t chunk = range->Begin(); chunk <= range->End(); chunk++) { + chunks.AppendElement(chunk); + + if (chunks.Length() == chunks.Capacity()) { + nsresult rv = WriteTArray(aOut, chunks); + + if (NS_FAILED(rv)) { + return rv; + } + + chunks.Clear(); + } + } + } + + nsresult rv = WriteTArray(aOut, chunks); + + if (NS_FAILED(rv)) { + return rv; + } + + return NS_OK; +} + +nsresult +ChunkSet::Read(nsIInputStream* aIn, uint32_t aNumElements) +{ + nsTArray<uint32_t> chunks(IO_BUFFER_SIZE); + + while (aNumElements != 0) { + chunks.Clear(); + + uint32_t numToRead = aNumElements > IO_BUFFER_SIZE ? IO_BUFFER_SIZE : aNumElements; + + nsresult rv = ReadTArray(aIn, &chunks, numToRead); + + if (NS_FAILED(rv)) { + return rv; + } + + aNumElements -= numToRead; + + for (const uint32_t* c = chunks.begin(); c != chunks.end(); c++) { + rv = Set(*c); + + if (NS_FAILED(rv)) { + return rv; + } + } + } + + return NS_OK; +} + +bool +ChunkSet::HasSubrange(const Range& aSubrange) const +{ + for (const Range* range = mRanges.begin(); range != mRanges.end(); range++) { + if (range->Contains(aSubrange)) { + return true; + } else if (!(aSubrange.Begin() > range->End() || + range->Begin() > aSubrange.End())) { + // In this case, aSubrange overlaps this range but is not a subrange. + // because the ChunkSet implementation ensures that there are no + // overlapping ranges, this means that aSubrange cannot be a subrange of + // any of the following ranges + return false; + } + } + + return false; +} + +uint32_t +ChunkSet::Range::Length() const +{ + return mEnd - mBegin + 1; +} + +nsresult +ChunkSet::Range::Remove(const Range& aRange, ChunkSet& aRemainderSet) const +{ + if (mBegin < aRange.mBegin && aRange.mBegin <= mEnd) { + // aRange overlaps & follows this range + Range range(mBegin, aRange.mBegin - 1); + if (!aRemainderSet.mRanges.AppendElement(range, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + } + + if (mBegin <= aRange.mEnd && aRange.mEnd < mEnd) { + // aRange overlaps & precedes this range + Range range(aRange.mEnd + 1, mEnd); + if (!aRemainderSet.mRanges.AppendElement(range, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + } + + return NS_OK; +} + +bool +ChunkSet::Range::FoldLeft(const Range& aRange) +{ + if (Contains(aRange)) { + return true; + } else if (Precedes(aRange) || + (mBegin <= aRange.mBegin && aRange.mBegin <= mEnd)) { + mEnd = aRange.mEnd; + return true; + } + + return false; +} + +} // namespace safebrowsing +} // namespace mozilla diff --git a/toolkit/components/url-classifier/ChunkSet.h b/toolkit/components/url-classifier/ChunkSet.h new file mode 100644 index 000000000..fc7c1eb15 --- /dev/null +++ b/toolkit/components/url-classifier/ChunkSet.h @@ -0,0 +1,99 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ChunkSet_h__ +#define ChunkSet_h__ + +#include "Entries.h" +#include "nsString.h" +#include "nsTArray.h" + +namespace mozilla { +namespace safebrowsing { + +/** + * Store the chunk numbers as an array of ranges of uint32_t. + * We need chunk numbers in order to ask for incremental updates from the + * server. + */ +class ChunkSet { +public: + nsresult Serialize(nsACString& aStr); + nsresult Set(uint32_t aChunk); + bool Has(uint32_t chunk) const; + nsresult Merge(const ChunkSet& aOther); + uint32_t Length() const; + nsresult Remove(const ChunkSet& aOther); + void Clear(); + + nsresult Write(nsIOutputStream* aOut); + nsresult Read(nsIInputStream* aIn, uint32_t aNumElements); + +private: + class Range { + public: + Range(uint32_t aBegin, uint32_t aEnd) : mBegin(aBegin), mEnd(aEnd) {} + + uint32_t Length() const; + nsresult Remove(const Range& aRange, ChunkSet& aRemainderSet) const; + bool FoldLeft(const Range& aRange); + + bool operator==(const Range& rhs) const { + return mBegin == rhs.mBegin; + } + bool operator<(const Range& rhs) const { + return mBegin < rhs.mBegin; + } + + uint32_t Begin() const { + return mBegin; + } + void Begin(const uint32_t aBegin) { + mBegin = aBegin; + } + uint32_t End() const { + return mEnd; + } + void End(const uint32_t aEnd) { + mEnd = aEnd; + } + + bool Contains(const Range& aRange) const { + return mBegin <= aRange.mBegin && aRange.mEnd <= mEnd; + } + bool Precedes(const Range& aRange) const { + return mEnd + 1 == aRange.mBegin; + } + + struct IntersectionComparator { + int operator()(const Range& aRange) const { + if (aRange.mBegin > mTarget.mEnd) { + return -1; + } + if (mTarget.mBegin > aRange.mEnd) { + return 1; + } + return 0; + } + + explicit IntersectionComparator(const Range& aTarget) : mTarget(aTarget){} + const Range& mTarget; + }; + + private: + uint32_t mBegin; + uint32_t mEnd; + }; + + static const size_t IO_BUFFER_SIZE = 1024; + FallibleTArray<Range> mRanges; + + bool HasSubrange(const Range& aSubrange) const; +}; + +} // namespace safebrowsing +} // namespace mozilla + +#endif diff --git a/toolkit/components/url-classifier/Classifier.cpp b/toolkit/components/url-classifier/Classifier.cpp new file mode 100644 index 000000000..bbaeaf535 --- /dev/null +++ b/toolkit/components/url-classifier/Classifier.cpp @@ -0,0 +1,1281 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "Classifier.h" +#include "LookupCacheV4.h" +#include "nsIPrefBranch.h" +#include "nsIPrefService.h" +#include "nsISimpleEnumerator.h" +#include "nsIRandomGenerator.h" +#include "nsIInputStream.h" +#include "nsISeekableStream.h" +#include "nsIFile.h" +#include "nsNetCID.h" +#include "nsPrintfCString.h" +#include "nsThreadUtils.h" +#include "mozilla/Telemetry.h" +#include "mozilla/Logging.h" +#include "mozilla/SyncRunnable.h" +#include "mozilla/Base64.h" +#include "mozilla/Unused.h" +#include "mozilla/TypedEnumBits.h" +#include "nsIUrlClassifierUtils.h" +#include "nsUrlClassifierDBService.h" + +// MOZ_LOG=UrlClassifierDbService:5 +extern mozilla::LazyLogModule gUrlClassifierDbServiceLog; +#define LOG(args) MOZ_LOG(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug, args) +#define LOG_ENABLED() MOZ_LOG_TEST(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug) + +#define STORE_DIRECTORY NS_LITERAL_CSTRING("safebrowsing") +#define TO_DELETE_DIR_SUFFIX NS_LITERAL_CSTRING("-to_delete") +#define BACKUP_DIR_SUFFIX NS_LITERAL_CSTRING("-backup") + +#define METADATA_SUFFIX NS_LITERAL_CSTRING(".metadata") + +namespace mozilla { +namespace safebrowsing { + +namespace { + +// A scoped-clearer for nsTArray<TableUpdate*>. +// The owning elements will be deleted and the array itself +// will be cleared on exiting the scope. +class ScopedUpdatesClearer { +public: + explicit ScopedUpdatesClearer(nsTArray<TableUpdate*> *aUpdates) + : mUpdatesArrayRef(aUpdates) + { + for (auto update : *aUpdates) { + mUpdatesPointerHolder.AppendElement(update); + } + } + + ~ScopedUpdatesClearer() + { + mUpdatesArrayRef->Clear(); + } + +private: + nsTArray<TableUpdate*>* mUpdatesArrayRef; + nsTArray<nsAutoPtr<TableUpdate>> mUpdatesPointerHolder; +}; + +} // End of unnamed namespace. + +void +Classifier::SplitTables(const nsACString& str, nsTArray<nsCString>& tables) +{ + tables.Clear(); + + nsACString::const_iterator begin, iter, end; + str.BeginReading(begin); + str.EndReading(end); + while (begin != end) { + iter = begin; + FindCharInReadable(',', iter, end); + nsDependentCSubstring table = Substring(begin,iter); + if (!table.IsEmpty()) { + tables.AppendElement(Substring(begin, iter)); + } + begin = iter; + if (begin != end) { + begin++; + } + } +} + +nsresult +Classifier::GetPrivateStoreDirectory(nsIFile* aRootStoreDirectory, + const nsACString& aTableName, + const nsACString& aProvider, + nsIFile** aPrivateStoreDirectory) +{ + NS_ENSURE_ARG_POINTER(aPrivateStoreDirectory); + + if (!StringEndsWith(aTableName, NS_LITERAL_CSTRING("-proto"))) { + // Only V4 table names (ends with '-proto') would be stored + // to per-provider sub-directory. + nsCOMPtr<nsIFile>(aRootStoreDirectory).forget(aPrivateStoreDirectory); + return NS_OK; + } + + if (aProvider.IsEmpty()) { + // When failing to get provider, just store in the root folder. + nsCOMPtr<nsIFile>(aRootStoreDirectory).forget(aPrivateStoreDirectory); + return NS_OK; + } + + nsCOMPtr<nsIFile> providerDirectory; + + // Clone first since we are gonna create a new directory. + nsresult rv = aRootStoreDirectory->Clone(getter_AddRefs(providerDirectory)); + NS_ENSURE_SUCCESS(rv, rv); + + // Append the provider name to the root store directory. + rv = providerDirectory->AppendNative(aProvider); + NS_ENSURE_SUCCESS(rv, rv); + + // Ensure existence of the provider directory. + bool dirExists; + rv = providerDirectory->Exists(&dirExists); + NS_ENSURE_SUCCESS(rv, rv); + + if (!dirExists) { + LOG(("Creating private directory for %s", nsCString(aTableName).get())); + rv = providerDirectory->Create(nsIFile::DIRECTORY_TYPE, 0755); + NS_ENSURE_SUCCESS(rv, rv); + providerDirectory.forget(aPrivateStoreDirectory); + return rv; + } + + // Store directory exists. Check if it's a directory. + bool isDir; + rv = providerDirectory->IsDirectory(&isDir); + NS_ENSURE_SUCCESS(rv, rv); + if (!isDir) { + return NS_ERROR_FILE_DESTINATION_NOT_DIR; + } + + providerDirectory.forget(aPrivateStoreDirectory); + + return NS_OK; +} + +Classifier::Classifier() +{ +} + +Classifier::~Classifier() +{ + Close(); +} + +nsresult +Classifier::SetupPathNames() +{ + // Get the root directory where to store all the databases. + nsresult rv = mCacheDirectory->Clone(getter_AddRefs(mRootStoreDirectory)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mRootStoreDirectory->AppendNative(STORE_DIRECTORY); + NS_ENSURE_SUCCESS(rv, rv); + + // Make sure LookupCaches (which are persistent and survive updates) + // are reading/writing in the right place. We will be moving their + // files "underneath" them during backup/restore. + for (uint32_t i = 0; i < mLookupCaches.Length(); i++) { + mLookupCaches[i]->UpdateRootDirHandle(mRootStoreDirectory); + } + + // Directory where to move a backup before an update. + rv = mCacheDirectory->Clone(getter_AddRefs(mBackupDirectory)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mBackupDirectory->AppendNative(STORE_DIRECTORY + BACKUP_DIR_SUFFIX); + NS_ENSURE_SUCCESS(rv, rv); + + // Directory where to move the backup so we can atomically + // delete (really move) it. + rv = mCacheDirectory->Clone(getter_AddRefs(mToDeleteDirectory)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mToDeleteDirectory->AppendNative(STORE_DIRECTORY + TO_DELETE_DIR_SUFFIX); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +Classifier::CreateStoreDirectory() +{ + // Ensure the safebrowsing directory exists. + bool storeExists; + nsresult rv = mRootStoreDirectory->Exists(&storeExists); + NS_ENSURE_SUCCESS(rv, rv); + + if (!storeExists) { + rv = mRootStoreDirectory->Create(nsIFile::DIRECTORY_TYPE, 0755); + NS_ENSURE_SUCCESS(rv, rv); + } else { + bool storeIsDir; + rv = mRootStoreDirectory->IsDirectory(&storeIsDir); + NS_ENSURE_SUCCESS(rv, rv); + if (!storeIsDir) + return NS_ERROR_FILE_DESTINATION_NOT_DIR; + } + + return NS_OK; +} + +nsresult +Classifier::Open(nsIFile& aCacheDirectory) +{ + // Remember the Local profile directory. + nsresult rv = aCacheDirectory.Clone(getter_AddRefs(mCacheDirectory)); + NS_ENSURE_SUCCESS(rv, rv); + + // Create the handles to the update and backup directories. + rv = SetupPathNames(); + NS_ENSURE_SUCCESS(rv, rv); + + // Clean up any to-delete directories that haven't been deleted yet. + rv = CleanToDelete(); + NS_ENSURE_SUCCESS(rv, rv); + + // Check whether we have an incomplete update and recover from the + // backup if so. + rv = RecoverBackups(); + NS_ENSURE_SUCCESS(rv, rv); + + // Make sure the main store directory exists. + rv = CreateStoreDirectory(); + NS_ENSURE_SUCCESS(rv, rv); + + mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + // Build the list of know urlclassifier lists + // XXX: Disk IO potentially on the main thread during startup + RegenActiveTables(); + + return NS_OK; +} + +void +Classifier::Close() +{ + DropStores(); +} + +void +Classifier::Reset() +{ + DropStores(); + + mRootStoreDirectory->Remove(true); + mBackupDirectory->Remove(true); + mToDeleteDirectory->Remove(true); + + CreateStoreDirectory(); + + mTableFreshness.Clear(); + RegenActiveTables(); +} + +void +Classifier::ResetTables(ClearType aType, const nsTArray<nsCString>& aTables) +{ + for (uint32_t i = 0; i < aTables.Length(); i++) { + LOG(("Resetting table: %s", aTables[i].get())); + // Spoil this table by marking it as no known freshness + mTableFreshness.Remove(aTables[i]); + LookupCache *cache = GetLookupCache(aTables[i]); + if (cache) { + // Remove any cached Completes for this table if clear type is Clear_Cache + if (aType == Clear_Cache) { + cache->ClearCache(); + } else { + cache->ClearAll(); + } + } + } + + // Clear on-disk database if clear type is Clear_All + if (aType == Clear_All) { + DeleteTables(mRootStoreDirectory, aTables); + + RegenActiveTables(); + } +} + +void +Classifier::DeleteTables(nsIFile* aDirectory, const nsTArray<nsCString>& aTables) +{ + nsCOMPtr<nsISimpleEnumerator> entries; + nsresult rv = aDirectory->GetDirectoryEntries(getter_AddRefs(entries)); + NS_ENSURE_SUCCESS_VOID(rv); + + bool hasMore; + while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore) { + nsCOMPtr<nsISupports> supports; + rv = entries->GetNext(getter_AddRefs(supports)); + NS_ENSURE_SUCCESS_VOID(rv); + + nsCOMPtr<nsIFile> file = do_QueryInterface(supports); + NS_ENSURE_TRUE_VOID(file); + + // If |file| is a directory, recurse to find its entries as well. + bool isDirectory; + if (NS_FAILED(file->IsDirectory(&isDirectory))) { + continue; + } + if (isDirectory) { + DeleteTables(file, aTables); + continue; + } + + nsCString leafName; + rv = file->GetNativeLeafName(leafName); + NS_ENSURE_SUCCESS_VOID(rv); + + leafName.Truncate(leafName.RFind(".")); + if (aTables.Contains(leafName)) { + if (NS_FAILED(file->Remove(false))) { + NS_WARNING(nsPrintfCString("Fail to remove file %s from the disk", + leafName.get()).get()); + } + } + } + NS_ENSURE_SUCCESS_VOID(rv); +} + +void +Classifier::AbortUpdateAndReset(const nsCString& aTable) +{ + // We don't need to reset while shutting down. It will only slow us down. + if (nsUrlClassifierDBService::ShutdownHasStarted()) { + return; + } + + LOG(("Abort updating table %s.", aTable.get())); + + // ResetTables will clear both in-memory & on-disk data. + ResetTables(Clear_All, nsTArray<nsCString> { aTable }); + + // Remove the backup and delete directory since we are aborting + // from an update. + Unused << RemoveBackupTables(); + Unused << CleanToDelete(); +} + +void +Classifier::TableRequest(nsACString& aResult) +{ + // Generating v2 table info. + nsTArray<nsCString> tables; + ActiveTables(tables); + for (uint32_t i = 0; i < tables.Length(); i++) { + HashStore store(tables[i], GetProvider(tables[i]), mRootStoreDirectory); + + nsresult rv = store.Open(); + if (NS_FAILED(rv)) + continue; + + aResult.Append(store.TableName()); + aResult.Append(';'); + + ChunkSet &adds = store.AddChunks(); + ChunkSet &subs = store.SubChunks(); + + if (adds.Length() > 0) { + aResult.AppendLiteral("a:"); + nsAutoCString addList; + adds.Serialize(addList); + aResult.Append(addList); + } + + if (subs.Length() > 0) { + if (adds.Length() > 0) + aResult.Append(':'); + aResult.AppendLiteral("s:"); + nsAutoCString subList; + subs.Serialize(subList); + aResult.Append(subList); + } + + aResult.Append('\n'); + } + + // Load meta data from *.metadata files in the root directory. + // Specifically for v4 tables. + nsCString metadata; + nsresult rv = LoadMetadata(mRootStoreDirectory, metadata); + NS_ENSURE_SUCCESS_VOID(rv); + aResult.Append(metadata); +} + +// This is used to record the matching statistics for v2 and v4. +enum class PrefixMatch : uint8_t { + eNoMatch = 0x00, + eMatchV2Prefix = 0x01, + eMatchV4Prefix = 0x02, + eMatchBoth = eMatchV2Prefix | eMatchV4Prefix +}; + +MOZ_MAKE_ENUM_CLASS_BITWISE_OPERATORS(PrefixMatch) + +nsresult +Classifier::Check(const nsACString& aSpec, + const nsACString& aTables, + uint32_t aFreshnessGuarantee, + LookupResultArray& aResults) +{ + Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_CL_CHECK_TIME> timer; + + // Get the set of fragments based on the url. This is necessary because we + // only look up at most 5 URLs per aSpec, even if aSpec has more than 5 + // components. + nsTArray<nsCString> fragments; + nsresult rv = LookupCache::GetLookupFragments(aSpec, &fragments); + NS_ENSURE_SUCCESS(rv, rv); + + nsTArray<nsCString> activeTables; + SplitTables(aTables, activeTables); + + nsTArray<LookupCache*> cacheArray; + for (uint32_t i = 0; i < activeTables.Length(); i++) { + LOG(("Checking table %s", activeTables[i].get())); + LookupCache *cache = GetLookupCache(activeTables[i]); + if (cache) { + cacheArray.AppendElement(cache); + } else { + return NS_ERROR_FAILURE; + } + } + + PrefixMatch matchingStatistics = PrefixMatch::eNoMatch; + + // Now check each lookup fragment against the entries in the DB. + for (uint32_t i = 0; i < fragments.Length(); i++) { + Completion lookupHash; + lookupHash.FromPlaintext(fragments[i], mCryptoHash); + + if (LOG_ENABLED()) { + nsAutoCString checking; + lookupHash.ToHexString(checking); + LOG(("Checking fragment %s, hash %s (%X)", fragments[i].get(), + checking.get(), lookupHash.ToUint32())); + } + + for (uint32_t i = 0; i < cacheArray.Length(); i++) { + LookupCache *cache = cacheArray[i]; + bool has, complete; + + if (LookupCache::Cast<LookupCacheV4>(cache)) { + // TODO Bug 1312339 Return length in LookupCache.Has and support + // VariableLengthPrefix in LookupResultArray + rv = cache->Has(lookupHash, &has, &complete); + if (NS_FAILED(rv)) { + LOG(("Failed to lookup fragment %s V4", fragments[i].get())); + } + if (has) { + matchingStatistics |= PrefixMatch::eMatchV4Prefix; + // TODO: Bug 1311935 - Implement Safe Browsing v4 caching + // Should check cache expired + } + continue; + } + + rv = cache->Has(lookupHash, &has, &complete); + NS_ENSURE_SUCCESS(rv, rv); + if (has) { + LookupResult *result = aResults.AppendElement(); + if (!result) + return NS_ERROR_OUT_OF_MEMORY; + + int64_t age; + bool found = mTableFreshness.Get(cache->TableName(), &age); + if (!found) { + age = 24 * 60 * 60; // just a large number + } else { + int64_t now = (PR_Now() / PR_USEC_PER_SEC); + age = now - age; + } + + LOG(("Found a result in %s: %s (Age: %Lds)", + cache->TableName().get(), + complete ? "complete." : "Not complete.", + age)); + + result->hash.complete = lookupHash; + result->mComplete = complete; + result->mFresh = (age < aFreshnessGuarantee); + result->mTableName.Assign(cache->TableName()); + + matchingStatistics |= PrefixMatch::eMatchV2Prefix; + } + } + + Telemetry::Accumulate(Telemetry::URLCLASSIFIER_PREFIX_MATCH, + static_cast<uint8_t>(matchingStatistics)); + } + + return NS_OK; +} + +nsresult +Classifier::ApplyUpdates(nsTArray<TableUpdate*>* aUpdates) +{ + Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_CL_UPDATE_TIME> timer; + + PRIntervalTime clockStart = 0; + if (LOG_ENABLED()) { + clockStart = PR_IntervalNow(); + } + + nsresult rv; + + { + ScopedUpdatesClearer scopedUpdatesClearer(aUpdates); + + LOG(("Backup before update.")); + + rv = BackupTables(); + NS_ENSURE_SUCCESS(rv, rv); + + LOG(("Applying %d table updates.", aUpdates->Length())); + + for (uint32_t i = 0; i < aUpdates->Length(); i++) { + // Previous UpdateHashStore() may have consumed this update.. + if ((*aUpdates)[i]) { + // Run all updates for one table + nsCString updateTable(aUpdates->ElementAt(i)->TableName()); + + if (TableUpdate::Cast<TableUpdateV2>((*aUpdates)[i])) { + rv = UpdateHashStore(aUpdates, updateTable); + } else { + rv = UpdateTableV4(aUpdates, updateTable); + } + + if (NS_FAILED(rv)) { + if (rv != NS_ERROR_OUT_OF_MEMORY) { +#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES + DumpFailedUpdate(); +#endif + AbortUpdateAndReset(updateTable); + } + return rv; + } + } + } + + } // End of scopedUpdatesClearer scope. + + rv = RegenActiveTables(); + NS_ENSURE_SUCCESS(rv, rv); + + LOG(("Cleaning up backups.")); + + // Move the backup directory away (signaling the transaction finished + // successfully). This is atomic. + rv = RemoveBackupTables(); + NS_ENSURE_SUCCESS(rv, rv); + + // Do the actual deletion of the backup files. + rv = CleanToDelete(); + NS_ENSURE_SUCCESS(rv, rv); + + LOG(("Done applying updates.")); + + if (LOG_ENABLED()) { + PRIntervalTime clockEnd = PR_IntervalNow(); + LOG(("update took %dms\n", + PR_IntervalToMilliseconds(clockEnd - clockStart))); + } + + return NS_OK; +} + +nsresult +Classifier::ApplyFullHashes(nsTArray<TableUpdate*>* aUpdates) +{ + LOG(("Applying %d table gethashes.", aUpdates->Length())); + + ScopedUpdatesClearer scopedUpdatesClearer(aUpdates); + for (uint32_t i = 0; i < aUpdates->Length(); i++) { + TableUpdate *update = aUpdates->ElementAt(i); + + nsresult rv = UpdateCache(update); + NS_ENSURE_SUCCESS(rv, rv); + + aUpdates->ElementAt(i) = nullptr; + } + + return NS_OK; +} + +int64_t +Classifier::GetLastUpdateTime(const nsACString& aTableName) +{ + int64_t age; + bool found = mTableFreshness.Get(aTableName, &age); + return found ? (age * PR_MSEC_PER_SEC) : 0; +} + +void +Classifier::SetLastUpdateTime(const nsACString &aTable, + uint64_t updateTime) +{ + LOG(("Marking table %s as last updated on %u", + PromiseFlatCString(aTable).get(), updateTime)); + mTableFreshness.Put(aTable, updateTime / PR_MSEC_PER_SEC); +} + +void +Classifier::DropStores() +{ + for (uint32_t i = 0; i < mLookupCaches.Length(); i++) { + delete mLookupCaches[i]; + } + mLookupCaches.Clear(); +} + +nsresult +Classifier::RegenActiveTables() +{ + mActiveTablesCache.Clear(); + + nsTArray<nsCString> foundTables; + ScanStoreDir(foundTables); + + for (uint32_t i = 0; i < foundTables.Length(); i++) { + nsCString table(foundTables[i]); + HashStore store(table, GetProvider(table), mRootStoreDirectory); + + nsresult rv = store.Open(); + if (NS_FAILED(rv)) + continue; + + LookupCache *lookupCache = GetLookupCache(store.TableName()); + if (!lookupCache) { + continue; + } + + if (!lookupCache->IsPrimed()) + continue; + + const ChunkSet &adds = store.AddChunks(); + const ChunkSet &subs = store.SubChunks(); + + if (adds.Length() == 0 && subs.Length() == 0) + continue; + + LOG(("Active table: %s", store.TableName().get())); + mActiveTablesCache.AppendElement(store.TableName()); + } + + return NS_OK; +} + +nsresult +Classifier::ScanStoreDir(nsTArray<nsCString>& aTables) +{ + nsCOMPtr<nsISimpleEnumerator> entries; + nsresult rv = mRootStoreDirectory->GetDirectoryEntries(getter_AddRefs(entries)); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasMore; + while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore) { + nsCOMPtr<nsISupports> supports; + rv = entries->GetNext(getter_AddRefs(supports)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIFile> file = do_QueryInterface(supports); + + nsCString leafName; + rv = file->GetNativeLeafName(leafName); + NS_ENSURE_SUCCESS(rv, rv); + + nsCString suffix(NS_LITERAL_CSTRING(".sbstore")); + + int32_t dot = leafName.RFind(suffix, 0); + if (dot != -1) { + leafName.Cut(dot, suffix.Length()); + aTables.AppendElement(leafName); + } + } + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +Classifier::ActiveTables(nsTArray<nsCString>& aTables) +{ + aTables = mActiveTablesCache; + return NS_OK; +} + +nsresult +Classifier::CleanToDelete() +{ + bool exists; + nsresult rv = mToDeleteDirectory->Exists(&exists); + NS_ENSURE_SUCCESS(rv, rv); + + if (exists) { + rv = mToDeleteDirectory->Remove(true); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES + +already_AddRefed<nsIFile> +Classifier::GetFailedUpdateDirectroy() +{ + nsCString failedUpdatekDirName = STORE_DIRECTORY + nsCString("-failedupdate"); + + nsCOMPtr<nsIFile> failedUpdatekDirectory; + if (NS_FAILED(mCacheDirectory->Clone(getter_AddRefs(failedUpdatekDirectory))) || + NS_FAILED(failedUpdatekDirectory->AppendNative(failedUpdatekDirName))) { + LOG(("Failed to init failedUpdatekDirectory.")); + return nullptr; + } + + return failedUpdatekDirectory.forget(); +} + +nsresult +Classifier::DumpRawTableUpdates(const nsACString& aRawUpdates) +{ + // DumpFailedUpdate() MUST be called first to create the + // "failed update" directory + + LOG(("Dumping raw table updates...")); + + nsCOMPtr<nsIFile> failedUpdatekDirectory = GetFailedUpdateDirectroy(); + + // Create tableupdate.bin and dump raw table update data. + nsCOMPtr<nsIFile> rawTableUpdatesFile; + nsCOMPtr<nsIOutputStream> outputStream; + if (NS_FAILED(failedUpdatekDirectory->Clone(getter_AddRefs(rawTableUpdatesFile))) || + NS_FAILED(rawTableUpdatesFile->AppendNative(nsCString("tableupdates.bin"))) || + NS_FAILED(NS_NewLocalFileOutputStream(getter_AddRefs(outputStream), + rawTableUpdatesFile, + PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE))) { + LOG(("Failed to create file to dump raw table updates.")); + return NS_ERROR_FAILURE; + } + + // Write out the data. + uint32_t written; + nsresult rv = outputStream->Write(aRawUpdates.BeginReading(), + aRawUpdates.Length(), &written); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(written == aRawUpdates.Length(), NS_ERROR_FAILURE); + + return rv; +} + +nsresult +Classifier::DumpFailedUpdate() +{ + LOG(("Dumping failed update...")); + + nsCOMPtr<nsIFile> failedUpdatekDirectory = GetFailedUpdateDirectroy(); + + // Remove the "failed update" directory no matter it exists or not. + // Failure is fine because the directory may not exist. + failedUpdatekDirectory->Remove(true); + + nsCString failedUpdatekDirName; + nsresult rv = failedUpdatekDirectory->GetNativeLeafName(failedUpdatekDirName); + NS_ENSURE_SUCCESS(rv, rv); + + // Move backup to a clean "failed update" directory. + nsCOMPtr<nsIFile> backupDirectory; + if (NS_FAILED(mBackupDirectory->Clone(getter_AddRefs(backupDirectory))) || + NS_FAILED(backupDirectory->MoveToNative(nullptr, failedUpdatekDirName))) { + LOG(("Failed to move backup to the \"failed update\" directory %s", + failedUpdatekDirName.get())); + return NS_ERROR_FAILURE; + } + + return rv; +} + +#endif // MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES + +nsresult +Classifier::BackupTables() +{ + // We have to work in reverse here: first move the normal directory + // away to be the backup directory, then copy the files over + // to the normal directory. This ensures that if we crash the backup + // dir always has a valid, complete copy, instead of a partial one, + // because that's the one we will copy over the normal store dir. + + nsCString backupDirName; + nsresult rv = mBackupDirectory->GetNativeLeafName(backupDirName); + NS_ENSURE_SUCCESS(rv, rv); + + nsCString storeDirName; + rv = mRootStoreDirectory->GetNativeLeafName(storeDirName); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mRootStoreDirectory->MoveToNative(nullptr, backupDirName); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mRootStoreDirectory->CopyToNative(nullptr, storeDirName); + NS_ENSURE_SUCCESS(rv, rv); + + // We moved some things to new places, so move the handles around, too. + rv = SetupPathNames(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +Classifier::RemoveBackupTables() +{ + nsCString toDeleteName; + nsresult rv = mToDeleteDirectory->GetNativeLeafName(toDeleteName); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mBackupDirectory->MoveToNative(nullptr, toDeleteName); + NS_ENSURE_SUCCESS(rv, rv); + + // mBackupDirectory now points to toDelete, fix that up. + rv = SetupPathNames(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +Classifier::RecoverBackups() +{ + bool backupExists; + nsresult rv = mBackupDirectory->Exists(&backupExists); + NS_ENSURE_SUCCESS(rv, rv); + + if (backupExists) { + // Remove the safebrowsing dir if it exists + nsCString storeDirName; + rv = mRootStoreDirectory->GetNativeLeafName(storeDirName); + NS_ENSURE_SUCCESS(rv, rv); + + bool storeExists; + rv = mRootStoreDirectory->Exists(&storeExists); + NS_ENSURE_SUCCESS(rv, rv); + + if (storeExists) { + rv = mRootStoreDirectory->Remove(true); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Move the backup to the store location + rv = mBackupDirectory->MoveToNative(nullptr, storeDirName); + NS_ENSURE_SUCCESS(rv, rv); + + // mBackupDirectory now points to storeDir, fix up. + rv = SetupPathNames(); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +bool +Classifier::CheckValidUpdate(nsTArray<TableUpdate*>* aUpdates, + const nsACString& aTable) +{ + // take the quick exit if there is no valid update for us + // (common case) + uint32_t validupdates = 0; + + for (uint32_t i = 0; i < aUpdates->Length(); i++) { + TableUpdate *update = aUpdates->ElementAt(i); + if (!update || !update->TableName().Equals(aTable)) + continue; + if (update->Empty()) { + aUpdates->ElementAt(i) = nullptr; + continue; + } + validupdates++; + } + + if (!validupdates) { + // This can happen if the update was only valid for one table. + return false; + } + + return true; +} + +nsCString +Classifier::GetProvider(const nsACString& aTableName) +{ + nsCOMPtr<nsIUrlClassifierUtils> urlUtil = + do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID); + + nsCString provider; + nsresult rv = urlUtil->GetProvider(aTableName, provider); + + return NS_SUCCEEDED(rv) ? provider : EmptyCString(); +} + +/* + * This will consume+delete updates from the passed nsTArray. +*/ +nsresult +Classifier::UpdateHashStore(nsTArray<TableUpdate*>* aUpdates, + const nsACString& aTable) +{ + if (nsUrlClassifierDBService::ShutdownHasStarted()) { + return NS_ERROR_ABORT; + } + + LOG(("Classifier::UpdateHashStore(%s)", PromiseFlatCString(aTable).get())); + + HashStore store(aTable, GetProvider(aTable), mRootStoreDirectory); + + if (!CheckValidUpdate(aUpdates, store.TableName())) { + return NS_OK; + } + + nsresult rv = store.Open(); + NS_ENSURE_SUCCESS(rv, rv); + rv = store.BeginUpdate(); + NS_ENSURE_SUCCESS(rv, rv); + + // Read the part of the store that is (only) in the cache + LookupCacheV2* lookupCache = + LookupCache::Cast<LookupCacheV2>(GetLookupCache(store.TableName())); + if (!lookupCache) { + return NS_ERROR_FAILURE; + } + + // Clear cache when update + lookupCache->ClearCache(); + + FallibleTArray<uint32_t> AddPrefixHashes; + rv = lookupCache->GetPrefixes(AddPrefixHashes); + NS_ENSURE_SUCCESS(rv, rv); + rv = store.AugmentAdds(AddPrefixHashes); + NS_ENSURE_SUCCESS(rv, rv); + AddPrefixHashes.Clear(); + + uint32_t applied = 0; + + for (uint32_t i = 0; i < aUpdates->Length(); i++) { + TableUpdate *update = aUpdates->ElementAt(i); + if (!update || !update->TableName().Equals(store.TableName())) + continue; + + rv = store.ApplyUpdate(*update); + NS_ENSURE_SUCCESS(rv, rv); + + applied++; + + auto updateV2 = TableUpdate::Cast<TableUpdateV2>(update); + if (updateV2) { + LOG(("Applied update to table %s:", store.TableName().get())); + LOG((" %d add chunks", updateV2->AddChunks().Length())); + LOG((" %d add prefixes", updateV2->AddPrefixes().Length())); + LOG((" %d add completions", updateV2->AddCompletes().Length())); + LOG((" %d sub chunks", updateV2->SubChunks().Length())); + LOG((" %d sub prefixes", updateV2->SubPrefixes().Length())); + LOG((" %d sub completions", updateV2->SubCompletes().Length())); + LOG((" %d add expirations", updateV2->AddExpirations().Length())); + LOG((" %d sub expirations", updateV2->SubExpirations().Length())); + } + + aUpdates->ElementAt(i) = nullptr; + } + + LOG(("Applied %d update(s) to %s.", applied, store.TableName().get())); + + rv = store.Rebuild(); + NS_ENSURE_SUCCESS(rv, rv); + + LOG(("Table %s now has:", store.TableName().get())); + LOG((" %d add chunks", store.AddChunks().Length())); + LOG((" %d add prefixes", store.AddPrefixes().Length())); + LOG((" %d add completions", store.AddCompletes().Length())); + LOG((" %d sub chunks", store.SubChunks().Length())); + LOG((" %d sub prefixes", store.SubPrefixes().Length())); + LOG((" %d sub completions", store.SubCompletes().Length())); + + rv = store.WriteFile(); + NS_ENSURE_SUCCESS(rv, rv); + + // At this point the store is updated and written out to disk, but + // the data is still in memory. Build our quick-lookup table here. + rv = lookupCache->Build(store.AddPrefixes(), store.AddCompletes()); + NS_ENSURE_SUCCESS(rv, rv); + +#if defined(DEBUG) + lookupCache->DumpCompletions(); +#endif + rv = lookupCache->WriteFile(); + NS_ENSURE_SUCCESS(rv, rv); + + int64_t now = (PR_Now() / PR_USEC_PER_SEC); + LOG(("Successfully updated %s", store.TableName().get())); + mTableFreshness.Put(store.TableName(), now); + + return NS_OK; +} + +nsresult +Classifier::UpdateTableV4(nsTArray<TableUpdate*>* aUpdates, + const nsACString& aTable) +{ + if (nsUrlClassifierDBService::ShutdownHasStarted()) { + return NS_ERROR_ABORT; + } + + LOG(("Classifier::UpdateTableV4(%s)", PromiseFlatCString(aTable).get())); + + if (!CheckValidUpdate(aUpdates, aTable)) { + return NS_OK; + } + + LookupCacheV4* lookupCache = + LookupCache::Cast<LookupCacheV4>(GetLookupCache(aTable)); + if (!lookupCache) { + return NS_ERROR_FAILURE; + } + + nsresult rv = NS_OK; + + // If there are multiple updates for the same table, prefixes1 & prefixes2 + // will act as input and output in turn to reduce memory copy overhead. + PrefixStringMap prefixes1, prefixes2; + PrefixStringMap* input = &prefixes1; + PrefixStringMap* output = &prefixes2; + + TableUpdateV4* lastAppliedUpdate = nullptr; + for (uint32_t i = 0; i < aUpdates->Length(); i++) { + TableUpdate *update = aUpdates->ElementAt(i); + if (!update || !update->TableName().Equals(aTable)) { + continue; + } + + auto updateV4 = TableUpdate::Cast<TableUpdateV4>(update); + NS_ENSURE_TRUE(updateV4, NS_ERROR_FAILURE); + + if (updateV4->IsFullUpdate()) { + input->Clear(); + output->Clear(); + rv = lookupCache->ApplyUpdate(updateV4, *input, *output); + if (NS_FAILED(rv)) { + return rv; + } + } else { + // If both prefix sets are empty, this means we are doing a partial update + // without a prior full/partial update in the loop. In this case we should + // get prefixes from the lookup cache first. + if (prefixes1.IsEmpty() && prefixes2.IsEmpty()) { + lookupCache->GetPrefixes(prefixes1); + } else { + MOZ_ASSERT(prefixes1.IsEmpty() ^ prefixes2.IsEmpty()); + + // When there are multiple partial updates, input should always point + // to the non-empty prefix set(filled by previous full/partial update). + // output should always point to the empty prefix set. + input = prefixes1.IsEmpty() ? &prefixes2 : &prefixes1; + output = prefixes1.IsEmpty() ? &prefixes1 : &prefixes2; + } + + rv = lookupCache->ApplyUpdate(updateV4, *input, *output); + if (NS_FAILED(rv)) { + return rv; + } + + input->Clear(); + } + + // Keep track of the last applied update. + lastAppliedUpdate = updateV4; + + aUpdates->ElementAt(i) = nullptr; + } + + rv = lookupCache->Build(*output); + NS_ENSURE_SUCCESS(rv, rv); + + rv = lookupCache->WriteFile(); + NS_ENSURE_SUCCESS(rv, rv); + + if (lastAppliedUpdate) { + LOG(("Write meta data of the last applied update.")); + rv = lookupCache->WriteMetadata(lastAppliedUpdate); + NS_ENSURE_SUCCESS(rv, rv); + } + + + int64_t now = (PR_Now() / PR_USEC_PER_SEC); + LOG(("Successfully updated %s\n", PromiseFlatCString(aTable).get())); + mTableFreshness.Put(aTable, now); + + return NS_OK; +} + +nsresult +Classifier::UpdateCache(TableUpdate* aUpdate) +{ + if (!aUpdate) { + return NS_OK; + } + + nsAutoCString table(aUpdate->TableName()); + LOG(("Classifier::UpdateCache(%s)", table.get())); + + LookupCache *lookupCache = GetLookupCache(table); + if (!lookupCache) { + return NS_ERROR_FAILURE; + } + + auto updateV2 = TableUpdate::Cast<TableUpdateV2>(aUpdate); + lookupCache->AddCompletionsToCache(updateV2->AddCompletes()); + +#if defined(DEBUG) + lookupCache->DumpCache(); +#endif + + return NS_OK; +} + +LookupCache * +Classifier::GetLookupCache(const nsACString& aTable) +{ + for (uint32_t i = 0; i < mLookupCaches.Length(); i++) { + if (mLookupCaches[i]->TableName().Equals(aTable)) { + return mLookupCaches[i]; + } + } + + // TODO : Bug 1302600, It would be better if we have a more general non-main + // thread method to convert table name to protocol version. Currently + // we can only know this by checking if the table name ends with '-proto'. + UniquePtr<LookupCache> cache; + nsCString provider = GetProvider(aTable); + if (StringEndsWith(aTable, NS_LITERAL_CSTRING("-proto"))) { + cache = MakeUnique<LookupCacheV4>(aTable, provider, mRootStoreDirectory); + } else { + cache = MakeUnique<LookupCacheV2>(aTable, provider, mRootStoreDirectory); + } + + nsresult rv = cache->Init(); + if (NS_FAILED(rv)) { + return nullptr; + } + rv = cache->Open(); + if (NS_FAILED(rv)) { + if (rv == NS_ERROR_FILE_CORRUPTED) { + Reset(); + } + return nullptr; + } + mLookupCaches.AppendElement(cache.get()); + return cache.release(); +} + +nsresult +Classifier::ReadNoiseEntries(const Prefix& aPrefix, + const nsACString& aTableName, + uint32_t aCount, + PrefixArray* aNoiseEntries) +{ + // TODO : Bug 1297962, support adding noise for v4 + LookupCacheV2 *cache = static_cast<LookupCacheV2*>(GetLookupCache(aTableName)); + if (!cache) { + return NS_ERROR_FAILURE; + } + + FallibleTArray<uint32_t> prefixes; + nsresult rv = cache->GetPrefixes(prefixes); + NS_ENSURE_SUCCESS(rv, rv); + + size_t idx = prefixes.BinaryIndexOf(aPrefix.ToUint32()); + + if (idx == nsTArray<uint32_t>::NoIndex) { + NS_WARNING("Could not find prefix in PrefixSet during noise lookup"); + return NS_ERROR_FAILURE; + } + + idx -= idx % aCount; + + for (size_t i = 0; (i < aCount) && ((idx+i) < prefixes.Length()); i++) { + Prefix newPref; + newPref.FromUint32(prefixes[idx+i]); + if (newPref != aPrefix) { + aNoiseEntries->AppendElement(newPref); + } + } + + return NS_OK; +} + +nsresult +Classifier::LoadMetadata(nsIFile* aDirectory, nsACString& aResult) +{ + nsCOMPtr<nsISimpleEnumerator> entries; + nsresult rv = aDirectory->GetDirectoryEntries(getter_AddRefs(entries)); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_ARG_POINTER(entries); + + bool hasMore; + while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore) { + nsCOMPtr<nsISupports> supports; + rv = entries->GetNext(getter_AddRefs(supports)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIFile> file = do_QueryInterface(supports); + + // If |file| is a directory, recurse to find its entries as well. + bool isDirectory; + if (NS_FAILED(file->IsDirectory(&isDirectory))) { + continue; + } + if (isDirectory) { + LoadMetadata(file, aResult); + continue; + } + + // Truncate file extension to get the table name. + nsCString tableName; + rv = file->GetNativeLeafName(tableName); + NS_ENSURE_SUCCESS(rv, rv); + + int32_t dot = tableName.RFind(METADATA_SUFFIX, 0); + if (dot == -1) { + continue; + } + tableName.Cut(dot, METADATA_SUFFIX.Length()); + + LookupCacheV4* lookupCache = + LookupCache::Cast<LookupCacheV4>(GetLookupCache(tableName)); + if (!lookupCache) { + continue; + } + + nsCString state; + nsCString checksum; + rv = lookupCache->LoadMetadata(state, checksum); + if (NS_FAILED(rv)) { + LOG(("Failed to get metadata for table %s", tableName.get())); + continue; + } + + // The state might include '\n' so that we have to encode. + nsAutoCString stateBase64; + rv = Base64Encode(state, stateBase64); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString checksumBase64; + rv = Base64Encode(checksum, checksumBase64); + NS_ENSURE_SUCCESS(rv, rv); + + LOG(("Appending state '%s' and checksum '%s' for table %s", + stateBase64.get(), checksumBase64.get(), tableName.get())); + + aResult.AppendPrintf("%s;%s:%s\n", tableName.get(), + stateBase64.get(), + checksumBase64.get()); + } + + return rv; +} + +} // namespace safebrowsing +} // namespace mozilla diff --git a/toolkit/components/url-classifier/Classifier.h b/toolkit/components/url-classifier/Classifier.h new file mode 100644 index 000000000..58c49fce5 --- /dev/null +++ b/toolkit/components/url-classifier/Classifier.h @@ -0,0 +1,167 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef Classifier_h__ +#define Classifier_h__ + +#include "Entries.h" +#include "HashStore.h" +#include "ProtocolParser.h" +#include "LookupCache.h" +#include "nsCOMPtr.h" +#include "nsString.h" +#include "nsIFile.h" +#include "nsICryptoHash.h" +#include "nsDataHashtable.h" + +namespace mozilla { +namespace safebrowsing { + +/** + * Maintains the stores and LookupCaches for the url classifier. + */ +class Classifier { +public: + typedef nsClassHashtable<nsCStringHashKey, nsCString> ProviderDictType; + +public: + Classifier(); + ~Classifier(); + + nsresult Open(nsIFile& aCacheDirectory); + void Close(); + void Reset(); + + /** + * Clear data for specific tables. + * If ClearType is Clear_Cache, this function will only clear cache in lookup + * cache, otherwise, it will clear data in lookup cache and data stored on disk. + */ + enum ClearType { + Clear_Cache, + Clear_All, + }; + void ResetTables(ClearType aType, const nsTArray<nsCString>& aTables); + + /** + * Get the list of active tables and their chunks in a format + * suitable for an update request. + */ + void TableRequest(nsACString& aResult); + + /* + * Get all tables that we know about. + */ + nsresult ActiveTables(nsTArray<nsCString>& aTables); + + /** + * Check a URL against the specified tables. + */ + nsresult Check(const nsACString& aSpec, + const nsACString& tables, + uint32_t aFreshnessGuarantee, + LookupResultArray& aResults); + + /** + * Apply the table updates in the array. Takes ownership of + * the updates in the array and clears it. Wacky! + */ + nsresult ApplyUpdates(nsTArray<TableUpdate*>* aUpdates); + + /** + * Apply full hashes retrived from gethash to cache. + */ + nsresult ApplyFullHashes(nsTArray<TableUpdate*>* aUpdates); + + void SetLastUpdateTime(const nsACString& aTableName, uint64_t updateTime); + int64_t GetLastUpdateTime(const nsACString& aTableName); + nsresult CacheCompletions(const CacheResultArray& aResults); + uint32_t GetHashKey(void) { return mHashKey; } + /* + * Get a bunch of extra prefixes to query for completion + * and mask the real entry being requested + */ + nsresult ReadNoiseEntries(const Prefix& aPrefix, + const nsACString& aTableName, + uint32_t aCount, + PrefixArray* aNoiseEntries); + +#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES + nsresult DumpRawTableUpdates(const nsACString& aRawUpdates); +#endif + + static void SplitTables(const nsACString& str, nsTArray<nsCString>& tables); + + // Given a root store directory, return a private store directory + // based on the table name. To avoid migration issue, the private + // store directory is only different from root directory for V4 tables. + // + // For V4 tables (suffixed by '-proto'), the private directory would + // be [root directory path]/[provider]. The provider of V4 tables is + // 'google4'. + // + // Note that if the table name is not owned by any provider, just use + // the root directory. + static nsresult GetPrivateStoreDirectory(nsIFile* aRootStoreDirectory, + const nsACString& aTableName, + const nsACString& aProvider, + nsIFile** aPrivateStoreDirectory); + +private: + void DropStores(); + void DeleteTables(nsIFile* aDirectory, const nsTArray<nsCString>& aTables); + void AbortUpdateAndReset(const nsCString& aTable); + + nsresult CreateStoreDirectory(); + nsresult SetupPathNames(); + nsresult RecoverBackups(); + nsresult CleanToDelete(); + nsresult BackupTables(); + nsresult RemoveBackupTables(); + nsresult RegenActiveTables(); + +#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES + already_AddRefed<nsIFile> GetFailedUpdateDirectroy(); + nsresult DumpFailedUpdate(); +#endif + + nsresult ScanStoreDir(nsTArray<nsCString>& aTables); + + nsresult UpdateHashStore(nsTArray<TableUpdate*>* aUpdates, + const nsACString& aTable); + + nsresult UpdateTableV4(nsTArray<TableUpdate*>* aUpdates, + const nsACString& aTable); + + nsresult UpdateCache(TableUpdate* aUpdates); + + LookupCache *GetLookupCache(const nsACString& aTable); + + bool CheckValidUpdate(nsTArray<TableUpdate*>* aUpdates, + const nsACString& aTable); + + nsresult LoadMetadata(nsIFile* aDirectory, nsACString& aResult); + + nsCString GetProvider(const nsACString& aTableName); + + // Root dir of the Local profile. + nsCOMPtr<nsIFile> mCacheDirectory; + // Main directory where to store the databases. + nsCOMPtr<nsIFile> mRootStoreDirectory; + // Used for atomically updating the other dirs. + nsCOMPtr<nsIFile> mBackupDirectory; + nsCOMPtr<nsIFile> mToDeleteDirectory; + nsCOMPtr<nsICryptoHash> mCryptoHash; + nsTArray<LookupCache*> mLookupCaches; + nsTArray<nsCString> mActiveTablesCache; + uint32_t mHashKey; + // Stores the last time a given table was updated (seconds). + nsDataHashtable<nsCStringHashKey, int64_t> mTableFreshness; +}; + +} // namespace safebrowsing +} // namespace mozilla + +#endif diff --git a/toolkit/components/url-classifier/Entries.h b/toolkit/components/url-classifier/Entries.h new file mode 100644 index 000000000..969f4f739 --- /dev/null +++ b/toolkit/components/url-classifier/Entries.h @@ -0,0 +1,322 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This header file defines the storage types of the actual safebrowsing +// chunk data, which may be either 32-bit hashes or complete 256-bit hashes. +// Chunk numbers are represented in ChunkSet.h. + +#ifndef SBEntries_h__ +#define SBEntries_h__ + +#include "nsTArray.h" +#include "nsString.h" +#include "nsICryptoHash.h" +#include "nsNetUtil.h" +#include "nsIOutputStream.h" +#include "nsClassHashtable.h" + +#if DEBUG +#include "plbase64.h" +#endif + +namespace mozilla { +namespace safebrowsing { + +#define PREFIX_SIZE 4 +#define COMPLETE_SIZE 32 + +// This is the struct that contains 4-byte hash prefixes. +template <uint32_t S, class Comparator> +struct SafebrowsingHash +{ + static_assert(S >= 4, "The SafebrowsingHash should be at least 4 bytes."); + + static const uint32_t sHashSize = S; + typedef SafebrowsingHash<S, Comparator> self_type; + uint8_t buf[S]; + + nsresult FromPlaintext(const nsACString& aPlainText, nsICryptoHash* aHash) { + // From the protocol doc: + // Each entry in the chunk is composed + // of the SHA 256 hash of a suffix/prefix expression. + + nsresult rv = aHash->Init(nsICryptoHash::SHA256); + NS_ENSURE_SUCCESS(rv, rv); + + rv = aHash->Update + (reinterpret_cast<const uint8_t*>(aPlainText.BeginReading()), + aPlainText.Length()); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString hashed; + rv = aHash->Finish(false, hashed); + NS_ENSURE_SUCCESS(rv, rv); + + NS_ASSERTION(hashed.Length() >= sHashSize, + "not enough characters in the hash"); + + memcpy(buf, hashed.BeginReading(), sHashSize); + + return NS_OK; + } + + void Assign(const nsACString& aStr) { + NS_ASSERTION(aStr.Length() >= sHashSize, + "string must be at least sHashSize characters long"); + memcpy(buf, aStr.BeginReading(), sHashSize); + } + + int Compare(const self_type& aOther) const { + return Comparator::Compare(buf, aOther.buf); + } + + bool operator==(const self_type& aOther) const { + return Comparator::Compare(buf, aOther.buf) == 0; + } + + bool operator!=(const self_type& aOther) const { + return Comparator::Compare(buf, aOther.buf) != 0; + } + + bool operator<(const self_type& aOther) const { + return Comparator::Compare(buf, aOther.buf) < 0; + } + +#ifdef DEBUG + void ToString(nsACString& aStr) const { + uint32_t len = ((sHashSize + 2) / 3) * 4; + aStr.SetCapacity(len + 1); + PL_Base64Encode((char*)buf, sHashSize, aStr.BeginWriting()); + aStr.BeginWriting()[len] = '\0'; + } +#endif + + void ToHexString(nsACString& aStr) const { + static const char* const lut = "0123456789ABCDEF"; + // 32 bytes is the longest hash + size_t len = 32; + + aStr.SetCapacity(2 * len); + for (size_t i = 0; i < len; ++i) { + const char c = static_cast<const char>(buf[i]); + aStr.Append(lut[(c >> 4) & 0x0F]); + aStr.Append(lut[c & 15]); + } + } + + uint32_t ToUint32() const { + uint32_t n; + memcpy(&n, buf, sizeof(n)); + return n; + } + void FromUint32(uint32_t aHash) { + memcpy(buf, &aHash, sizeof(aHash)); + } +}; + +class PrefixComparator { +public: + static int Compare(const uint8_t* a, const uint8_t* b) { + uint32_t first; + memcpy(&first, a, sizeof(uint32_t)); + + uint32_t second; + memcpy(&second, b, sizeof(uint32_t)); + + if (first > second) { + return 1; + } else if (first == second) { + return 0; + } else { + return -1; + } + } +}; +// Use this for 4-byte hashes +typedef SafebrowsingHash<PREFIX_SIZE, PrefixComparator> Prefix; +typedef nsTArray<Prefix> PrefixArray; + +class CompletionComparator { +public: + static int Compare(const uint8_t* a, const uint8_t* b) { + return memcmp(a, b, COMPLETE_SIZE); + } +}; +// Use this for 32-byte hashes +typedef SafebrowsingHash<COMPLETE_SIZE, CompletionComparator> Completion; +typedef nsTArray<Completion> CompletionArray; + +struct AddPrefix { + // The truncated hash. + Prefix prefix; + // The chunk number to which it belongs. + uint32_t addChunk; + + AddPrefix() : addChunk(0) {} + + // Returns the chunk number. + uint32_t Chunk() const { return addChunk; } + const Prefix &PrefixHash() const { return prefix; } + + template<class T> + int Compare(const T& other) const { + int cmp = prefix.Compare(other.PrefixHash()); + if (cmp != 0) { + return cmp; + } + return addChunk - other.addChunk; + } +}; + +struct AddComplete { + Completion complete; + uint32_t addChunk; + + AddComplete() : addChunk(0) {} + + uint32_t Chunk() const { return addChunk; } + // The 4-byte prefix of the sha256 hash. + uint32_t ToUint32() const { return complete.ToUint32(); } + // The 32-byte sha256 hash. + const Completion &CompleteHash() const { return complete; } + + template<class T> + int Compare(const T& other) const { + int cmp = complete.Compare(other.CompleteHash()); + if (cmp != 0) { + return cmp; + } + return addChunk - other.addChunk; + } + + bool operator!=(const AddComplete& aOther) const { + if (addChunk != aOther.addChunk) { + return true; + } + return complete != aOther.complete; + } +}; + +struct SubPrefix { + // The hash to subtract. + Prefix prefix; + // The chunk number of the add chunk to which the hash belonged. + uint32_t addChunk; + // The chunk number of this sub chunk. + uint32_t subChunk; + + SubPrefix(): addChunk(0), subChunk(0) {} + + uint32_t Chunk() const { return subChunk; } + uint32_t AddChunk() const { return addChunk; } + const Prefix &PrefixHash() const { return prefix; } + + template<class T> + // Returns 0 if and only if the chunks are the same in every way. + int Compare(const T& aOther) const { + int cmp = prefix.Compare(aOther.PrefixHash()); + if (cmp != 0) + return cmp; + if (addChunk != aOther.addChunk) + return addChunk - aOther.addChunk; + return subChunk - aOther.subChunk; + } + + template<class T> + int CompareAlt(const T& aOther) const { + Prefix other; + other.FromUint32(aOther.ToUint32()); + int cmp = prefix.Compare(other); + if (cmp != 0) + return cmp; + return addChunk - aOther.addChunk; + } +}; + +struct SubComplete { + Completion complete; + uint32_t addChunk; + uint32_t subChunk; + + SubComplete() : addChunk(0), subChunk(0) {} + + uint32_t Chunk() const { return subChunk; } + uint32_t AddChunk() const { return addChunk; } + const Completion &CompleteHash() const { return complete; } + // The 4-byte prefix of the sha256 hash. + uint32_t ToUint32() const { return complete.ToUint32(); } + + int Compare(const SubComplete& aOther) const { + int cmp = complete.Compare(aOther.complete); + if (cmp != 0) + return cmp; + if (addChunk != aOther.addChunk) + return addChunk - aOther.addChunk; + return subChunk - aOther.subChunk; + } +}; + +typedef FallibleTArray<AddPrefix> AddPrefixArray; +typedef FallibleTArray<AddComplete> AddCompleteArray; +typedef FallibleTArray<SubPrefix> SubPrefixArray; +typedef FallibleTArray<SubComplete> SubCompleteArray; + +/** + * Compares chunks by their add chunk, then their prefix. + */ +template<class T> +class EntryCompare { +public: + typedef T elem_type; + static int Compare(const void* e1, const void* e2) { + const elem_type* a = static_cast<const elem_type*>(e1); + const elem_type* b = static_cast<const elem_type*>(e2); + return a->Compare(*b); + } +}; + +/** + * Sort an array of store entries. nsTArray::Sort uses Equal/LessThan + * to sort, this does a single Compare so it's a bit quicker over the + * large sorts we do. + */ +template<class T, class Alloc> +void +EntrySort(nsTArray_Impl<T, Alloc>& aArray) +{ + qsort(aArray.Elements(), aArray.Length(), sizeof(T), + EntryCompare<T>::Compare); +} + +template<class T, class Alloc> +nsresult +ReadTArray(nsIInputStream* aStream, nsTArray_Impl<T, Alloc>* aArray, uint32_t aNumElements) +{ + if (!aArray->SetLength(aNumElements, fallible)) + return NS_ERROR_OUT_OF_MEMORY; + + void *buffer = aArray->Elements(); + nsresult rv = NS_ReadInputStreamToBuffer(aStream, &buffer, + (aNumElements * sizeof(T))); + NS_ENSURE_SUCCESS(rv, rv); + return NS_OK; +} + +template<class T, class Alloc> +nsresult +WriteTArray(nsIOutputStream* aStream, nsTArray_Impl<T, Alloc>& aArray) +{ + uint32_t written; + return aStream->Write(reinterpret_cast<char*>(aArray.Elements()), + aArray.Length() * sizeof(T), + &written); +} + +typedef nsClassHashtable<nsUint32HashKey, nsCString> PrefixStringMap; + +} // namespace safebrowsing +} // namespace mozilla + +#endif // SBEntries_h__ diff --git a/toolkit/components/url-classifier/HashStore.cpp b/toolkit/components/url-classifier/HashStore.cpp new file mode 100644 index 000000000..c298612aa --- /dev/null +++ b/toolkit/components/url-classifier/HashStore.cpp @@ -0,0 +1,1248 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +// Originally based on Chrome sources: +// Copyright (c) 2010 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +#include "HashStore.h" +#include "nsICryptoHash.h" +#include "nsISeekableStream.h" +#include "nsIStreamConverterService.h" +#include "nsNetUtil.h" +#include "nsCheckSummedOutputStream.h" +#include "prio.h" +#include "mozilla/Logging.h" +#include "zlib.h" +#include "Classifier.h" +#include "nsUrlClassifierDBService.h" + +// Main store for SafeBrowsing protocol data. We store +// known add/sub chunks, prefixes and completions in memory +// during an update, and serialize to disk. +// We do not store the add prefixes, those are retrieved by +// decompressing the PrefixSet cache whenever we need to apply +// an update. +// +// byte slicing: Many of the 4-byte values stored here are strongly +// correlated in the upper bytes, and uncorrelated in the lower +// bytes. Because zlib/DEFLATE requires match lengths of at least +// 3 to achieve good compression, and we don't get those if only +// the upper 16-bits are correlated, it is worthwhile to slice 32-bit +// values into 4 1-byte slices and compress the slices individually. +// The slices corresponding to MSBs will compress very well, and the +// slice corresponding to LSB almost nothing. Because of this, we +// only apply DEFLATE to the 3 most significant bytes, and store the +// LSB uncompressed. +// +// byte sliced (numValues) data format: +// uint32_t compressed-size +// compressed-size bytes zlib DEFLATE data +// 0...numValues byte MSB of 4-byte numValues data +// uint32_t compressed-size +// compressed-size bytes zlib DEFLATE data +// 0...numValues byte 2nd byte of 4-byte numValues data +// uint32_t compressed-size +// compressed-size bytes zlib DEFLATE data +// 0...numValues byte 3rd byte of 4-byte numValues data +// 0...numValues byte LSB of 4-byte numValues data +// +// Store data format: +// uint32_t magic +// uint32_t version +// uint32_t numAddChunks +// uint32_t numSubChunks +// uint32_t numAddPrefixes +// uint32_t numSubPrefixes +// uint32_t numAddCompletes +// uint32_t numSubCompletes +// 0...numAddChunks uint32_t addChunk +// 0...numSubChunks uint32_t subChunk +// byte sliced (numAddPrefixes) uint32_t add chunk of AddPrefixes +// byte sliced (numSubPrefixes) uint32_t add chunk of SubPrefixes +// byte sliced (numSubPrefixes) uint32_t sub chunk of SubPrefixes +// byte sliced (numSubPrefixes) uint32_t SubPrefixes +// 0...numAddCompletes 32-byte Completions + uint32_t addChunk +// 0...numSubCompletes 32-byte Completions + uint32_t addChunk +// + uint32_t subChunk +// 16-byte MD5 of all preceding data + +// Name of the SafeBrowsing store +#define STORE_SUFFIX ".sbstore" + +// MOZ_LOG=UrlClassifierDbService:5 +extern mozilla::LazyLogModule gUrlClassifierDbServiceLog; +#define LOG(args) MOZ_LOG(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug, args) +#define LOG_ENABLED() MOZ_LOG_TEST(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug) + +// Either the return was successful or we call the Reset function (unless we +// hit an OOM). Used while reading in the store. +#define SUCCESS_OR_RESET(res) \ + do { \ + nsresult __rv = res; /* Don't evaluate |res| more than once */ \ + if (__rv == NS_ERROR_OUT_OF_MEMORY) { \ + NS_WARNING("SafeBrowsing OOM."); \ + return __rv; \ + } \ + if (NS_FAILED(__rv)) { \ + NS_WARNING("SafeBrowsing store corrupted or out of date."); \ + Reset(); \ + return __rv; \ + } \ + } while(0) + +namespace mozilla { +namespace safebrowsing { + +const uint32_t STORE_MAGIC = 0x1231af3b; +const uint32_t CURRENT_VERSION = 3; + +nsresult +TableUpdateV2::NewAddPrefix(uint32_t aAddChunk, const Prefix& aHash) +{ + AddPrefix *add = mAddPrefixes.AppendElement(fallible); + if (!add) return NS_ERROR_OUT_OF_MEMORY; + add->addChunk = aAddChunk; + add->prefix = aHash; + return NS_OK; +} + +nsresult +TableUpdateV2::NewSubPrefix(uint32_t aAddChunk, const Prefix& aHash, uint32_t aSubChunk) +{ + SubPrefix *sub = mSubPrefixes.AppendElement(fallible); + if (!sub) return NS_ERROR_OUT_OF_MEMORY; + sub->addChunk = aAddChunk; + sub->prefix = aHash; + sub->subChunk = aSubChunk; + return NS_OK; +} + +nsresult +TableUpdateV2::NewAddComplete(uint32_t aAddChunk, const Completion& aHash) +{ + AddComplete *add = mAddCompletes.AppendElement(fallible); + if (!add) return NS_ERROR_OUT_OF_MEMORY; + add->addChunk = aAddChunk; + add->complete = aHash; + return NS_OK; +} + +nsresult +TableUpdateV2::NewSubComplete(uint32_t aAddChunk, const Completion& aHash, uint32_t aSubChunk) +{ + SubComplete *sub = mSubCompletes.AppendElement(fallible); + if (!sub) return NS_ERROR_OUT_OF_MEMORY; + sub->addChunk = aAddChunk; + sub->complete = aHash; + sub->subChunk = aSubChunk; + return NS_OK; +} + +void +TableUpdateV4::NewPrefixes(int32_t aSize, std::string& aPrefixes) +{ + NS_ENSURE_TRUE_VOID(aPrefixes.size() % aSize == 0); + NS_ENSURE_TRUE_VOID(!mPrefixesMap.Get(aSize)); + + if (LOG_ENABLED() && 4 == aSize) { + int numOfPrefixes = aPrefixes.size() / 4; + uint32_t* p = (uint32_t*)aPrefixes.c_str(); + + // Dump the first/last 10 fixed-length prefixes for debugging. + LOG(("* The first 10 (maximum) fixed-length prefixes: ")); + for (int i = 0; i < std::min(10, numOfPrefixes); i++) { + uint8_t* c = (uint8_t*)&p[i]; + LOG(("%.2X%.2X%.2X%.2X", c[0], c[1], c[2], c[3])); + } + + LOG(("* The last 10 (maximum) fixed-length prefixes: ")); + for (int i = std::max(0, numOfPrefixes - 10); i < numOfPrefixes; i++) { + uint8_t* c = (uint8_t*)&p[i]; + LOG(("%.2X%.2X%.2X%.2X", c[0], c[1], c[2], c[3])); + } + + LOG(("---- %d fixed-length prefixes in total.", aPrefixes.size() / aSize)); + } + + PrefixStdString* prefix = new PrefixStdString(aPrefixes); + mPrefixesMap.Put(aSize, prefix); +} + +void +TableUpdateV4::NewRemovalIndices(const uint32_t* aIndices, size_t aNumOfIndices) +{ + for (size_t i = 0; i < aNumOfIndices; i++) { + mRemovalIndiceArray.AppendElement(aIndices[i]); + } +} + +void +TableUpdateV4::NewChecksum(const std::string& aChecksum) +{ + mChecksum.Assign(aChecksum.data(), aChecksum.size()); +} + +HashStore::HashStore(const nsACString& aTableName, + const nsACString& aProvider, + nsIFile* aRootStoreDir) + : mTableName(aTableName) + , mInUpdate(false) + , mFileSize(0) +{ + nsresult rv = Classifier::GetPrivateStoreDirectory(aRootStoreDir, + aTableName, + aProvider, + getter_AddRefs(mStoreDirectory)); + if (NS_FAILED(rv)) { + LOG(("Failed to get private store directory for %s", mTableName.get())); + mStoreDirectory = aRootStoreDir; + } +} + +HashStore::~HashStore() +{ +} + +nsresult +HashStore::Reset() +{ + LOG(("HashStore resetting")); + + nsCOMPtr<nsIFile> storeFile; + nsresult rv = mStoreDirectory->Clone(getter_AddRefs(storeFile)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = storeFile->AppendNative(mTableName + NS_LITERAL_CSTRING(STORE_SUFFIX)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = storeFile->Remove(false); + NS_ENSURE_SUCCESS(rv, rv); + + mFileSize = 0; + + return NS_OK; +} + +nsresult +HashStore::CheckChecksum(uint32_t aFileSize) +{ + if (!mInputStream) { + return NS_OK; + } + + // Check for file corruption by + // comparing the stored checksum to actual checksum of data + nsAutoCString hash; + nsAutoCString compareHash; + char *data; + uint32_t read; + + nsresult rv = CalculateChecksum(hash, aFileSize, true); + NS_ENSURE_SUCCESS(rv, rv); + + compareHash.GetMutableData(&data, hash.Length()); + + if (hash.Length() > aFileSize) { + NS_WARNING("SafeBrowing file not long enough to store its hash"); + return NS_ERROR_FAILURE; + } + nsCOMPtr<nsISeekableStream> seekIn = do_QueryInterface(mInputStream); + rv = seekIn->Seek(nsISeekableStream::NS_SEEK_SET, aFileSize - hash.Length()); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mInputStream->Read(data, hash.Length(), &read); + NS_ENSURE_SUCCESS(rv, rv); + NS_ASSERTION(read == hash.Length(), "Could not read hash bytes"); + + if (!hash.Equals(compareHash)) { + NS_WARNING("Safebrowing file failed checksum."); + return NS_ERROR_FAILURE; + } + + return NS_OK; +} + +nsresult +HashStore::Open() +{ + nsCOMPtr<nsIFile> storeFile; + nsresult rv = mStoreDirectory->Clone(getter_AddRefs(storeFile)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = storeFile->AppendNative(mTableName + NS_LITERAL_CSTRING(".sbstore")); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIInputStream> origStream; + rv = NS_NewLocalFileInputStream(getter_AddRefs(origStream), storeFile, + PR_RDONLY | nsIFile::OS_READAHEAD); + + if (rv == NS_ERROR_FILE_NOT_FOUND) { + UpdateHeader(); + return NS_OK; + } else { + SUCCESS_OR_RESET(rv); + } + + int64_t fileSize; + rv = storeFile->GetFileSize(&fileSize); + NS_ENSURE_SUCCESS(rv, rv); + + if (fileSize < 0 || fileSize > UINT32_MAX) { + return NS_ERROR_FAILURE; + } + + mFileSize = static_cast<uint32_t>(fileSize); + mInputStream = NS_BufferInputStream(origStream, mFileSize); + + rv = ReadHeader(); + SUCCESS_OR_RESET(rv); + + rv = SanityCheck(); + SUCCESS_OR_RESET(rv); + + return NS_OK; +} + +nsresult +HashStore::ReadHeader() +{ + if (!mInputStream) { + UpdateHeader(); + return NS_OK; + } + + nsCOMPtr<nsISeekableStream> seekable = do_QueryInterface(mInputStream); + nsresult rv = seekable->Seek(nsISeekableStream::NS_SEEK_SET, 0); + NS_ENSURE_SUCCESS(rv, rv); + + void *buffer = &mHeader; + rv = NS_ReadInputStreamToBuffer(mInputStream, + &buffer, + sizeof(Header)); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +HashStore::SanityCheck() +{ + if (mHeader.magic != STORE_MAGIC || mHeader.version != CURRENT_VERSION) { + NS_WARNING("Unexpected header data in the store."); + return NS_ERROR_FAILURE; + } + + return NS_OK; +} + +nsresult +HashStore::CalculateChecksum(nsAutoCString& aChecksum, + uint32_t aFileSize, + bool aChecksumPresent) +{ + aChecksum.Truncate(); + + // Reset mInputStream to start + nsCOMPtr<nsISeekableStream> seekable = do_QueryInterface(mInputStream); + nsresult rv = seekable->Seek(nsISeekableStream::NS_SEEK_SET, 0); + + nsCOMPtr<nsICryptoHash> hash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + // Size of MD5 hash in bytes + const uint32_t CHECKSUM_SIZE = 16; + + // MD5 is not a secure hash function, but since this is a filesystem integrity + // check, this usage is ok. + rv = hash->Init(nsICryptoHash::MD5); + NS_ENSURE_SUCCESS(rv, rv); + + if (!aChecksumPresent) { + // Hash entire file + rv = hash->UpdateFromStream(mInputStream, UINT32_MAX); + } else { + // Hash everything but last checksum bytes + if (aFileSize < CHECKSUM_SIZE) { + NS_WARNING("SafeBrowsing file isn't long enough to store its checksum"); + return NS_ERROR_FAILURE; + } + rv = hash->UpdateFromStream(mInputStream, aFileSize - CHECKSUM_SIZE); + } + NS_ENSURE_SUCCESS(rv, rv); + + rv = hash->Finish(false, aChecksum); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +void +HashStore::UpdateHeader() +{ + mHeader.magic = STORE_MAGIC; + mHeader.version = CURRENT_VERSION; + + mHeader.numAddChunks = mAddChunks.Length(); + mHeader.numSubChunks = mSubChunks.Length(); + mHeader.numAddPrefixes = mAddPrefixes.Length(); + mHeader.numSubPrefixes = mSubPrefixes.Length(); + mHeader.numAddCompletes = mAddCompletes.Length(); + mHeader.numSubCompletes = mSubCompletes.Length(); +} + +nsresult +HashStore::ReadChunkNumbers() +{ + if (!mInputStream || AlreadyReadChunkNumbers()) { + return NS_OK; + } + + nsCOMPtr<nsISeekableStream> seekable = do_QueryInterface(mInputStream); + nsresult rv = seekable->Seek(nsISeekableStream::NS_SEEK_SET, + sizeof(Header)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mAddChunks.Read(mInputStream, mHeader.numAddChunks); + NS_ENSURE_SUCCESS(rv, rv); + NS_ASSERTION(mAddChunks.Length() == mHeader.numAddChunks, "Read the right amount of add chunks."); + + rv = mSubChunks.Read(mInputStream, mHeader.numSubChunks); + NS_ENSURE_SUCCESS(rv, rv); + NS_ASSERTION(mSubChunks.Length() == mHeader.numSubChunks, "Read the right amount of sub chunks."); + + return NS_OK; +} + +nsresult +HashStore::ReadHashes() +{ + if (!mInputStream) { + // BeginUpdate has been called but Open hasn't initialized mInputStream, + // because the existing HashStore is empty. + return NS_OK; + } + + nsCOMPtr<nsISeekableStream> seekable = do_QueryInterface(mInputStream); + + uint32_t offset = sizeof(Header); + offset += (mHeader.numAddChunks + mHeader.numSubChunks) * sizeof(uint32_t); + nsresult rv = seekable->Seek(nsISeekableStream::NS_SEEK_SET, offset); + NS_ENSURE_SUCCESS(rv, rv); + + rv = ReadAddPrefixes(); + NS_ENSURE_SUCCESS(rv, rv); + + rv = ReadSubPrefixes(); + NS_ENSURE_SUCCESS(rv, rv); + + // If completions was read before, then we are done here. + if (AlreadyReadCompletions()) { + return NS_OK; + } + + rv = ReadTArray(mInputStream, &mAddCompletes, mHeader.numAddCompletes); + NS_ENSURE_SUCCESS(rv, rv); + + rv = ReadTArray(mInputStream, &mSubCompletes, mHeader.numSubCompletes); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + + +nsresult +HashStore::ReadCompletions() +{ + if (!mInputStream || AlreadyReadCompletions()) { + return NS_OK; + } + + nsCOMPtr<nsIFile> storeFile; + nsresult rv = mStoreDirectory->Clone(getter_AddRefs(storeFile)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = storeFile->AppendNative(mTableName + NS_LITERAL_CSTRING(STORE_SUFFIX)); + NS_ENSURE_SUCCESS(rv, rv); + + uint32_t offset = mFileSize - + sizeof(struct AddComplete) * mHeader.numAddCompletes - + sizeof(struct SubComplete) * mHeader.numSubCompletes - + nsCheckSummedOutputStream::CHECKSUM_SIZE; + + nsCOMPtr<nsISeekableStream> seekable = do_QueryInterface(mInputStream); + + rv = seekable->Seek(nsISeekableStream::NS_SEEK_SET, offset); + NS_ENSURE_SUCCESS(rv, rv); + + rv = ReadTArray(mInputStream, &mAddCompletes, mHeader.numAddCompletes); + NS_ENSURE_SUCCESS(rv, rv); + + rv = ReadTArray(mInputStream, &mSubCompletes, mHeader.numSubCompletes); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +HashStore::PrepareForUpdate() +{ + nsresult rv = CheckChecksum(mFileSize); + SUCCESS_OR_RESET(rv); + + rv = ReadChunkNumbers(); + SUCCESS_OR_RESET(rv); + + rv = ReadHashes(); + SUCCESS_OR_RESET(rv); + + return NS_OK; +} + +nsresult +HashStore::BeginUpdate() +{ + // Check wether the file is corrupted and read the rest of the store + // in memory. + nsresult rv = PrepareForUpdate(); + NS_ENSURE_SUCCESS(rv, rv); + + // Close input stream, won't be needed any more and + // we will rewrite ourselves. + if (mInputStream) { + rv = mInputStream->Close(); + NS_ENSURE_SUCCESS(rv, rv); + } + + mInUpdate = true; + + return NS_OK; +} + +template<class T> +static nsresult +Merge(ChunkSet* aStoreChunks, + FallibleTArray<T>* aStorePrefixes, + ChunkSet& aUpdateChunks, + FallibleTArray<T>& aUpdatePrefixes, + bool aAllowMerging = false) +{ + EntrySort(aUpdatePrefixes); + + T* updateIter = aUpdatePrefixes.Elements(); + T* updateEnd = aUpdatePrefixes.Elements() + aUpdatePrefixes.Length(); + + T* storeIter = aStorePrefixes->Elements(); + T* storeEnd = aStorePrefixes->Elements() + aStorePrefixes->Length(); + + // use a separate array so we can keep the iterators valid + // if the nsTArray grows + nsTArray<T> adds; + + for (; updateIter != updateEnd; updateIter++) { + // skip this chunk if we already have it, unless we're + // merging completions, in which case we'll always already + // have the chunk from the original prefix + if (aStoreChunks->Has(updateIter->Chunk())) + if (!aAllowMerging) + continue; + // XXX: binary search for insertion point might be faster in common + // case? + while (storeIter < storeEnd && (storeIter->Compare(*updateIter) < 0)) { + // skip forward to matching element (or not...) + storeIter++; + } + // no match, add + if (storeIter == storeEnd + || storeIter->Compare(*updateIter) != 0) { + if (!adds.AppendElement(*updateIter)) + return NS_ERROR_OUT_OF_MEMORY; + } + } + + // Chunks can be empty, but we should still report we have them + // to make the chunkranges continuous. + aStoreChunks->Merge(aUpdateChunks); + + if (!aStorePrefixes->AppendElements(adds, fallible)) + return NS_ERROR_OUT_OF_MEMORY; + + EntrySort(*aStorePrefixes); + + return NS_OK; +} + +nsresult +HashStore::ApplyUpdate(TableUpdate &aUpdate) +{ + auto updateV2 = TableUpdate::Cast<TableUpdateV2>(&aUpdate); + NS_ENSURE_TRUE(updateV2, NS_ERROR_FAILURE); + + TableUpdateV2& update = *updateV2; + + nsresult rv = mAddExpirations.Merge(update.AddExpirations()); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mSubExpirations.Merge(update.SubExpirations()); + NS_ENSURE_SUCCESS(rv, rv); + + rv = Expire(); + NS_ENSURE_SUCCESS(rv, rv); + + rv = Merge(&mAddChunks, &mAddPrefixes, + update.AddChunks(), update.AddPrefixes()); + NS_ENSURE_SUCCESS(rv, rv); + + rv = Merge(&mAddChunks, &mAddCompletes, + update.AddChunks(), update.AddCompletes(), true); + NS_ENSURE_SUCCESS(rv, rv); + + rv = Merge(&mSubChunks, &mSubPrefixes, + update.SubChunks(), update.SubPrefixes()); + NS_ENSURE_SUCCESS(rv, rv); + + rv = Merge(&mSubChunks, &mSubCompletes, + update.SubChunks(), update.SubCompletes(), true); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +HashStore::Rebuild() +{ + NS_ASSERTION(mInUpdate, "Must be in update to rebuild."); + + nsresult rv = ProcessSubs(); + NS_ENSURE_SUCCESS(rv, rv); + + UpdateHeader(); + + return NS_OK; +} + +void +HashStore::ClearCompletes() +{ + NS_ASSERTION(mInUpdate, "Must be in update to clear completes."); + + mAddCompletes.Clear(); + mSubCompletes.Clear(); + + UpdateHeader(); +} + +template<class T> +static void +ExpireEntries(FallibleTArray<T>* aEntries, ChunkSet& aExpirations) +{ + T* addIter = aEntries->Elements(); + T* end = aEntries->Elements() + aEntries->Length(); + + for (T *iter = addIter; iter != end; iter++) { + if (!aExpirations.Has(iter->Chunk())) { + *addIter = *iter; + addIter++; + } + } + + aEntries->TruncateLength(addIter - aEntries->Elements()); +} + +nsresult +HashStore::Expire() +{ + ExpireEntries(&mAddPrefixes, mAddExpirations); + ExpireEntries(&mAddCompletes, mAddExpirations); + ExpireEntries(&mSubPrefixes, mSubExpirations); + ExpireEntries(&mSubCompletes, mSubExpirations); + + mAddChunks.Remove(mAddExpirations); + mSubChunks.Remove(mSubExpirations); + + mAddExpirations.Clear(); + mSubExpirations.Clear(); + + return NS_OK; +} + +template<class T> +nsresult DeflateWriteTArray(nsIOutputStream* aStream, nsTArray<T>& aIn) +{ + uLongf insize = aIn.Length() * sizeof(T); + uLongf outsize = compressBound(insize); + FallibleTArray<char> outBuff; + if (!outBuff.SetLength(outsize, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + int zerr = compress(reinterpret_cast<Bytef*>(outBuff.Elements()), + &outsize, + reinterpret_cast<const Bytef*>(aIn.Elements()), + insize); + if (zerr != Z_OK) { + return NS_ERROR_FAILURE; + } + LOG(("DeflateWriteTArray: %d in %d out", insize, outsize)); + + outBuff.TruncateLength(outsize); + + // Length of compressed data stream + uint32_t dataLen = outBuff.Length(); + uint32_t written; + nsresult rv = aStream->Write(reinterpret_cast<char*>(&dataLen), sizeof(dataLen), &written); + NS_ENSURE_SUCCESS(rv, rv); + + NS_ASSERTION(written == sizeof(dataLen), "Error writing deflate length"); + + // Store to stream + rv = WriteTArray(aStream, outBuff); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +template<class T> +nsresult InflateReadTArray(nsIInputStream* aStream, FallibleTArray<T>* aOut, + uint32_t aExpectedSize) +{ + + uint32_t inLen; + uint32_t read; + nsresult rv = aStream->Read(reinterpret_cast<char*>(&inLen), sizeof(inLen), &read); + NS_ENSURE_SUCCESS(rv, rv); + + NS_ASSERTION(read == sizeof(inLen), "Error reading inflate length"); + + FallibleTArray<char> inBuff; + if (!inBuff.SetLength(inLen, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + rv = ReadTArray(aStream, &inBuff, inLen); + NS_ENSURE_SUCCESS(rv, rv); + + uLongf insize = inLen; + uLongf outsize = aExpectedSize * sizeof(T); + if (!aOut->SetLength(aExpectedSize, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + int zerr = uncompress(reinterpret_cast<Bytef*>(aOut->Elements()), + &outsize, + reinterpret_cast<const Bytef*>(inBuff.Elements()), + insize); + if (zerr != Z_OK) { + return NS_ERROR_FAILURE; + } + LOG(("InflateReadTArray: %d in %d out", insize, outsize)); + + NS_ASSERTION(outsize == aExpectedSize * sizeof(T), "Decompression size mismatch"); + + return NS_OK; +} + +static nsresult +ByteSliceWrite(nsIOutputStream* aOut, nsTArray<uint32_t>& aData) +{ + nsTArray<uint8_t> slice; + uint32_t count = aData.Length(); + + // Only process one slice at a time to avoid using too much memory. + if (!slice.SetLength(count, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + // Process slice 1. + for (uint32_t i = 0; i < count; i++) { + slice[i] = (aData[i] >> 24); + } + + nsresult rv = DeflateWriteTArray(aOut, slice); + NS_ENSURE_SUCCESS(rv, rv); + + // Process slice 2. + for (uint32_t i = 0; i < count; i++) { + slice[i] = ((aData[i] >> 16) & 0xFF); + } + + rv = DeflateWriteTArray(aOut, slice); + NS_ENSURE_SUCCESS(rv, rv); + + // Process slice 3. + for (uint32_t i = 0; i < count; i++) { + slice[i] = ((aData[i] >> 8) & 0xFF); + } + + rv = DeflateWriteTArray(aOut, slice); + NS_ENSURE_SUCCESS(rv, rv); + + // Process slice 4. + for (uint32_t i = 0; i < count; i++) { + slice[i] = (aData[i] & 0xFF); + } + + // The LSB slice is generally uncompressible, don't bother + // compressing it. + rv = WriteTArray(aOut, slice); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +static nsresult +ByteSliceRead(nsIInputStream* aInStream, FallibleTArray<uint32_t>* aData, uint32_t count) +{ + FallibleTArray<uint8_t> slice1; + FallibleTArray<uint8_t> slice2; + FallibleTArray<uint8_t> slice3; + FallibleTArray<uint8_t> slice4; + + nsresult rv = InflateReadTArray(aInStream, &slice1, count); + NS_ENSURE_SUCCESS(rv, rv); + + rv = InflateReadTArray(aInStream, &slice2, count); + NS_ENSURE_SUCCESS(rv, rv); + + rv = InflateReadTArray(aInStream, &slice3, count); + NS_ENSURE_SUCCESS(rv, rv); + + rv = ReadTArray(aInStream, &slice4, count); + NS_ENSURE_SUCCESS(rv, rv); + + if (!aData->SetCapacity(count, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + for (uint32_t i = 0; i < count; i++) { + aData->AppendElement((slice1[i] << 24) | + (slice2[i] << 16) | + (slice3[i] << 8) | + (slice4[i]), + fallible); + } + + return NS_OK; +} + +nsresult +HashStore::ReadAddPrefixes() +{ + FallibleTArray<uint32_t> chunks; + uint32_t count = mHeader.numAddPrefixes; + + nsresult rv = ByteSliceRead(mInputStream, &chunks, count); + NS_ENSURE_SUCCESS(rv, rv); + + if (!mAddPrefixes.SetCapacity(count, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + for (uint32_t i = 0; i < count; i++) { + AddPrefix *add = mAddPrefixes.AppendElement(fallible); + add->prefix.FromUint32(0); + add->addChunk = chunks[i]; + } + + return NS_OK; +} + +nsresult +HashStore::ReadSubPrefixes() +{ + FallibleTArray<uint32_t> addchunks; + FallibleTArray<uint32_t> subchunks; + FallibleTArray<uint32_t> prefixes; + uint32_t count = mHeader.numSubPrefixes; + + nsresult rv = ByteSliceRead(mInputStream, &addchunks, count); + NS_ENSURE_SUCCESS(rv, rv); + + rv = ByteSliceRead(mInputStream, &subchunks, count); + NS_ENSURE_SUCCESS(rv, rv); + + rv = ByteSliceRead(mInputStream, &prefixes, count); + NS_ENSURE_SUCCESS(rv, rv); + + if (!mSubPrefixes.SetCapacity(count, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + for (uint32_t i = 0; i < count; i++) { + SubPrefix *sub = mSubPrefixes.AppendElement(fallible); + sub->addChunk = addchunks[i]; + sub->prefix.FromUint32(prefixes[i]); + sub->subChunk = subchunks[i]; + } + + return NS_OK; +} + +// Split up PrefixArray back into the constituents +nsresult +HashStore::WriteAddPrefixes(nsIOutputStream* aOut) +{ + nsTArray<uint32_t> chunks; + uint32_t count = mAddPrefixes.Length(); + if (!chunks.SetCapacity(count, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + for (uint32_t i = 0; i < count; i++) { + chunks.AppendElement(mAddPrefixes[i].Chunk()); + } + + nsresult rv = ByteSliceWrite(aOut, chunks); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +HashStore::WriteSubPrefixes(nsIOutputStream* aOut) +{ + nsTArray<uint32_t> addchunks; + nsTArray<uint32_t> subchunks; + nsTArray<uint32_t> prefixes; + uint32_t count = mSubPrefixes.Length(); + addchunks.SetCapacity(count); + subchunks.SetCapacity(count); + prefixes.SetCapacity(count); + + for (uint32_t i = 0; i < count; i++) { + addchunks.AppendElement(mSubPrefixes[i].AddChunk()); + prefixes.AppendElement(mSubPrefixes[i].PrefixHash().ToUint32()); + subchunks.AppendElement(mSubPrefixes[i].Chunk()); + } + + nsresult rv = ByteSliceWrite(aOut, addchunks); + NS_ENSURE_SUCCESS(rv, rv); + + rv = ByteSliceWrite(aOut, subchunks); + NS_ENSURE_SUCCESS(rv, rv); + + rv = ByteSliceWrite(aOut, prefixes); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +HashStore::WriteFile() +{ + NS_ASSERTION(mInUpdate, "Must be in update to write database."); + if (nsUrlClassifierDBService::ShutdownHasStarted()) { + return NS_ERROR_ABORT; + } + + nsCOMPtr<nsIFile> storeFile; + nsresult rv = mStoreDirectory->Clone(getter_AddRefs(storeFile)); + NS_ENSURE_SUCCESS(rv, rv); + rv = storeFile->AppendNative(mTableName + NS_LITERAL_CSTRING(".sbstore")); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIOutputStream> out; + rv = NS_NewCheckSummedOutputStream(getter_AddRefs(out), storeFile, + PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE); + NS_ENSURE_SUCCESS(rv, rv); + + uint32_t written; + rv = out->Write(reinterpret_cast<char*>(&mHeader), sizeof(mHeader), &written); + NS_ENSURE_SUCCESS(rv, rv); + + // Write chunk numbers. + rv = mAddChunks.Write(out); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mSubChunks.Write(out); + NS_ENSURE_SUCCESS(rv, rv); + + // Write hashes. + rv = WriteAddPrefixes(out); + NS_ENSURE_SUCCESS(rv, rv); + + rv = WriteSubPrefixes(out); + NS_ENSURE_SUCCESS(rv, rv); + + rv = WriteTArray(out, mAddCompletes); + NS_ENSURE_SUCCESS(rv, rv); + + rv = WriteTArray(out, mSubCompletes); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsISafeOutputStream> safeOut = do_QueryInterface(out, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = safeOut->Finish(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +template <class T> +static void +Erase(FallibleTArray<T>* array, T* iterStart, T* iterEnd) +{ + uint32_t start = iterStart - array->Elements(); + uint32_t count = iterEnd - iterStart; + + if (count > 0) { + array->RemoveElementsAt(start, count); + } +} + +// Find items matching between |subs| and |adds|, and remove them, +// recording the item from |adds| in |adds_removed|. To minimize +// copies, the inputs are processing in parallel, so |subs| and |adds| +// should be compatibly ordered (either by SBAddPrefixLess or +// SBAddPrefixHashLess). +// +// |predAS| provides add < sub, |predSA| provides sub < add, for the +// tightest compare appropriate (see calls in SBProcessSubs). +template<class TSub, class TAdd> +static void +KnockoutSubs(FallibleTArray<TSub>* aSubs, FallibleTArray<TAdd>* aAdds) +{ + // Keep a pair of output iterators for writing kept items. Due to + // deletions, these may lag the main iterators. Using erase() on + // individual items would result in O(N^2) copies. Using a list + // would work around that, at double or triple the memory cost. + TAdd* addOut = aAdds->Elements(); + TAdd* addIter = aAdds->Elements(); + + TSub* subOut = aSubs->Elements(); + TSub* subIter = aSubs->Elements(); + + TAdd* addEnd = addIter + aAdds->Length(); + TSub* subEnd = subIter + aSubs->Length(); + + while (addIter != addEnd && subIter != subEnd) { + // additer compare, so it compares on add chunk + int32_t cmp = addIter->Compare(*subIter); + if (cmp > 0) { + // If |*sub_iter| < |*add_iter|, retain the sub. + *subOut = *subIter; + ++subOut; + ++subIter; + } else if (cmp < 0) { + // If |*add_iter| < |*sub_iter|, retain the add. + *addOut = *addIter; + ++addOut; + ++addIter; + } else { + // Drop equal items + ++addIter; + ++subIter; + } + } + + Erase(aAdds, addOut, addIter); + Erase(aSubs, subOut, subIter); +} + +// Remove items in |removes| from |fullHashes|. |fullHashes| and +// |removes| should be ordered by SBAddPrefix component. +template <class T> +static void +RemoveMatchingPrefixes(const SubPrefixArray& aSubs, FallibleTArray<T>* aFullHashes) +{ + // Where to store kept items. + T* out = aFullHashes->Elements(); + T* hashIter = out; + T* hashEnd = aFullHashes->Elements() + aFullHashes->Length(); + + SubPrefix const * removeIter = aSubs.Elements(); + SubPrefix const * removeEnd = aSubs.Elements() + aSubs.Length(); + + while (hashIter != hashEnd && removeIter != removeEnd) { + int32_t cmp = removeIter->CompareAlt(*hashIter); + if (cmp > 0) { + // Keep items less than |*removeIter|. + *out = *hashIter; + ++out; + ++hashIter; + } else if (cmp < 0) { + // No hit for |*removeIter|, bump it forward. + ++removeIter; + } else { + // Drop equal items, there may be multiple hits. + do { + ++hashIter; + } while (hashIter != hashEnd && + !(removeIter->CompareAlt(*hashIter) < 0)); + ++removeIter; + } + } + Erase(aFullHashes, out, hashIter); +} + +static void +RemoveDeadSubPrefixes(SubPrefixArray& aSubs, ChunkSet& aAddChunks) +{ + SubPrefix * subIter = aSubs.Elements(); + SubPrefix * subEnd = aSubs.Elements() + aSubs.Length(); + + for (SubPrefix * iter = subIter; iter != subEnd; iter++) { + bool hasChunk = aAddChunks.Has(iter->AddChunk()); + // Keep the subprefix if the chunk it refers to is one + // we haven't seen it yet. + if (!hasChunk) { + *subIter = *iter; + subIter++; + } + } + + LOG(("Removed %u dead SubPrefix entries.", subEnd - subIter)); + aSubs.TruncateLength(subIter - aSubs.Elements()); +} + +#ifdef DEBUG +template <class T> +static void EnsureSorted(FallibleTArray<T>* aArray) +{ + T* start = aArray->Elements(); + T* end = aArray->Elements() + aArray->Length(); + T* iter = start; + T* previous = start; + + while (iter != end) { + previous = iter; + ++iter; + if (iter != end) { + MOZ_ASSERT(iter->Compare(*previous) >= 0); + } + } + + return; +} +#endif + +nsresult +HashStore::ProcessSubs() +{ +#ifdef DEBUG + EnsureSorted(&mAddPrefixes); + EnsureSorted(&mSubPrefixes); + EnsureSorted(&mAddCompletes); + EnsureSorted(&mSubCompletes); + LOG(("All databases seem to have a consistent sort order.")); +#endif + + RemoveMatchingPrefixes(mSubPrefixes, &mAddCompletes); + RemoveMatchingPrefixes(mSubPrefixes, &mSubCompletes); + + // Remove any remaining subbed prefixes from both addprefixes + // and addcompletes. + KnockoutSubs(&mSubPrefixes, &mAddPrefixes); + KnockoutSubs(&mSubCompletes, &mAddCompletes); + + // Remove any remaining subprefixes referring to addchunks that + // we have (and hence have been processed above). + RemoveDeadSubPrefixes(mSubPrefixes, mAddChunks); + +#ifdef DEBUG + EnsureSorted(&mAddPrefixes); + EnsureSorted(&mSubPrefixes); + EnsureSorted(&mAddCompletes); + EnsureSorted(&mSubCompletes); + LOG(("All databases seem to have a consistent sort order.")); +#endif + + return NS_OK; +} + +nsresult +HashStore::AugmentAdds(const nsTArray<uint32_t>& aPrefixes) +{ + uint32_t cnt = aPrefixes.Length(); + if (cnt != mAddPrefixes.Length()) { + LOG(("Amount of prefixes in cache not consistent with store (%d vs %d)", + aPrefixes.Length(), mAddPrefixes.Length())); + return NS_ERROR_FAILURE; + } + for (uint32_t i = 0; i < cnt; i++) { + mAddPrefixes[i].prefix.FromUint32(aPrefixes[i]); + } + return NS_OK; +} + +ChunkSet& +HashStore::AddChunks() +{ + ReadChunkNumbers(); + + return mAddChunks; +} + +ChunkSet& +HashStore::SubChunks() +{ + ReadChunkNumbers(); + + return mSubChunks; +} + +AddCompleteArray& +HashStore::AddCompletes() +{ + ReadCompletions(); + + return mAddCompletes; +} + +SubCompleteArray& +HashStore::SubCompletes() +{ + ReadCompletions(); + + return mSubCompletes; +} + +bool +HashStore::AlreadyReadChunkNumbers() +{ + // If there are chunks but chunk set not yet contains any data + // Then we haven't read chunk numbers. + if ((mHeader.numAddChunks != 0 && mAddChunks.Length() == 0) || + (mHeader.numSubChunks != 0 && mSubChunks.Length() == 0)) { + return false; + } + return true; +} + +bool +HashStore::AlreadyReadCompletions() +{ + // If there are completions but completion set not yet contains any data + // Then we haven't read completions. + if ((mHeader.numAddCompletes != 0 && mAddCompletes.Length() == 0) || + (mHeader.numSubCompletes != 0 && mSubCompletes.Length() == 0)) { + return false; + } + return true; +} + +} // namespace safebrowsing +} // namespace mozilla diff --git a/toolkit/components/url-classifier/HashStore.h b/toolkit/components/url-classifier/HashStore.h new file mode 100644 index 000000000..3473b2f02 --- /dev/null +++ b/toolkit/components/url-classifier/HashStore.h @@ -0,0 +1,314 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef HashStore_h__ +#define HashStore_h__ + +#include "Entries.h" +#include "ChunkSet.h" + +#include "nsString.h" +#include "nsTArray.h" +#include "nsIFile.h" +#include "nsIFileStreams.h" +#include "nsCOMPtr.h" +#include "nsClassHashtable.h" +#include "safebrowsing.pb.h" +#include <string> + +namespace mozilla { +namespace safebrowsing { + +// The abstract class of TableUpdateV2 and TableUpdateV4. This +// is convenient for passing the TableUpdate* around associated +// with v2 and v4 instance. +class TableUpdate { +public: + TableUpdate(const nsACString& aTable) + : mTable(aTable) + { + } + + virtual ~TableUpdate() {} + + // To be overriden. + virtual bool Empty() const = 0; + + // Common interfaces. + const nsCString& TableName() const { return mTable; } + + template<typename T> + static T* Cast(TableUpdate* aThat) { + return (T::TAG == aThat->Tag() ? reinterpret_cast<T*>(aThat) : nullptr); + } + +private: + virtual int Tag() const = 0; + + nsCString mTable; +}; + +// A table update is built from a single update chunk from the server. As the +// protocol parser processes each chunk, it constructs a table update with the +// new hashes. +class TableUpdateV2 : public TableUpdate { +public: + explicit TableUpdateV2(const nsACString& aTable) + : TableUpdate(aTable) {} + + bool Empty() const override { + return mAddChunks.Length() == 0 && + mSubChunks.Length() == 0 && + mAddExpirations.Length() == 0 && + mSubExpirations.Length() == 0 && + mAddPrefixes.Length() == 0 && + mSubPrefixes.Length() == 0 && + mAddCompletes.Length() == 0 && + mSubCompletes.Length() == 0; + } + + // Throughout, uint32_t aChunk refers only to the chunk number. Chunk data is + // stored in the Prefix structures. + MOZ_MUST_USE nsresult NewAddChunk(uint32_t aChunk) { + return mAddChunks.Set(aChunk); + }; + MOZ_MUST_USE nsresult NewSubChunk(uint32_t aChunk) { + return mSubChunks.Set(aChunk); + }; + MOZ_MUST_USE nsresult NewAddExpiration(uint32_t aChunk) { + return mAddExpirations.Set(aChunk); + }; + MOZ_MUST_USE nsresult NewSubExpiration(uint32_t aChunk) { + return mSubExpirations.Set(aChunk); + }; + MOZ_MUST_USE nsresult NewAddPrefix(uint32_t aAddChunk, const Prefix& aPrefix); + MOZ_MUST_USE nsresult NewSubPrefix(uint32_t aAddChunk, + const Prefix& aPrefix, + uint32_t aSubChunk); + MOZ_MUST_USE nsresult NewAddComplete(uint32_t aChunk, + const Completion& aCompletion); + MOZ_MUST_USE nsresult NewSubComplete(uint32_t aAddChunk, + const Completion& aCompletion, + uint32_t aSubChunk); + + ChunkSet& AddChunks() { return mAddChunks; } + ChunkSet& SubChunks() { return mSubChunks; } + + // Expirations for chunks. + ChunkSet& AddExpirations() { return mAddExpirations; } + ChunkSet& SubExpirations() { return mSubExpirations; } + + // Hashes associated with this chunk. + AddPrefixArray& AddPrefixes() { return mAddPrefixes; } + SubPrefixArray& SubPrefixes() { return mSubPrefixes; } + AddCompleteArray& AddCompletes() { return mAddCompletes; } + SubCompleteArray& SubCompletes() { return mSubCompletes; } + + // For downcasting. + static const int TAG = 2; + +private: + + // The list of chunk numbers that we have for each of the type of chunks. + ChunkSet mAddChunks; + ChunkSet mSubChunks; + ChunkSet mAddExpirations; + ChunkSet mSubExpirations; + + // 4-byte sha256 prefixes. + AddPrefixArray mAddPrefixes; + SubPrefixArray mSubPrefixes; + + // 32-byte hashes. + AddCompleteArray mAddCompletes; + SubCompleteArray mSubCompletes; + + virtual int Tag() const override { return TAG; } +}; + +// Structure for DBService/HashStore/Classifiers to update. +// It would contain the prefixes (both fixed and variable length) +// for addition and indices to removal. See Bug 1283009. +class TableUpdateV4 : public TableUpdate { +public: + struct PrefixStdString { + private: + std::string mStorage; + nsDependentCSubstring mString; + + public: + explicit PrefixStdString(std::string& aString) + { + aString.swap(mStorage); + mString.Rebind(mStorage.data(), mStorage.size()); + }; + + const nsACString& GetPrefixString() const { return mString; }; + }; + + typedef nsClassHashtable<nsUint32HashKey, PrefixStdString> PrefixStdStringMap; + typedef nsTArray<int32_t> RemovalIndiceArray; + +public: + explicit TableUpdateV4(const nsACString& aTable) + : TableUpdate(aTable) + , mFullUpdate(false) + { + } + + bool Empty() const override + { + return mPrefixesMap.IsEmpty() && mRemovalIndiceArray.IsEmpty(); + } + + bool IsFullUpdate() const { return mFullUpdate; } + PrefixStdStringMap& Prefixes() { return mPrefixesMap; } + RemovalIndiceArray& RemovalIndices() { return mRemovalIndiceArray; } + const nsACString& ClientState() const { return mClientState; } + const nsACString& Checksum() const { return mChecksum; } + + // For downcasting. + static const int TAG = 4; + + void SetFullUpdate(bool aIsFullUpdate) { mFullUpdate = aIsFullUpdate; } + void NewPrefixes(int32_t aSize, std::string& aPrefixes); + void NewRemovalIndices(const uint32_t* aIndices, size_t aNumOfIndices); + void SetNewClientState(const nsACString& aState) { mClientState = aState; } + void NewChecksum(const std::string& aChecksum); + +private: + virtual int Tag() const override { return TAG; } + + bool mFullUpdate; + PrefixStdStringMap mPrefixesMap; + RemovalIndiceArray mRemovalIndiceArray; + nsCString mClientState; + nsCString mChecksum; +}; + +// There is one hash store per table. +class HashStore { +public: + HashStore(const nsACString& aTableName, + const nsACString& aProvider, + nsIFile* aRootStoreFile); + ~HashStore(); + + const nsCString& TableName() const { return mTableName; } + + nsresult Open(); + // Add Prefixes are stored partly in the PrefixSet (contains the + // Prefix data organized for fast lookup/low RAM usage) and partly in the + // HashStore (Add Chunk numbers - only used for updates, slow retrieval). + // AugmentAdds function joins the separate datasets into one complete + // prefixes+chunknumbers dataset. + nsresult AugmentAdds(const nsTArray<uint32_t>& aPrefixes); + + ChunkSet& AddChunks(); + ChunkSet& SubChunks(); + AddPrefixArray& AddPrefixes() { return mAddPrefixes; } + SubPrefixArray& SubPrefixes() { return mSubPrefixes; } + AddCompleteArray& AddCompletes(); + SubCompleteArray& SubCompletes(); + + // ======= + // Updates + // ======= + // Begin the update process. Reads the store into memory. + nsresult BeginUpdate(); + + // Imports the data from a TableUpdate. + nsresult ApplyUpdate(TableUpdate &aUpdate); + + // Process expired chunks + nsresult Expire(); + + // Rebuild the store, Incorporating all the applied updates. + nsresult Rebuild(); + + // Write the current state of the store to disk. + // If you call between ApplyUpdate() and Rebuild(), you'll + // have a mess on your hands. + nsresult WriteFile(); + + // Wipe out all Completes. + void ClearCompletes(); + +private: + nsresult Reset(); + + nsresult ReadHeader(); + nsresult SanityCheck(); + nsresult CalculateChecksum(nsAutoCString& aChecksum, uint32_t aFileSize, + bool aChecksumPresent); + nsresult CheckChecksum(uint32_t aFileSize); + void UpdateHeader(); + + nsresult ReadCompletions(); + nsresult ReadChunkNumbers(); + nsresult ReadHashes(); + + nsresult ReadAddPrefixes(); + nsresult ReadSubPrefixes(); + + nsresult WriteAddPrefixes(nsIOutputStream* aOut); + nsresult WriteSubPrefixes(nsIOutputStream* aOut); + + nsresult ProcessSubs(); + + nsresult PrepareForUpdate(); + + bool AlreadyReadChunkNumbers(); + bool AlreadyReadCompletions(); + + // This is used for checking that the database is correct and for figuring out + // the number of chunks, etc. to read from disk on restart. + struct Header { + uint32_t magic; + uint32_t version; + uint32_t numAddChunks; + uint32_t numSubChunks; + uint32_t numAddPrefixes; + uint32_t numSubPrefixes; + uint32_t numAddCompletes; + uint32_t numSubCompletes; + }; + + Header mHeader; + + // The name of the table (must end in -shavar or -digest256, or evidently + // -simple for unittesting. + nsCString mTableName; + nsCOMPtr<nsIFile> mStoreDirectory; + + bool mInUpdate; + + nsCOMPtr<nsIInputStream> mInputStream; + + // Chunk numbers, stored as uint32_t arrays. + ChunkSet mAddChunks; + ChunkSet mSubChunks; + + ChunkSet mAddExpirations; + ChunkSet mSubExpirations; + + // Chunk data for shavar tables. See Entries.h for format. + AddPrefixArray mAddPrefixes; + SubPrefixArray mSubPrefixes; + + // See bug 806422 for background. We must be able to distinguish between + // updates from the completion server and updates from the regular server. + AddCompleteArray mAddCompletes; + SubCompleteArray mSubCompletes; + + uint32_t mFileSize; + + // For gtest to inspect private members. + friend class PerProviderDirectoryTestUtils; +}; + +} // namespace safebrowsing +} // namespace mozilla + +#endif diff --git a/toolkit/components/url-classifier/LookupCache.cpp b/toolkit/components/url-classifier/LookupCache.cpp new file mode 100644 index 000000000..5a3b1e36d --- /dev/null +++ b/toolkit/components/url-classifier/LookupCache.cpp @@ -0,0 +1,599 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "LookupCache.h" +#include "HashStore.h" +#include "nsISeekableStream.h" +#include "mozilla/Telemetry.h" +#include "mozilla/Logging.h" +#include "nsNetUtil.h" +#include "prprf.h" +#include "Classifier.h" + +// We act as the main entry point for all the real lookups, +// so note that those are not done to the actual HashStore. +// The latter solely exists to store the data needed to handle +// the updates from the protocol. + +// This module provides a front for PrefixSet, mUpdateCompletions, +// and mGetHashCache, which together contain everything needed to +// provide a classification as long as the data is up to date. + +// PrefixSet stores and provides lookups for 4-byte prefixes. +// mUpdateCompletions contains 32-byte completions which were +// contained in updates. They are retrieved from HashStore/.sbtore +// on startup. +// mGetHashCache contains 32-byte completions which were +// returned from the gethash server. They are not serialized, +// only cached until the next update. + +// Name of the persistent PrefixSet storage +#define PREFIXSET_SUFFIX ".pset" + +// MOZ_LOG=UrlClassifierDbService:5 +extern mozilla::LazyLogModule gUrlClassifierDbServiceLog; +#define LOG(args) MOZ_LOG(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug, args) +#define LOG_ENABLED() MOZ_LOG_TEST(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug) + +namespace mozilla { +namespace safebrowsing { + +const int LookupCacheV2::VER = 2; + +LookupCache::LookupCache(const nsACString& aTableName, + const nsACString& aProvider, + nsIFile* aRootStoreDir) + : mPrimed(false) + , mTableName(aTableName) + , mProvider(aProvider) + , mRootStoreDirectory(aRootStoreDir) +{ + UpdateRootDirHandle(mRootStoreDirectory); +} + +nsresult +LookupCache::Open() +{ + LOG(("Loading PrefixSet")); + nsresult rv = LoadPrefixSet(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +LookupCache::UpdateRootDirHandle(nsIFile* aNewRootStoreDirectory) +{ + nsresult rv; + + if (aNewRootStoreDirectory != mRootStoreDirectory) { + rv = aNewRootStoreDirectory->Clone(getter_AddRefs(mRootStoreDirectory)); + NS_ENSURE_SUCCESS(rv, rv); + } + + rv = Classifier::GetPrivateStoreDirectory(mRootStoreDirectory, + mTableName, + mProvider, + getter_AddRefs(mStoreDirectory)); + + if (NS_FAILED(rv)) { + LOG(("Failed to get private store directory for %s", mTableName.get())); + mStoreDirectory = mRootStoreDirectory; + } + + if (LOG_ENABLED()) { + nsString path; + mStoreDirectory->GetPath(path); + LOG(("Private store directory for %s is %s", mTableName.get(), + NS_ConvertUTF16toUTF8(path).get())); + } + + return rv; +} + +nsresult +LookupCache::Reset() +{ + LOG(("LookupCache resetting")); + + nsCOMPtr<nsIFile> prefixsetFile; + nsresult rv = mStoreDirectory->Clone(getter_AddRefs(prefixsetFile)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = prefixsetFile->AppendNative(mTableName + NS_LITERAL_CSTRING(PREFIXSET_SUFFIX)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = prefixsetFile->Remove(false); + NS_ENSURE_SUCCESS(rv, rv); + + ClearAll(); + + return NS_OK; +} + +nsresult +LookupCache::AddCompletionsToCache(AddCompleteArray& aAddCompletes) +{ + for (uint32_t i = 0; i < aAddCompletes.Length(); i++) { + if (mGetHashCache.BinaryIndexOf(aAddCompletes[i].CompleteHash()) == mGetHashCache.NoIndex) { + mGetHashCache.AppendElement(aAddCompletes[i].CompleteHash()); + } + } + mGetHashCache.Sort(); + + return NS_OK; +} + +#if defined(DEBUG) +void +LookupCache::DumpCache() +{ + if (!LOG_ENABLED()) + return; + + for (uint32_t i = 0; i < mGetHashCache.Length(); i++) { + nsAutoCString str; + mGetHashCache[i].ToHexString(str); + LOG(("Caches: %s", str.get())); + } +} +#endif + +nsresult +LookupCache::WriteFile() +{ + if (nsUrlClassifierDBService::ShutdownHasStarted()) { + return NS_ERROR_ABORT; + } + + nsCOMPtr<nsIFile> psFile; + nsresult rv = mStoreDirectory->Clone(getter_AddRefs(psFile)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = psFile->AppendNative(mTableName + NS_LITERAL_CSTRING(PREFIXSET_SUFFIX)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = StoreToFile(psFile); + NS_WARNING_ASSERTION(NS_SUCCEEDED(rv), "failed to store the prefixset"); + + return NS_OK; +} + +void +LookupCache::ClearAll() +{ + ClearCache(); + ClearPrefixes(); + mPrimed = false; +} + +void +LookupCache::ClearCache() +{ + mGetHashCache.Clear(); +} + +/* static */ bool +LookupCache::IsCanonicalizedIP(const nsACString& aHost) +{ + // The canonicalization process will have left IP addresses in dotted + // decimal with no surprises. + uint32_t i1, i2, i3, i4; + char c; + if (PR_sscanf(PromiseFlatCString(aHost).get(), "%u.%u.%u.%u%c", + &i1, &i2, &i3, &i4, &c) == 4) { + return (i1 <= 0xFF && i2 <= 0xFF && i3 <= 0xFF && i4 <= 0xFF); + } + + return false; +} + +/* static */ nsresult +LookupCache::GetLookupFragments(const nsACString& aSpec, + nsTArray<nsCString>* aFragments) + +{ + aFragments->Clear(); + + nsACString::const_iterator begin, end, iter; + aSpec.BeginReading(begin); + aSpec.EndReading(end); + + iter = begin; + if (!FindCharInReadable('/', iter, end)) { + return NS_OK; + } + + const nsCSubstring& host = Substring(begin, iter++); + nsAutoCString path; + path.Assign(Substring(iter, end)); + + /** + * From the protocol doc: + * For the hostname, the client will try at most 5 different strings. They + * are: + * a) The exact hostname of the url + * b) The 4 hostnames formed by starting with the last 5 components and + * successivly removing the leading component. The top-level component + * can be skipped. This is not done if the hostname is a numerical IP. + */ + nsTArray<nsCString> hosts; + hosts.AppendElement(host); + + if (!IsCanonicalizedIP(host)) { + host.BeginReading(begin); + host.EndReading(end); + int numHostComponents = 0; + while (RFindInReadable(NS_LITERAL_CSTRING("."), begin, end) && + numHostComponents < MAX_HOST_COMPONENTS) { + // don't bother checking toplevel domains + if (++numHostComponents >= 2) { + host.EndReading(iter); + hosts.AppendElement(Substring(end, iter)); + } + end = begin; + host.BeginReading(begin); + } + } + + /** + * From the protocol doc: + * For the path, the client will also try at most 6 different strings. + * They are: + * a) the exact path of the url, including query parameters + * b) the exact path of the url, without query parameters + * c) the 4 paths formed by starting at the root (/) and + * successively appending path components, including a trailing + * slash. This behavior should only extend up to the next-to-last + * path component, that is, a trailing slash should never be + * appended that was not present in the original url. + */ + nsTArray<nsCString> paths; + nsAutoCString pathToAdd; + + path.BeginReading(begin); + path.EndReading(end); + iter = begin; + if (FindCharInReadable('?', iter, end)) { + pathToAdd = Substring(begin, iter); + paths.AppendElement(pathToAdd); + end = iter; + } + + int numPathComponents = 1; + iter = begin; + while (FindCharInReadable('/', iter, end) && + numPathComponents < MAX_PATH_COMPONENTS) { + iter++; + pathToAdd.Assign(Substring(begin, iter)); + paths.AppendElement(pathToAdd); + numPathComponents++; + } + + // If we haven't already done so, add the full path + if (!pathToAdd.Equals(path)) { + paths.AppendElement(path); + } + // Check an empty path (for whole-domain blacklist entries) + paths.AppendElement(EmptyCString()); + + for (uint32_t hostIndex = 0; hostIndex < hosts.Length(); hostIndex++) { + for (uint32_t pathIndex = 0; pathIndex < paths.Length(); pathIndex++) { + nsCString key; + key.Assign(hosts[hostIndex]); + key.Append('/'); + key.Append(paths[pathIndex]); + LOG(("Checking fragment %s", key.get())); + + aFragments->AppendElement(key); + } + } + + return NS_OK; +} + +/* static */ nsresult +LookupCache::GetHostKeys(const nsACString& aSpec, + nsTArray<nsCString>* aHostKeys) +{ + nsACString::const_iterator begin, end, iter; + aSpec.BeginReading(begin); + aSpec.EndReading(end); + + iter = begin; + if (!FindCharInReadable('/', iter, end)) { + return NS_OK; + } + + const nsCSubstring& host = Substring(begin, iter); + + if (IsCanonicalizedIP(host)) { + nsCString *key = aHostKeys->AppendElement(); + if (!key) + return NS_ERROR_OUT_OF_MEMORY; + + key->Assign(host); + key->Append("/"); + return NS_OK; + } + + nsTArray<nsCString> hostComponents; + ParseString(PromiseFlatCString(host), '.', hostComponents); + + if (hostComponents.Length() < 2) { + // no host or toplevel host, this won't match anything in the db + return NS_OK; + } + + // First check with two domain components + int32_t last = int32_t(hostComponents.Length()) - 1; + nsCString *lookupHost = aHostKeys->AppendElement(); + if (!lookupHost) + return NS_ERROR_OUT_OF_MEMORY; + + lookupHost->Assign(hostComponents[last - 1]); + lookupHost->Append("."); + lookupHost->Append(hostComponents[last]); + lookupHost->Append("/"); + + // Now check with three domain components + if (hostComponents.Length() > 2) { + nsCString *lookupHost2 = aHostKeys->AppendElement(); + if (!lookupHost2) + return NS_ERROR_OUT_OF_MEMORY; + lookupHost2->Assign(hostComponents[last - 2]); + lookupHost2->Append("."); + lookupHost2->Append(*lookupHost); + } + + return NS_OK; +} + +nsresult +LookupCache::LoadPrefixSet() +{ + nsCOMPtr<nsIFile> psFile; + nsresult rv = mStoreDirectory->Clone(getter_AddRefs(psFile)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = psFile->AppendNative(mTableName + NS_LITERAL_CSTRING(PREFIXSET_SUFFIX)); + NS_ENSURE_SUCCESS(rv, rv); + + bool exists; + rv = psFile->Exists(&exists); + NS_ENSURE_SUCCESS(rv, rv); + + if (exists) { + LOG(("stored PrefixSet exists, loading from disk")); + rv = LoadFromFile(psFile); + if (NS_FAILED(rv)) { + if (rv == NS_ERROR_FILE_CORRUPTED) { + Reset(); + } + return rv; + } + mPrimed = true; + } else { + LOG(("no (usable) stored PrefixSet found")); + } + +#ifdef DEBUG + if (mPrimed) { + uint32_t size = SizeOfPrefixSet(); + LOG(("SB tree done, size = %d bytes\n", size)); + } +#endif + + return NS_OK; +} + +nsresult +LookupCacheV2::Init() +{ + mPrefixSet = new nsUrlClassifierPrefixSet(); + nsresult rv = mPrefixSet->Init(mTableName); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +LookupCacheV2::Open() +{ + nsresult rv = LookupCache::Open(); + NS_ENSURE_SUCCESS(rv, rv); + + LOG(("Reading Completions")); + rv = ReadCompletions(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +void +LookupCacheV2::ClearAll() +{ + LookupCache::ClearAll(); + mUpdateCompletions.Clear(); +} + +nsresult +LookupCacheV2::Has(const Completion& aCompletion, + bool* aHas, bool* aComplete) +{ + *aHas = *aComplete = false; + + uint32_t prefix = aCompletion.ToUint32(); + + bool found; + nsresult rv = mPrefixSet->Contains(prefix, &found); + NS_ENSURE_SUCCESS(rv, rv); + + LOG(("Probe in %s: %X, found %d", mTableName.get(), prefix, found)); + + if (found) { + *aHas = true; + } + + if ((mGetHashCache.BinaryIndexOf(aCompletion) != nsTArray<Completion>::NoIndex) || + (mUpdateCompletions.BinaryIndexOf(aCompletion) != nsTArray<Completion>::NoIndex)) { + LOG(("Complete in %s", mTableName.get())); + *aComplete = true; + *aHas = true; + } + + return NS_OK; +} + +nsresult +LookupCacheV2::Build(AddPrefixArray& aAddPrefixes, + AddCompleteArray& aAddCompletes) +{ + Telemetry::Accumulate(Telemetry::URLCLASSIFIER_LC_COMPLETIONS, + static_cast<uint32_t>(aAddCompletes.Length())); + + mUpdateCompletions.Clear(); + mUpdateCompletions.SetCapacity(aAddCompletes.Length()); + for (uint32_t i = 0; i < aAddCompletes.Length(); i++) { + mUpdateCompletions.AppendElement(aAddCompletes[i].CompleteHash()); + } + aAddCompletes.Clear(); + mUpdateCompletions.Sort(); + + Telemetry::Accumulate(Telemetry::URLCLASSIFIER_LC_PREFIXES, + static_cast<uint32_t>(aAddPrefixes.Length())); + + nsresult rv = ConstructPrefixSet(aAddPrefixes); + NS_ENSURE_SUCCESS(rv, rv); + mPrimed = true; + + return NS_OK; +} + +nsresult +LookupCacheV2::GetPrefixes(FallibleTArray<uint32_t>& aAddPrefixes) +{ + if (!mPrimed) { + // This can happen if its a new table, so no error. + LOG(("GetPrefixes from empty LookupCache")); + return NS_OK; + } + return mPrefixSet->GetPrefixesNative(aAddPrefixes); +} + +nsresult +LookupCacheV2::ReadCompletions() +{ + HashStore store(mTableName, mProvider, mRootStoreDirectory); + + nsresult rv = store.Open(); + NS_ENSURE_SUCCESS(rv, rv); + + mUpdateCompletions.Clear(); + + const AddCompleteArray& addComplete = store.AddCompletes(); + for (uint32_t i = 0; i < addComplete.Length(); i++) { + mUpdateCompletions.AppendElement(addComplete[i].complete); + } + + return NS_OK; +} + +nsresult +LookupCacheV2::ClearPrefixes() +{ + return mPrefixSet->SetPrefixes(nullptr, 0); +} + +nsresult +LookupCacheV2::StoreToFile(nsIFile* aFile) +{ + return mPrefixSet->StoreToFile(aFile); +} + +nsresult +LookupCacheV2::LoadFromFile(nsIFile* aFile) +{ + return mPrefixSet->LoadFromFile(aFile); +} + +size_t +LookupCacheV2::SizeOfPrefixSet() +{ + return mPrefixSet->SizeOfIncludingThis(moz_malloc_size_of); +} + +#ifdef DEBUG +template <class T> +static void EnsureSorted(T* aArray) +{ + typename T::elem_type* start = aArray->Elements(); + typename T::elem_type* end = aArray->Elements() + aArray->Length(); + typename T::elem_type* iter = start; + typename T::elem_type* previous = start; + + while (iter != end) { + previous = iter; + ++iter; + if (iter != end) { + MOZ_ASSERT(*previous <= *iter); + } + } + return; +} +#endif + +nsresult +LookupCacheV2::ConstructPrefixSet(AddPrefixArray& aAddPrefixes) +{ + Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_CONSTRUCT_TIME> timer; + + nsTArray<uint32_t> array; + if (!array.SetCapacity(aAddPrefixes.Length(), fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + for (uint32_t i = 0; i < aAddPrefixes.Length(); i++) { + array.AppendElement(aAddPrefixes[i].PrefixHash().ToUint32()); + } + aAddPrefixes.Clear(); + +#ifdef DEBUG + // PrefixSet requires sorted order + EnsureSorted(&array); +#endif + + // construct new one, replace old entries + nsresult rv = mPrefixSet->SetPrefixes(array.Elements(), array.Length()); + NS_ENSURE_SUCCESS(rv, rv); + +#ifdef DEBUG + uint32_t size; + size = mPrefixSet->SizeOfIncludingThis(moz_malloc_size_of); + LOG(("SB tree done, size = %d bytes\n", size)); +#endif + + mPrimed = true; + + return NS_OK; +} + +#if defined(DEBUG) +void +LookupCacheV2::DumpCompletions() +{ + if (!LOG_ENABLED()) + return; + + for (uint32_t i = 0; i < mUpdateCompletions.Length(); i++) { + nsAutoCString str; + mUpdateCompletions[i].ToHexString(str); + LOG(("Update: %s", str.get())); + } +} +#endif + +} // namespace safebrowsing +} // namespace mozilla diff --git a/toolkit/components/url-classifier/LookupCache.h b/toolkit/components/url-classifier/LookupCache.h new file mode 100644 index 000000000..d815ed4fc --- /dev/null +++ b/toolkit/components/url-classifier/LookupCache.h @@ -0,0 +1,213 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef LookupCache_h__ +#define LookupCache_h__ + +#include "Entries.h" +#include "nsString.h" +#include "nsTArray.h" +#include "nsCOMPtr.h" +#include "nsIFile.h" +#include "nsIFileStreams.h" +#include "mozilla/RefPtr.h" +#include "nsUrlClassifierPrefixSet.h" +#include "VariableLengthPrefixSet.h" +#include "mozilla/Logging.h" + +namespace mozilla { +namespace safebrowsing { + +#define MAX_HOST_COMPONENTS 5 +#define MAX_PATH_COMPONENTS 4 + +class LookupResult { +public: + LookupResult() : mComplete(false), mNoise(false), + mFresh(false), mProtocolConfirmed(false) {} + + // The fragment that matched in the LookupCache + union { + Prefix prefix; + Completion complete; + } hash; + + const Prefix &PrefixHash() { + return hash.prefix; + } + const Completion &CompleteHash() { + MOZ_ASSERT(!mNoise); + return hash.complete; + } + + bool Confirmed() const { return (mComplete && mFresh) || mProtocolConfirmed; } + bool Complete() const { return mComplete; } + + // True if we have a complete match for this hash in the table. + bool mComplete; + + // True if this is a noise entry, i.e. an extra entry + // that is inserted to mask the true URL we are requesting. + // Noise entries will not have a complete 256-bit hash as + // they are fetched from the local 32-bit database and we + // don't know the corresponding full URL. + bool mNoise; + + // True if we've updated this table recently-enough. + bool mFresh; + + bool mProtocolConfirmed; + + nsCString mTableName; +}; + +typedef nsTArray<LookupResult> LookupResultArray; + +struct CacheResult { + AddComplete entry; + nsCString table; + + bool operator==(const CacheResult& aOther) const { + if (entry != aOther.entry) { + return false; + } + return table == aOther.table; + } +}; +typedef nsTArray<CacheResult> CacheResultArray; + +class LookupCache { +public: + // Check for a canonicalized IP address. + static bool IsCanonicalizedIP(const nsACString& aHost); + + // take a lookup string (www.hostname.com/path/to/resource.html) and + // expand it into the set of fragments that should be searched for in an + // entry + static nsresult GetLookupFragments(const nsACString& aSpec, + nsTArray<nsCString>* aFragments); + // Similar to GetKey(), but if the domain contains three or more components, + // two keys will be returned: + // hostname.com/foo/bar -> [hostname.com] + // mail.hostname.com/foo/bar -> [hostname.com, mail.hostname.com] + // www.mail.hostname.com/foo/bar -> [hostname.com, mail.hostname.com] + static nsresult GetHostKeys(const nsACString& aSpec, + nsTArray<nsCString>* aHostKeys); + + LookupCache(const nsACString& aTableName, + const nsACString& aProvider, + nsIFile* aStoreFile); + virtual ~LookupCache() {} + + const nsCString &TableName() const { return mTableName; } + + // The directory handle where we operate will + // be moved away when a backup is made. + nsresult UpdateRootDirHandle(nsIFile* aRootStoreDirectory); + + // This will Clear() the passed arrays when done. + nsresult AddCompletionsToCache(AddCompleteArray& aAddCompletes); + + // Write data stored in lookup cache to disk. + nsresult WriteFile(); + + // Clear completions retrieved from gethash request. + void ClearCache(); + + bool IsPrimed() const { return mPrimed; }; + +#if DEBUG + void DumpCache(); +#endif + + virtual nsresult Open(); + virtual nsresult Init() = 0; + virtual nsresult ClearPrefixes() = 0; + virtual nsresult Has(const Completion& aCompletion, + bool* aHas, bool* aComplete) = 0; + + virtual void ClearAll(); + + template<typename T> + static T* Cast(LookupCache* aThat) { + return ((aThat && T::VER == aThat->Ver()) ? reinterpret_cast<T*>(aThat) : nullptr); + } + +private: + nsresult Reset(); + nsresult LoadPrefixSet(); + + virtual nsresult StoreToFile(nsIFile* aFile) = 0; + virtual nsresult LoadFromFile(nsIFile* aFile) = 0; + virtual size_t SizeOfPrefixSet() = 0; + + virtual int Ver() const = 0; + +protected: + bool mPrimed; + nsCString mTableName; + nsCString mProvider; + nsCOMPtr<nsIFile> mRootStoreDirectory; + nsCOMPtr<nsIFile> mStoreDirectory; + + // Full length hashes obtained in gethash request + CompletionArray mGetHashCache; + + // For gtest to inspect private members. + friend class PerProviderDirectoryTestUtils; +}; + +class LookupCacheV2 final : public LookupCache +{ +public: + explicit LookupCacheV2(const nsACString& aTableName, + const nsACString& aProvider, + nsIFile* aStoreFile) + : LookupCache(aTableName, aProvider, aStoreFile) {} + ~LookupCacheV2() {} + + virtual nsresult Init() override; + virtual nsresult Open() override; + virtual void ClearAll() override; + virtual nsresult Has(const Completion& aCompletion, + bool* aHas, bool* aComplete) override; + + nsresult Build(AddPrefixArray& aAddPrefixes, + AddCompleteArray& aAddCompletes); + + nsresult GetPrefixes(FallibleTArray<uint32_t>& aAddPrefixes); + +#if DEBUG + void DumpCompletions(); +#endif + + static const int VER; + +protected: + nsresult ReadCompletions(); + + virtual nsresult ClearPrefixes() override; + virtual nsresult StoreToFile(nsIFile* aFile) override; + virtual nsresult LoadFromFile(nsIFile* aFile) override; + virtual size_t SizeOfPrefixSet() override; + +private: + virtual int Ver() const override { return VER; } + + // Construct a Prefix Set with known prefixes. + // This will Clear() aAddPrefixes when done. + nsresult ConstructPrefixSet(AddPrefixArray& aAddPrefixes); + + // Full length hashes obtained in update request + CompletionArray mUpdateCompletions; + + // Set of prefixes known to be in the database + RefPtr<nsUrlClassifierPrefixSet> mPrefixSet; +}; + +} // namespace safebrowsing +} // namespace mozilla + +#endif diff --git a/toolkit/components/url-classifier/LookupCacheV4.cpp b/toolkit/components/url-classifier/LookupCacheV4.cpp new file mode 100644 index 000000000..7258ae358 --- /dev/null +++ b/toolkit/components/url-classifier/LookupCacheV4.cpp @@ -0,0 +1,584 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "LookupCacheV4.h" +#include "HashStore.h" +#include "mozilla/Unused.h" +#include <string> + +// MOZ_LOG=UrlClassifierDbService:5 +extern mozilla::LazyLogModule gUrlClassifierDbServiceLog; +#define LOG(args) MOZ_LOG(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug, args) +#define LOG_ENABLED() MOZ_LOG_TEST(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug) + +#define METADATA_SUFFIX NS_LITERAL_CSTRING(".metadata") + +namespace mozilla { +namespace safebrowsing { + +const int LookupCacheV4::VER = 4; + +// Prefixes coming from updates and VLPrefixSet are both stored in the HashTable +// where the (key, value) pair is a prefix size and a lexicographic-sorted string. +// The difference is prefixes from updates use std:string(to avoid additional copies) +// and prefixes from VLPrefixSet use nsCString. +// This class provides a common interface for the partial update algorithm to make it +// easier to operate on two different kind prefix string map.. +class VLPrefixSet +{ +public: + explicit VLPrefixSet(const PrefixStringMap& aMap); + explicit VLPrefixSet(const TableUpdateV4::PrefixStdStringMap& aMap); + + // This function will merge the prefix map in VLPrefixSet to aPrefixMap. + void Merge(PrefixStringMap& aPrefixMap); + + // Find the smallest string from the map in VLPrefixSet. + bool GetSmallestPrefix(nsDependentCSubstring& aOutString); + + // Return the number of prefixes in the map + uint32_t Count() const { return mCount; } + +private: + // PrefixString structure contains a lexicographic-sorted string with + // a |pos| variable to indicate which substring we are pointing to right now. + // |pos| increases each time GetSmallestPrefix finds the smallest string. + struct PrefixString { + PrefixString(const nsACString& aStr, uint32_t aSize) + : pos(0) + , size(aSize) + { + data.Rebind(aStr.BeginReading(), aStr.Length()); + } + + const char* get() { + return pos < data.Length() ? data.BeginReading() + pos : nullptr; + } + void next() { pos += size; } + uint32_t remaining() { return data.Length() - pos; } + + nsDependentCSubstring data; + uint32_t pos; + uint32_t size; + }; + + nsClassHashtable<nsUint32HashKey, PrefixString> mMap; + uint32_t mCount; +}; + +nsresult +LookupCacheV4::Init() +{ + mVLPrefixSet = new VariableLengthPrefixSet(); + nsresult rv = mVLPrefixSet->Init(mTableName); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +LookupCacheV4::Has(const Completion& aCompletion, + bool* aHas, bool* aComplete) +{ + *aHas = false; + + uint32_t length = 0; + nsDependentCSubstring fullhash; + fullhash.Rebind((const char *)aCompletion.buf, COMPLETE_SIZE); + + nsresult rv = mVLPrefixSet->Matches(fullhash, &length); + NS_ENSURE_SUCCESS(rv, rv); + + *aHas = length >= PREFIX_SIZE; + *aComplete = length == COMPLETE_SIZE; + + if (LOG_ENABLED()) { + uint32_t prefix = aCompletion.ToUint32(); + LOG(("Probe in V4 %s: %X, found %d, complete %d", mTableName.get(), + prefix, *aHas, *aComplete)); + } + + return NS_OK; +} + +nsresult +LookupCacheV4::Build(PrefixStringMap& aPrefixMap) +{ + return mVLPrefixSet->SetPrefixes(aPrefixMap); +} + +nsresult +LookupCacheV4::GetPrefixes(PrefixStringMap& aPrefixMap) +{ + return mVLPrefixSet->GetPrefixes(aPrefixMap); +} + +nsresult +LookupCacheV4::ClearPrefixes() +{ + // Clear by seting a empty map + PrefixStringMap map; + return mVLPrefixSet->SetPrefixes(map); +} + +nsresult +LookupCacheV4::StoreToFile(nsIFile* aFile) +{ + return mVLPrefixSet->StoreToFile(aFile); +} + +nsresult +LookupCacheV4::LoadFromFile(nsIFile* aFile) +{ + nsresult rv = mVLPrefixSet->LoadFromFile(aFile); + if (NS_FAILED(rv)) { + return rv; + } + + nsCString state, checksum; + rv = LoadMetadata(state, checksum); + if (NS_FAILED(rv)) { + return rv; + } + + rv = VerifyChecksum(checksum); + Telemetry::Accumulate(Telemetry::URLCLASSIFIER_VLPS_LOAD_CORRUPT, + rv == NS_ERROR_FILE_CORRUPTED); + + return rv; +} + +size_t +LookupCacheV4::SizeOfPrefixSet() +{ + return mVLPrefixSet->SizeOfIncludingThis(moz_malloc_size_of); +} + +static void +AppendPrefixToMap(PrefixStringMap& prefixes, nsDependentCSubstring& prefix) +{ + if (!prefix.Length()) { + return; + } + + nsCString* prefixString = prefixes.LookupOrAdd(prefix.Length()); + prefixString->Append(prefix.BeginReading(), prefix.Length()); +} + +// Read prefix into a buffer and also update the hash which +// keeps track of the checksum +static void +UpdateChecksum(nsICryptoHash* aCrypto, const nsACString& aPrefix) +{ + MOZ_ASSERT(aCrypto); + aCrypto->Update(reinterpret_cast<uint8_t*>(const_cast<char*>( + aPrefix.BeginReading())), + aPrefix.Length()); +} + +// Please see https://bug1287058.bmoattachments.org/attachment.cgi?id=8795366 +// for detail about partial update algorithm. +nsresult +LookupCacheV4::ApplyUpdate(TableUpdateV4* aTableUpdate, + PrefixStringMap& aInputMap, + PrefixStringMap& aOutputMap) +{ + MOZ_ASSERT(aOutputMap.IsEmpty()); + + nsCOMPtr<nsICryptoHash> crypto; + nsresult rv = InitCrypto(crypto); + if (NS_FAILED(rv)) { + return rv; + } + + // oldPSet contains prefixes we already have or we just merged last round. + // addPSet contains prefixes stored in tableUpdate which should be merged with oldPSet. + VLPrefixSet oldPSet(aInputMap); + VLPrefixSet addPSet(aTableUpdate->Prefixes()); + + // RemovalIndiceArray is a sorted integer array indicating the index of prefix we should + // remove from the old prefix set(according to lexigraphic order). + // |removalIndex| is the current index of RemovalIndiceArray. + // |numOldPrefixPicked| is used to record how many prefixes we picked from the old map. + TableUpdateV4::RemovalIndiceArray& removalArray = aTableUpdate->RemovalIndices(); + uint32_t removalIndex = 0; + int32_t numOldPrefixPicked = -1; + + nsDependentCSubstring smallestOldPrefix; + nsDependentCSubstring smallestAddPrefix; + + bool isOldMapEmpty = false, isAddMapEmpty = false; + + // This is used to avoid infinite loop for partial update algorithm. + // The maximum loops will be the number of old prefixes plus the number of add prefixes. + int32_t index = oldPSet.Count() + addPSet.Count() + 1; + for(;index > 0; index--) { + // Get smallest prefix from the old prefix set if we don't have one + if (smallestOldPrefix.IsEmpty() && !isOldMapEmpty) { + isOldMapEmpty = !oldPSet.GetSmallestPrefix(smallestOldPrefix); + } + + // Get smallest prefix from add prefix set if we don't have one + if (smallestAddPrefix.IsEmpty() && !isAddMapEmpty) { + isAddMapEmpty = !addPSet.GetSmallestPrefix(smallestAddPrefix); + } + + bool pickOld; + + // If both prefix sets are not empty, then compare to find the smaller one. + if (!isOldMapEmpty && !isAddMapEmpty) { + if (smallestOldPrefix == smallestAddPrefix) { + LOG(("Add prefix should not exist in the original prefix set.")); + Telemetry::Accumulate(Telemetry::URLCLASSIFIER_UPDATE_ERROR_TYPE, + DUPLICATE_PREFIX); + return NS_ERROR_FAILURE; + } + + // Compare the smallest string in old prefix set and add prefix set, + // merge the smaller one into new map to ensure merged string still + // follows lexigraphic order. + pickOld = smallestOldPrefix < smallestAddPrefix; + } else if (!isOldMapEmpty && isAddMapEmpty) { + pickOld = true; + } else if (isOldMapEmpty && !isAddMapEmpty) { + pickOld = false; + // If both maps are empty, then partial update is complete. + } else { + break; + } + + if (pickOld) { + numOldPrefixPicked++; + + // If the number of picks from old map matches the removalIndex, then this prefix + // will be removed by not merging it to new map. + if (removalIndex < removalArray.Length() && + numOldPrefixPicked == removalArray[removalIndex]) { + removalIndex++; + } else { + AppendPrefixToMap(aOutputMap, smallestOldPrefix); + UpdateChecksum(crypto, smallestOldPrefix); + } + smallestOldPrefix.SetLength(0); + } else { + AppendPrefixToMap(aOutputMap, smallestAddPrefix); + UpdateChecksum(crypto, smallestAddPrefix); + + smallestAddPrefix.SetLength(0); + } + } + + // We expect index will be greater to 0 because max number of runs will be + // the number of original prefix plus add prefix. + if (index <= 0) { + LOG(("There are still prefixes remaining after reaching maximum runs.")); + Telemetry::Accumulate(Telemetry::URLCLASSIFIER_UPDATE_ERROR_TYPE, + INFINITE_LOOP); + return NS_ERROR_FAILURE; + } + + if (removalIndex < removalArray.Length()) { + LOG(("There are still prefixes to remove after exhausting the old PrefixSet.")); + Telemetry::Accumulate(Telemetry::URLCLASSIFIER_UPDATE_ERROR_TYPE, + WRONG_REMOVAL_INDICES); + return NS_ERROR_FAILURE; + } + + nsAutoCString checksum; + crypto->Finish(false, checksum); + if (aTableUpdate->Checksum().IsEmpty()) { + LOG(("Update checksum missing.")); + Telemetry::Accumulate(Telemetry::URLCLASSIFIER_UPDATE_ERROR_TYPE, + MISSING_CHECKSUM); + + // Generate our own checksum to tableUpdate to ensure there is always + // checksum in .metadata + std::string stdChecksum(checksum.BeginReading(), checksum.Length()); + aTableUpdate->NewChecksum(stdChecksum); + + } else if (aTableUpdate->Checksum() != checksum){ + LOG(("Checksum mismatch after applying partial update")); + Telemetry::Accumulate(Telemetry::URLCLASSIFIER_UPDATE_ERROR_TYPE, + CHECKSUM_MISMATCH); + return NS_ERROR_FAILURE; + } + + return NS_OK; +} + +nsresult +LookupCacheV4::InitCrypto(nsCOMPtr<nsICryptoHash>& aCrypto) +{ + nsresult rv; + aCrypto = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + + rv = aCrypto->Init(nsICryptoHash::SHA256); + NS_WARNING_ASSERTION(NS_SUCCEEDED(rv), "InitCrypto failed"); + + return rv; +} + +nsresult +LookupCacheV4::VerifyChecksum(const nsACString& aChecksum) +{ + nsCOMPtr<nsICryptoHash> crypto; + nsresult rv = InitCrypto(crypto); + if (NS_FAILED(rv)) { + return rv; + } + + PrefixStringMap map; + mVLPrefixSet->GetPrefixes(map); + + VLPrefixSet loadPSet(map); + uint32_t index = loadPSet.Count() + 1; + for(;index > 0; index--) { + nsDependentCSubstring prefix; + if (!loadPSet.GetSmallestPrefix(prefix)) { + break; + } + UpdateChecksum(crypto, prefix); + } + + nsAutoCString checksum; + crypto->Finish(false, checksum); + + if (checksum != aChecksum) { + LOG(("Checksum mismatch when loading prefixes from file.")); + return NS_ERROR_FILE_CORRUPTED; + } + + return NS_OK; +} + +////////////////////////////////////////////////////////////////////////// +// A set of lightweight functions for reading/writing value from/to file. + +namespace { + +template<typename T> +struct ValueTraits +{ + static uint32_t Length(const T& aValue) { return sizeof(T); } + static char* WritePtr(T& aValue, uint32_t aLength) { return (char*)&aValue; } + static const char* ReadPtr(const T& aValue) { return (char*)&aValue; } + static bool IsFixedLength() { return true; } +}; + +template<> +struct ValueTraits<nsACString> +{ + static bool IsFixedLength() { return false; } + + static uint32_t Length(const nsACString& aValue) + { + return aValue.Length(); + } + + static char* WritePtr(nsACString& aValue, uint32_t aLength) + { + aValue.SetLength(aLength); + return aValue.BeginWriting(); + } + + static const char* ReadPtr(const nsACString& aValue) + { + return aValue.BeginReading(); + } +}; + +template<typename T> static nsresult +WriteValue(nsIOutputStream *aOutputStream, const T& aValue) +{ + uint32_t writeLength = ValueTraits<T>::Length(aValue); + if (!ValueTraits<T>::IsFixedLength()) { + // We need to write out the variable value length. + nsresult rv = WriteValue(aOutputStream, writeLength); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Write out the value. + auto valueReadPtr = ValueTraits<T>::ReadPtr(aValue); + uint32_t written; + nsresult rv = aOutputStream->Write(valueReadPtr, writeLength, &written); + if (NS_FAILED(rv) || written != writeLength) { + LOG(("Failed to write the value.")); + return NS_FAILED(rv) ? rv : NS_ERROR_FAILURE; + } + + return rv; +} + +template<typename T> static nsresult +ReadValue(nsIInputStream* aInputStream, T& aValue) +{ + nsresult rv; + + uint32_t readLength; + if (ValueTraits<T>::IsFixedLength()) { + readLength = ValueTraits<T>::Length(aValue); + } else { + // Read the variable value length from file. + nsresult rv = ReadValue(aInputStream, readLength); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Read the value. + uint32_t read; + auto valueWritePtr = ValueTraits<T>::WritePtr(aValue, readLength); + rv = aInputStream->Read(valueWritePtr, readLength, &read); + if (NS_FAILED(rv) || read != readLength) { + LOG(("Failed to read the value.")); + return NS_FAILED(rv) ? rv : NS_ERROR_FAILURE; + } + + return rv; +} + +} // end of unnamed namespace. +//////////////////////////////////////////////////////////////////////// + +nsresult +LookupCacheV4::WriteMetadata(TableUpdateV4* aTableUpdate) +{ + NS_ENSURE_ARG_POINTER(aTableUpdate); + if (nsUrlClassifierDBService::ShutdownHasStarted()) { + return NS_ERROR_ABORT; + } + + nsCOMPtr<nsIFile> metaFile; + nsresult rv = mStoreDirectory->Clone(getter_AddRefs(metaFile)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = metaFile->AppendNative(mTableName + METADATA_SUFFIX); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIOutputStream> outputStream; + rv = NS_NewLocalFileOutputStream(getter_AddRefs(outputStream), metaFile, + PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE); + if (!NS_SUCCEEDED(rv)) { + LOG(("Unable to create file to store metadata.")); + return rv; + } + + // Write the state. + rv = WriteValue(outputStream, aTableUpdate->ClientState()); + if (NS_FAILED(rv)) { + LOG(("Failed to write the list state.")); + return rv; + } + + // Write the checksum. + rv = WriteValue(outputStream, aTableUpdate->Checksum()); + if (NS_FAILED(rv)) { + LOG(("Failed to write the list checksum.")); + return rv; + } + + return rv; +} + +nsresult +LookupCacheV4::LoadMetadata(nsACString& aState, nsACString& aChecksum) +{ + nsCOMPtr<nsIFile> metaFile; + nsresult rv = mStoreDirectory->Clone(getter_AddRefs(metaFile)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = metaFile->AppendNative(mTableName + METADATA_SUFFIX); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIInputStream> localInFile; + rv = NS_NewLocalFileInputStream(getter_AddRefs(localInFile), metaFile, + PR_RDONLY | nsIFile::OS_READAHEAD); + if (NS_FAILED(rv)) { + LOG(("Unable to open metadata file.")); + return rv; + } + + // Read the list state. + rv = ReadValue(localInFile, aState); + if (NS_FAILED(rv)) { + LOG(("Failed to read state.")); + return rv; + } + + // Read the checksum. + rv = ReadValue(localInFile, aChecksum); + if (NS_FAILED(rv)) { + LOG(("Failed to read checksum.")); + return rv; + } + + return rv; +} + +VLPrefixSet::VLPrefixSet(const PrefixStringMap& aMap) + : mCount(0) +{ + for (auto iter = aMap.ConstIter(); !iter.Done(); iter.Next()) { + uint32_t size = iter.Key(); + mMap.Put(size, new PrefixString(*iter.Data(), size)); + mCount += iter.Data()->Length() / size; + } +} + +VLPrefixSet::VLPrefixSet(const TableUpdateV4::PrefixStdStringMap& aMap) + : mCount(0) +{ + for (auto iter = aMap.ConstIter(); !iter.Done(); iter.Next()) { + uint32_t size = iter.Key(); + mMap.Put(size, new PrefixString(iter.Data()->GetPrefixString(), size)); + mCount += iter.Data()->GetPrefixString().Length() / size; + } +} + +void +VLPrefixSet::Merge(PrefixStringMap& aPrefixMap) { + for (auto iter = mMap.ConstIter(); !iter.Done(); iter.Next()) { + nsCString* prefixString = aPrefixMap.LookupOrAdd(iter.Key()); + PrefixString* str = iter.Data(); + + if (str->get()) { + prefixString->Append(str->get(), str->remaining()); + } + } +} + +bool +VLPrefixSet::GetSmallestPrefix(nsDependentCSubstring& aOutString) { + PrefixString* pick = nullptr; + for (auto iter = mMap.ConstIter(); !iter.Done(); iter.Next()) { + PrefixString* str = iter.Data(); + + if (!str->get()) { + continue; + } + + if (aOutString.IsEmpty()) { + aOutString.Rebind(str->get(), iter.Key()); + pick = str; + continue; + } + + nsDependentCSubstring cur(str->get(), iter.Key()); + if (cur < aOutString) { + aOutString.Rebind(str->get(), iter.Key()); + pick = str; + } + } + + if (pick) { + pick->next(); + } + + return pick != nullptr; +} + +} // namespace safebrowsing +} // namespace mozilla diff --git a/toolkit/components/url-classifier/LookupCacheV4.h b/toolkit/components/url-classifier/LookupCacheV4.h new file mode 100644 index 000000000..c2f3cafd2 --- /dev/null +++ b/toolkit/components/url-classifier/LookupCacheV4.h @@ -0,0 +1,70 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef LookupCacheV4_h__ +#define LookupCacheV4_h__ + +#include "LookupCache.h" + +namespace mozilla { +namespace safebrowsing { + +// Forward declaration. +class TableUpdateV4; + +class LookupCacheV4 final : public LookupCache +{ +public: + explicit LookupCacheV4(const nsACString& aTableName, + const nsACString& aProvider, + nsIFile* aStoreFile) + : LookupCache(aTableName, aProvider, aStoreFile) {} + ~LookupCacheV4() {} + + virtual nsresult Init() override; + virtual nsresult Has(const Completion& aCompletion, + bool* aHas, bool* aComplete) override; + + nsresult Build(PrefixStringMap& aPrefixMap); + + nsresult GetPrefixes(PrefixStringMap& aPrefixMap); + + // ApplyUpdate will merge data stored in aTableUpdate with prefixes in aInputMap. + nsresult ApplyUpdate(TableUpdateV4* aTableUpdate, + PrefixStringMap& aInputMap, + PrefixStringMap& aOutputMap); + + nsresult WriteMetadata(TableUpdateV4* aTableUpdate); + nsresult LoadMetadata(nsACString& aState, nsACString& aChecksum); + + static const int VER; + +protected: + virtual nsresult ClearPrefixes() override; + virtual nsresult StoreToFile(nsIFile* aFile) override; + virtual nsresult LoadFromFile(nsIFile* aFile) override; + virtual size_t SizeOfPrefixSet() override; + +private: + virtual int Ver() const override { return VER; } + + nsresult InitCrypto(nsCOMPtr<nsICryptoHash>& aCrypto); + nsresult VerifyChecksum(const nsACString& aChecksum); + + enum UPDATE_ERROR_TYPES { + DUPLICATE_PREFIX = 0, + INFINITE_LOOP = 1, + WRONG_REMOVAL_INDICES = 2, + CHECKSUM_MISMATCH = 3, + MISSING_CHECKSUM = 4, + }; + + RefPtr<VariableLengthPrefixSet> mVLPrefixSet; +}; + +} // namespace safebrowsing +} // namespace mozilla + +#endif diff --git a/toolkit/components/url-classifier/ProtocolParser.cpp b/toolkit/components/url-classifier/ProtocolParser.cpp new file mode 100644 index 000000000..5da7787be --- /dev/null +++ b/toolkit/components/url-classifier/ProtocolParser.cpp @@ -0,0 +1,1108 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ProtocolParser.h" +#include "LookupCache.h" +#include "nsNetCID.h" +#include "mozilla/Logging.h" +#include "prnetdb.h" +#include "prprf.h" + +#include "nsUrlClassifierDBService.h" +#include "nsUrlClassifierUtils.h" +#include "nsPrintfCString.h" +#include "mozilla/Base64.h" +#include "RiceDeltaDecoder.h" +#include "mozilla/EndianUtils.h" + +// MOZ_LOG=UrlClassifierProtocolParser:5 +mozilla::LazyLogModule gUrlClassifierProtocolParserLog("UrlClassifierProtocolParser"); +#define PARSER_LOG(args) MOZ_LOG(gUrlClassifierProtocolParserLog, mozilla::LogLevel::Debug, args) + +namespace mozilla { +namespace safebrowsing { + +// Updates will fail if fed chunks larger than this +const uint32_t MAX_CHUNK_SIZE = (1024 * 1024); +// Updates will fail if the total number of tocuhed chunks is larger than this +const uint32_t MAX_CHUNK_RANGE = 1000000; + +const uint32_t DOMAIN_SIZE = 4; + +// Parse one stringified range of chunks of the form "n" or "n-m" from a +// comma-separated list of chunks. Upon return, 'begin' will point to the +// next range of chunks in the list of chunks. +static bool +ParseChunkRange(nsACString::const_iterator& aBegin, + const nsACString::const_iterator& aEnd, + uint32_t* aFirst, uint32_t* aLast) +{ + nsACString::const_iterator iter = aBegin; + FindCharInReadable(',', iter, aEnd); + + nsAutoCString element(Substring(aBegin, iter)); + aBegin = iter; + if (aBegin != aEnd) + aBegin++; + + uint32_t numRead = PR_sscanf(element.get(), "%u-%u", aFirst, aLast); + if (numRead == 2) { + if (*aFirst > *aLast) { + uint32_t tmp = *aFirst; + *aFirst = *aLast; + *aLast = tmp; + } + return true; + } + + if (numRead == 1) { + *aLast = *aFirst; + return true; + } + + return false; +} + +/////////////////////////////////////////////////////////////// +// ProtocolParser implementation + +ProtocolParser::ProtocolParser() + : mUpdateStatus(NS_OK) + , mUpdateWaitSec(0) +{ +} + +ProtocolParser::~ProtocolParser() +{ + CleanupUpdates(); +} + +nsresult +ProtocolParser::Init(nsICryptoHash* aHasher) +{ + mCryptoHash = aHasher; + return NS_OK; +} + +void +ProtocolParser::CleanupUpdates() +{ + for (uint32_t i = 0; i < mTableUpdates.Length(); i++) { + delete mTableUpdates[i]; + } + mTableUpdates.Clear(); +} + +TableUpdate * +ProtocolParser::GetTableUpdate(const nsACString& aTable) +{ + for (uint32_t i = 0; i < mTableUpdates.Length(); i++) { + if (aTable.Equals(mTableUpdates[i]->TableName())) { + return mTableUpdates[i]; + } + } + + // We free automatically on destruction, ownership of these + // updates can be transferred to DBServiceWorker, which passes + // them back to Classifier when doing the updates, and that + // will free them. + TableUpdate *update = CreateTableUpdate(aTable); + mTableUpdates.AppendElement(update); + return update; +} + +/////////////////////////////////////////////////////////////////////// +// ProtocolParserV2 + +ProtocolParserV2::ProtocolParserV2() + : mState(PROTOCOL_STATE_CONTROL) + , mResetRequested(false) + , mTableUpdate(nullptr) +{ +} + +ProtocolParserV2::~ProtocolParserV2() +{ +} + +void +ProtocolParserV2::SetCurrentTable(const nsACString& aTable) +{ + auto update = GetTableUpdate(aTable); + mTableUpdate = TableUpdate::Cast<TableUpdateV2>(update); +} + +nsresult +ProtocolParserV2::AppendStream(const nsACString& aData) +{ + if (NS_FAILED(mUpdateStatus)) + return mUpdateStatus; + + nsresult rv; + mPending.Append(aData); + + bool done = false; + while (!done) { + if (nsUrlClassifierDBService::ShutdownHasStarted()) { + return NS_ERROR_ABORT; + } + + if (mState == PROTOCOL_STATE_CONTROL) { + rv = ProcessControl(&done); + } else if (mState == PROTOCOL_STATE_CHUNK) { + rv = ProcessChunk(&done); + } else { + NS_ERROR("Unexpected protocol state"); + rv = NS_ERROR_FAILURE; + } + if (NS_FAILED(rv)) { + mUpdateStatus = rv; + return rv; + } + } + return NS_OK; +} + +void +ProtocolParserV2::End() +{ + // Inbound data has already been processed in every AppendStream() call. +} + +nsresult +ProtocolParserV2::ProcessControl(bool* aDone) +{ + nsresult rv; + + nsAutoCString line; + *aDone = true; + while (NextLine(line)) { + PARSER_LOG(("Processing %s\n", line.get())); + + if (StringBeginsWith(line, NS_LITERAL_CSTRING("i:"))) { + // Set the table name from the table header line. + SetCurrentTable(Substring(line, 2)); + } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("n:"))) { + if (PR_sscanf(line.get(), "n:%d", &mUpdateWaitSec) != 1) { + PARSER_LOG(("Error parsing n: '%s' (%d)", line.get(), mUpdateWaitSec)); + return NS_ERROR_FAILURE; + } + } else if (line.EqualsLiteral("r:pleasereset")) { + mResetRequested = true; + } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("u:"))) { + rv = ProcessForward(line); + NS_ENSURE_SUCCESS(rv, rv); + } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("a:")) || + StringBeginsWith(line, NS_LITERAL_CSTRING("s:"))) { + rv = ProcessChunkControl(line); + NS_ENSURE_SUCCESS(rv, rv); + *aDone = false; + return NS_OK; + } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("ad:")) || + StringBeginsWith(line, NS_LITERAL_CSTRING("sd:"))) { + rv = ProcessExpirations(line); + NS_ENSURE_SUCCESS(rv, rv); + } + } + + *aDone = true; + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessExpirations(const nsCString& aLine) +{ + if (!mTableUpdate) { + NS_WARNING("Got an expiration without a table."); + return NS_ERROR_FAILURE; + } + const nsCSubstring &list = Substring(aLine, 3); + nsACString::const_iterator begin, end; + list.BeginReading(begin); + list.EndReading(end); + while (begin != end) { + uint32_t first, last; + if (ParseChunkRange(begin, end, &first, &last)) { + if (last < first) return NS_ERROR_FAILURE; + if (last - first > MAX_CHUNK_RANGE) return NS_ERROR_FAILURE; + for (uint32_t num = first; num <= last; num++) { + if (aLine[0] == 'a') { + nsresult rv = mTableUpdate->NewAddExpiration(num); + if (NS_FAILED(rv)) { + return rv; + } + } else { + nsresult rv = mTableUpdate->NewSubExpiration(num); + if (NS_FAILED(rv)) { + return rv; + } + } + } + } else { + return NS_ERROR_FAILURE; + } + } + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessChunkControl(const nsCString& aLine) +{ + if (!mTableUpdate) { + NS_WARNING("Got a chunk before getting a table."); + return NS_ERROR_FAILURE; + } + + mState = PROTOCOL_STATE_CHUNK; + char command; + + mChunkState.Clear(); + + if (PR_sscanf(aLine.get(), + "%c:%d:%d:%d", + &command, + &mChunkState.num, &mChunkState.hashSize, &mChunkState.length) + != 4) + { + NS_WARNING(("PR_sscanf failed")); + return NS_ERROR_FAILURE; + } + + if (mChunkState.length > MAX_CHUNK_SIZE) { + NS_WARNING("Invalid length specified in update."); + return NS_ERROR_FAILURE; + } + + if (!(mChunkState.hashSize == PREFIX_SIZE || mChunkState.hashSize == COMPLETE_SIZE)) { + NS_WARNING("Invalid hash size specified in update."); + return NS_ERROR_FAILURE; + } + + if (StringEndsWith(mTableUpdate->TableName(), + NS_LITERAL_CSTRING("-shavar")) || + StringEndsWith(mTableUpdate->TableName(), + NS_LITERAL_CSTRING("-simple"))) { + // Accommodate test tables ending in -simple for now. + mChunkState.type = (command == 'a') ? CHUNK_ADD : CHUNK_SUB; + } else if (StringEndsWith(mTableUpdate->TableName(), + NS_LITERAL_CSTRING("-digest256"))) { + mChunkState.type = (command == 'a') ? CHUNK_ADD_DIGEST : CHUNK_SUB_DIGEST; + } + nsresult rv; + switch (mChunkState.type) { + case CHUNK_ADD: + rv = mTableUpdate->NewAddChunk(mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + break; + case CHUNK_SUB: + rv = mTableUpdate->NewSubChunk(mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + break; + case CHUNK_ADD_DIGEST: + rv = mTableUpdate->NewAddChunk(mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + break; + case CHUNK_SUB_DIGEST: + rv = mTableUpdate->NewSubChunk(mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + break; + } + + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessForward(const nsCString& aLine) +{ + const nsCSubstring &forward = Substring(aLine, 2); + return AddForward(forward); +} + +nsresult +ProtocolParserV2::AddForward(const nsACString& aUrl) +{ + if (!mTableUpdate) { + NS_WARNING("Forward without a table name."); + return NS_ERROR_FAILURE; + } + + ForwardedUpdate *forward = mForwards.AppendElement(); + forward->table = mTableUpdate->TableName(); + forward->url.Assign(aUrl); + + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessChunk(bool* aDone) +{ + if (!mTableUpdate) { + NS_WARNING("Processing chunk without an active table."); + return NS_ERROR_FAILURE; + } + + NS_ASSERTION(mChunkState.num != 0, "Must have a chunk number."); + + if (mPending.Length() < mChunkState.length) { + *aDone = true; + return NS_OK; + } + + // Pull the chunk out of the pending stream data. + nsAutoCString chunk; + chunk.Assign(Substring(mPending, 0, mChunkState.length)); + mPending.Cut(0, mChunkState.length); + + *aDone = false; + mState = PROTOCOL_STATE_CONTROL; + + if (StringEndsWith(mTableUpdate->TableName(), + NS_LITERAL_CSTRING("-shavar"))) { + return ProcessShaChunk(chunk); + } + if (StringEndsWith(mTableUpdate->TableName(), + NS_LITERAL_CSTRING("-digest256"))) { + return ProcessDigestChunk(chunk); + } + return ProcessPlaintextChunk(chunk); +} + +/** + * Process a plaintext chunk (currently only used in unit tests). + */ +nsresult +ProtocolParserV2::ProcessPlaintextChunk(const nsACString& aChunk) +{ + if (!mTableUpdate) { + NS_WARNING("Chunk received with no table."); + return NS_ERROR_FAILURE; + } + + PARSER_LOG(("Handling a %d-byte simple chunk", aChunk.Length())); + + nsTArray<nsCString> lines; + ParseString(PromiseFlatCString(aChunk), '\n', lines); + + // non-hashed tables need to be hashed + for (uint32_t i = 0; i < lines.Length(); i++) { + nsCString& line = lines[i]; + + if (mChunkState.type == CHUNK_ADD) { + if (mChunkState.hashSize == COMPLETE_SIZE) { + Completion hash; + hash.FromPlaintext(line, mCryptoHash); + nsresult rv = mTableUpdate->NewAddComplete(mChunkState.num, hash); + if (NS_FAILED(rv)) { + return rv; + } + } else { + NS_ASSERTION(mChunkState.hashSize == 4, "Only 32- or 4-byte hashes can be used for add chunks."); + Prefix hash; + hash.FromPlaintext(line, mCryptoHash); + nsresult rv = mTableUpdate->NewAddPrefix(mChunkState.num, hash); + if (NS_FAILED(rv)) { + return rv; + } + } + } else { + nsCString::const_iterator begin, iter, end; + line.BeginReading(begin); + line.EndReading(end); + iter = begin; + uint32_t addChunk; + if (!FindCharInReadable(':', iter, end) || + PR_sscanf(lines[i].get(), "%d:", &addChunk) != 1) { + NS_WARNING("Received sub chunk without associated add chunk."); + return NS_ERROR_FAILURE; + } + iter++; + + if (mChunkState.hashSize == COMPLETE_SIZE) { + Completion hash; + hash.FromPlaintext(Substring(iter, end), mCryptoHash); + nsresult rv = mTableUpdate->NewSubComplete(addChunk, hash, mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + } else { + NS_ASSERTION(mChunkState.hashSize == 4, "Only 32- or 4-byte hashes can be used for add chunks."); + Prefix hash; + hash.FromPlaintext(Substring(iter, end), mCryptoHash); + nsresult rv = mTableUpdate->NewSubPrefix(addChunk, hash, mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + } + } + } + + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessShaChunk(const nsACString& aChunk) +{ + uint32_t start = 0; + while (start < aChunk.Length()) { + // First four bytes are the domain key. + Prefix domain; + domain.Assign(Substring(aChunk, start, DOMAIN_SIZE)); + start += DOMAIN_SIZE; + + // Then a count of entries. + uint8_t numEntries = static_cast<uint8_t>(aChunk[start]); + start++; + + PARSER_LOG(("Handling a %d-byte shavar chunk containing %u entries" + " for domain %X", aChunk.Length(), numEntries, + domain.ToUint32())); + + nsresult rv; + if (mChunkState.type == CHUNK_ADD && mChunkState.hashSize == PREFIX_SIZE) { + rv = ProcessHostAdd(domain, numEntries, aChunk, &start); + } else if (mChunkState.type == CHUNK_ADD && mChunkState.hashSize == COMPLETE_SIZE) { + rv = ProcessHostAddComplete(numEntries, aChunk, &start); + } else if (mChunkState.type == CHUNK_SUB && mChunkState.hashSize == PREFIX_SIZE) { + rv = ProcessHostSub(domain, numEntries, aChunk, &start); + } else if (mChunkState.type == CHUNK_SUB && mChunkState.hashSize == COMPLETE_SIZE) { + rv = ProcessHostSubComplete(numEntries, aChunk, &start); + } else { + NS_WARNING("Unexpected chunk type/hash size!"); + PARSER_LOG(("Got an unexpected chunk type/hash size: %s:%d", + mChunkState.type == CHUNK_ADD ? "add" : "sub", + mChunkState.hashSize)); + return NS_ERROR_FAILURE; + } + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessDigestChunk(const nsACString& aChunk) +{ + PARSER_LOG(("Handling a %d-byte digest256 chunk", aChunk.Length())); + + if (mChunkState.type == CHUNK_ADD_DIGEST) { + return ProcessDigestAdd(aChunk); + } + if (mChunkState.type == CHUNK_SUB_DIGEST) { + return ProcessDigestSub(aChunk); + } + return NS_ERROR_UNEXPECTED; +} + +nsresult +ProtocolParserV2::ProcessDigestAdd(const nsACString& aChunk) +{ + // The ABNF format for add chunks is (HASH)+, where HASH is 32 bytes. + MOZ_ASSERT(aChunk.Length() % 32 == 0, + "Chunk length in bytes must be divisible by 4"); + uint32_t start = 0; + while (start < aChunk.Length()) { + Completion hash; + hash.Assign(Substring(aChunk, start, COMPLETE_SIZE)); + start += COMPLETE_SIZE; + nsresult rv = mTableUpdate->NewAddComplete(mChunkState.num, hash); + if (NS_FAILED(rv)) { + return rv; + } + } + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessDigestSub(const nsACString& aChunk) +{ + // The ABNF format for sub chunks is (ADDCHUNKNUM HASH)+, where ADDCHUNKNUM + // is a 4 byte chunk number, and HASH is 32 bytes. + MOZ_ASSERT(aChunk.Length() % 36 == 0, + "Chunk length in bytes must be divisible by 36"); + uint32_t start = 0; + while (start < aChunk.Length()) { + // Read ADDCHUNKNUM + const nsCSubstring& addChunkStr = Substring(aChunk, start, 4); + start += 4; + + uint32_t addChunk; + memcpy(&addChunk, addChunkStr.BeginReading(), 4); + addChunk = PR_ntohl(addChunk); + + // Read the hash + Completion hash; + hash.Assign(Substring(aChunk, start, COMPLETE_SIZE)); + start += COMPLETE_SIZE; + + nsresult rv = mTableUpdate->NewSubComplete(addChunk, hash, mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + } + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessHostAdd(const Prefix& aDomain, uint8_t aNumEntries, + const nsACString& aChunk, uint32_t* aStart) +{ + NS_ASSERTION(mChunkState.hashSize == PREFIX_SIZE, + "ProcessHostAdd should only be called for prefix hashes."); + + if (aNumEntries == 0) { + nsresult rv = mTableUpdate->NewAddPrefix(mChunkState.num, aDomain); + if (NS_FAILED(rv)) { + return rv; + } + return NS_OK; + } + + if (*aStart + (PREFIX_SIZE * aNumEntries) > aChunk.Length()) { + NS_WARNING("Chunk is not long enough to contain the expected entries."); + return NS_ERROR_FAILURE; + } + + for (uint8_t i = 0; i < aNumEntries; i++) { + Prefix hash; + hash.Assign(Substring(aChunk, *aStart, PREFIX_SIZE)); + PARSER_LOG(("Add prefix %X", hash.ToUint32())); + nsresult rv = mTableUpdate->NewAddPrefix(mChunkState.num, hash); + if (NS_FAILED(rv)) { + return rv; + } + *aStart += PREFIX_SIZE; + } + + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessHostSub(const Prefix& aDomain, uint8_t aNumEntries, + const nsACString& aChunk, uint32_t *aStart) +{ + NS_ASSERTION(mChunkState.hashSize == PREFIX_SIZE, + "ProcessHostSub should only be called for prefix hashes."); + + if (aNumEntries == 0) { + if ((*aStart) + 4 > aChunk.Length()) { + NS_WARNING("Received a zero-entry sub chunk without an associated add."); + return NS_ERROR_FAILURE; + } + + const nsCSubstring& addChunkStr = Substring(aChunk, *aStart, 4); + *aStart += 4; + + uint32_t addChunk; + memcpy(&addChunk, addChunkStr.BeginReading(), 4); + addChunk = PR_ntohl(addChunk); + + PARSER_LOG(("Sub prefix (addchunk=%u)", addChunk)); + nsresult rv = mTableUpdate->NewSubPrefix(addChunk, aDomain, mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + return NS_OK; + } + + if (*aStart + ((PREFIX_SIZE + 4) * aNumEntries) > aChunk.Length()) { + NS_WARNING("Chunk is not long enough to contain the expected entries."); + return NS_ERROR_FAILURE; + } + + for (uint8_t i = 0; i < aNumEntries; i++) { + const nsCSubstring& addChunkStr = Substring(aChunk, *aStart, 4); + *aStart += 4; + + uint32_t addChunk; + memcpy(&addChunk, addChunkStr.BeginReading(), 4); + addChunk = PR_ntohl(addChunk); + + Prefix prefix; + prefix.Assign(Substring(aChunk, *aStart, PREFIX_SIZE)); + *aStart += PREFIX_SIZE; + + PARSER_LOG(("Sub prefix %X (addchunk=%u)", prefix.ToUint32(), addChunk)); + nsresult rv = mTableUpdate->NewSubPrefix(addChunk, prefix, mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + } + + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessHostAddComplete(uint8_t aNumEntries, + const nsACString& aChunk, uint32_t* aStart) +{ + NS_ASSERTION(mChunkState.hashSize == COMPLETE_SIZE, + "ProcessHostAddComplete should only be called for complete hashes."); + + if (aNumEntries == 0) { + // this is totally comprehensible. + // My sarcasm detector is going off! + NS_WARNING("Expected > 0 entries for a 32-byte hash add."); + return NS_OK; + } + + if (*aStart + (COMPLETE_SIZE * aNumEntries) > aChunk.Length()) { + NS_WARNING("Chunk is not long enough to contain the expected entries."); + return NS_ERROR_FAILURE; + } + + for (uint8_t i = 0; i < aNumEntries; i++) { + Completion hash; + hash.Assign(Substring(aChunk, *aStart, COMPLETE_SIZE)); + nsresult rv = mTableUpdate->NewAddComplete(mChunkState.num, hash); + if (NS_FAILED(rv)) { + return rv; + } + *aStart += COMPLETE_SIZE; + } + + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessHostSubComplete(uint8_t aNumEntries, + const nsACString& aChunk, uint32_t* aStart) +{ + NS_ASSERTION(mChunkState.hashSize == COMPLETE_SIZE, + "ProcessHostSubComplete should only be called for complete hashes."); + + if (aNumEntries == 0) { + // this is totally comprehensible. + NS_WARNING("Expected > 0 entries for a 32-byte hash sub."); + return NS_OK; + } + + if (*aStart + ((COMPLETE_SIZE + 4) * aNumEntries) > aChunk.Length()) { + NS_WARNING("Chunk is not long enough to contain the expected entries."); + return NS_ERROR_FAILURE; + } + + for (uint8_t i = 0; i < aNumEntries; i++) { + Completion hash; + hash.Assign(Substring(aChunk, *aStart, COMPLETE_SIZE)); + *aStart += COMPLETE_SIZE; + + const nsCSubstring& addChunkStr = Substring(aChunk, *aStart, 4); + *aStart += 4; + + uint32_t addChunk; + memcpy(&addChunk, addChunkStr.BeginReading(), 4); + addChunk = PR_ntohl(addChunk); + + nsresult rv = mTableUpdate->NewSubComplete(addChunk, hash, mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + } + + return NS_OK; +} + +bool +ProtocolParserV2::NextLine(nsACString& aLine) +{ + int32_t newline = mPending.FindChar('\n'); + if (newline == kNotFound) { + return false; + } + aLine.Assign(Substring(mPending, 0, newline)); + mPending.Cut(0, newline + 1); + return true; +} + +TableUpdate* +ProtocolParserV2::CreateTableUpdate(const nsACString& aTableName) const +{ + return new TableUpdateV2(aTableName); +} + +/////////////////////////////////////////////////////////////////////// +// ProtocolParserProtobuf + +ProtocolParserProtobuf::ProtocolParserProtobuf() +{ +} + +ProtocolParserProtobuf::~ProtocolParserProtobuf() +{ +} + +void +ProtocolParserProtobuf::SetCurrentTable(const nsACString& aTable) +{ + // Should never occur. + MOZ_ASSERT_UNREACHABLE("SetCurrentTable shouldn't be called"); +} + + +TableUpdate* +ProtocolParserProtobuf::CreateTableUpdate(const nsACString& aTableName) const +{ + return new TableUpdateV4(aTableName); +} + +nsresult +ProtocolParserProtobuf::AppendStream(const nsACString& aData) +{ + // Protobuf data cannot be parsed progressively. Just save the incoming data. + mPending.Append(aData); + return NS_OK; +} + +void +ProtocolParserProtobuf::End() +{ + // mUpdateStatus will be updated to success as long as not all + // the responses are invalid. + mUpdateStatus = NS_ERROR_FAILURE; + + FetchThreatListUpdatesResponse response; + if (!response.ParseFromArray(mPending.get(), mPending.Length())) { + NS_WARNING("ProtocolParserProtobuf failed parsing data."); + return; + } + + auto minWaitDuration = response.minimum_wait_duration(); + mUpdateWaitSec = minWaitDuration.seconds() + + minWaitDuration.nanos() / 1000000000; + + for (int i = 0; i < response.list_update_responses_size(); i++) { + auto r = response.list_update_responses(i); + nsresult rv = ProcessOneResponse(r); + if (NS_SUCCEEDED(rv)) { + mUpdateStatus = rv; + } else { + NS_WARNING("Failed to process one response."); + } + } +} + +nsresult +ProtocolParserProtobuf::ProcessOneResponse(const ListUpdateResponse& aResponse) +{ + // A response must have a threat type. + if (!aResponse.has_threat_type()) { + NS_WARNING("Threat type not initialized. This seems to be an invalid response."); + return NS_ERROR_FAILURE; + } + + // Convert threat type to list name. + nsCOMPtr<nsIUrlClassifierUtils> urlUtil = + do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID); + nsCString possibleListNames; + nsresult rv = urlUtil->ConvertThreatTypeToListNames(aResponse.threat_type(), + possibleListNames); + if (NS_FAILED(rv)) { + PARSER_LOG((nsPrintfCString("Threat type to list name conversion error: %d", + aResponse.threat_type())).get()); + return NS_ERROR_FAILURE; + } + + // Match the table name we received with one of the ones we requested. + // We ignore the case where a threat type matches more than one list + // per provider and return the first one. See bug 1287059." + nsCString listName; + nsTArray<nsCString> possibleListNameArray; + Classifier::SplitTables(possibleListNames, possibleListNameArray); + for (auto possibleName : possibleListNameArray) { + if (mRequestedTables.Contains(possibleName)) { + listName = possibleName; + break; + } + } + + if (listName.IsEmpty()) { + PARSER_LOG(("We received an update for a list we didn't ask for. Ignoring it.")); + return NS_ERROR_FAILURE; + } + + // Test if this is a full update. + bool isFullUpdate = false; + if (aResponse.has_response_type()) { + isFullUpdate = + aResponse.response_type() == ListUpdateResponse::FULL_UPDATE; + } else { + NS_WARNING("Response type not initialized."); + return NS_ERROR_FAILURE; + } + + // Warn if there's no new state. + if (!aResponse.has_new_client_state()) { + NS_WARNING("New state not initialized."); + return NS_ERROR_FAILURE; + } + + auto tu = GetTableUpdate(nsCString(listName.get())); + auto tuV4 = TableUpdate::Cast<TableUpdateV4>(tu); + NS_ENSURE_TRUE(tuV4, NS_ERROR_FAILURE); + + nsCString state(aResponse.new_client_state().c_str(), + aResponse.new_client_state().size()); + tuV4->SetNewClientState(state); + + if (aResponse.has_checksum()) { + tuV4->NewChecksum(aResponse.checksum().sha256()); + } + + PARSER_LOG(("==== Update for threat type '%d' ====", aResponse.threat_type())); + PARSER_LOG(("* listName: %s\n", listName.get())); + PARSER_LOG(("* newState: %s\n", aResponse.new_client_state().c_str())); + PARSER_LOG(("* isFullUpdate: %s\n", (isFullUpdate ? "yes" : "no"))); + PARSER_LOG(("* hasChecksum: %s\n", (aResponse.has_checksum() ? "yes" : "no"))); + + tuV4->SetFullUpdate(isFullUpdate); + + rv = ProcessAdditionOrRemoval(*tuV4, aResponse.additions(), true /*aIsAddition*/); + NS_ENSURE_SUCCESS(rv, rv); + rv = ProcessAdditionOrRemoval(*tuV4, aResponse.removals(), false); + NS_ENSURE_SUCCESS(rv, rv); + + PARSER_LOG(("\n\n")); + + return NS_OK; +} + +nsresult +ProtocolParserProtobuf::ProcessAdditionOrRemoval(TableUpdateV4& aTableUpdate, + const ThreatEntrySetList& aUpdate, + bool aIsAddition) +{ + nsresult ret = NS_OK; + + for (int i = 0; i < aUpdate.size(); i++) { + auto update = aUpdate.Get(i); + if (!update.has_compression_type()) { + NS_WARNING(nsPrintfCString("%s with no compression type.", + aIsAddition ? "Addition" : "Removal").get()); + continue; + } + + switch (update.compression_type()) { + case COMPRESSION_TYPE_UNSPECIFIED: + NS_WARNING("Unspecified compression type."); + break; + + case RAW: + ret = (aIsAddition ? ProcessRawAddition(aTableUpdate, update) + : ProcessRawRemoval(aTableUpdate, update)); + break; + + case RICE: + ret = (aIsAddition ? ProcessEncodedAddition(aTableUpdate, update) + : ProcessEncodedRemoval(aTableUpdate, update)); + break; + } + } + + return ret; +} + +nsresult +ProtocolParserProtobuf::ProcessRawAddition(TableUpdateV4& aTableUpdate, + const ThreatEntrySet& aAddition) +{ + if (!aAddition.has_raw_hashes()) { + PARSER_LOG(("* No raw addition.")); + return NS_OK; + } + + auto rawHashes = aAddition.raw_hashes(); + if (!rawHashes.has_prefix_size()) { + NS_WARNING("Raw hash has no prefix size"); + return NS_OK; + } + + auto prefixes = rawHashes.raw_hashes(); + if (4 == rawHashes.prefix_size()) { + // Process fixed length prefixes separately. + uint32_t* fixedLengthPrefixes = (uint32_t*)prefixes.c_str(); + size_t numOfFixedLengthPrefixes = prefixes.size() / 4; + PARSER_LOG(("* Raw addition (4 bytes)")); + PARSER_LOG((" - # of prefixes: %d", numOfFixedLengthPrefixes)); + PARSER_LOG((" - Memory address: 0x%p", fixedLengthPrefixes)); + } else { + // TODO: Process variable length prefixes including full hashes. + // See Bug 1283009. + PARSER_LOG((" Raw addition (%d bytes)", rawHashes.prefix_size())); + } + + if (!rawHashes.mutable_raw_hashes()) { + PARSER_LOG(("Unable to get mutable raw hashes. Can't perform a string move.")); + return NS_ERROR_FAILURE; + } + + aTableUpdate.NewPrefixes(rawHashes.prefix_size(), + *rawHashes.mutable_raw_hashes()); + + return NS_OK; +} + +nsresult +ProtocolParserProtobuf::ProcessRawRemoval(TableUpdateV4& aTableUpdate, + const ThreatEntrySet& aRemoval) +{ + if (!aRemoval.has_raw_indices()) { + NS_WARNING("A removal has no indices."); + return NS_OK; + } + + // indices is an array of int32. + auto indices = aRemoval.raw_indices().indices(); + PARSER_LOG(("* Raw removal")); + PARSER_LOG((" - # of removal: %d", indices.size())); + + aTableUpdate.NewRemovalIndices((const uint32_t*)indices.data(), + indices.size()); + + return NS_OK; +} + +static nsresult +DoRiceDeltaDecode(const RiceDeltaEncoding& aEncoding, + nsTArray<uint32_t>& aDecoded) +{ + if (!aEncoding.has_first_value()) { + PARSER_LOG(("The encoding info is incomplete.")); + return NS_ERROR_FAILURE; + } + if (aEncoding.num_entries() > 0 && + (!aEncoding.has_rice_parameter() || !aEncoding.has_encoded_data())) { + PARSER_LOG(("Rice parameter or encoded data is missing.")); + return NS_ERROR_FAILURE; + } + + PARSER_LOG(("* Encoding info:")); + PARSER_LOG((" - First value: %d", aEncoding.first_value())); + PARSER_LOG((" - Num of entries: %d", aEncoding.num_entries())); + PARSER_LOG((" - Rice parameter: %d", aEncoding.rice_parameter())); + + // Set up the input buffer. Note that the bits should be read + // from LSB to MSB so that we in-place reverse the bits before + // feeding to the decoder. + auto encoded = const_cast<RiceDeltaEncoding&>(aEncoding).mutable_encoded_data(); + RiceDeltaDecoder decoder((uint8_t*)encoded->c_str(), encoded->size()); + + // Setup the output buffer. The "first value" is included in + // the output buffer. + aDecoded.SetLength(aEncoding.num_entries() + 1); + + // Decode! + bool rv = decoder.Decode(aEncoding.rice_parameter(), + aEncoding.first_value(), // first value. + aEncoding.num_entries(), // # of entries (first value not included). + &aDecoded[0]); + + NS_ENSURE_TRUE(rv, NS_ERROR_FAILURE); + + return NS_OK; +} + +nsresult +ProtocolParserProtobuf::ProcessEncodedAddition(TableUpdateV4& aTableUpdate, + const ThreatEntrySet& aAddition) +{ + if (!aAddition.has_rice_hashes()) { + PARSER_LOG(("* No rice encoded addition.")); + return NS_OK; + } + + nsTArray<uint32_t> decoded; + nsresult rv = DoRiceDeltaDecode(aAddition.rice_hashes(), decoded); + if (NS_FAILED(rv)) { + PARSER_LOG(("Failed to parse encoded prefixes.")); + return rv; + } + + // Say we have the following raw prefixes + // BE LE + // 00 00 00 01 1 16777216 + // 00 00 02 00 512 131072 + // 00 03 00 00 196608 768 + // 04 00 00 00 67108864 4 + // + // which can be treated as uint32 (big-endian) sorted in increasing order: + // + // [1, 512, 196608, 67108864] + // + // According to https://developers.google.com/safe-browsing/v4/compression, + // the following should be done prior to compression: + // + // 1) re-interpret in little-endian ==> [16777216, 131072, 768, 4] + // 2) sort in increasing order ==> [4, 768, 131072, 16777216] + // + // In order to get the original byte stream from |decoded| + // ([4, 768, 131072, 16777216] in this case), we have to: + // + // 1) sort in big-endian order ==> [16777216, 131072, 768, 4] + // 2) copy each uint32 in little-endian to the result string + // + + // The 4-byte prefixes have to be re-sorted in Big-endian increasing order. + struct CompareBigEndian + { + bool Equals(const uint32_t& aA, const uint32_t& aB) const + { + return aA == aB; + } + + bool LessThan(const uint32_t& aA, const uint32_t& aB) const + { + return NativeEndian::swapToBigEndian(aA) < + NativeEndian::swapToBigEndian(aB); + } + }; + decoded.Sort(CompareBigEndian()); + + // The encoded prefixes are always 4 bytes. + std::string prefixes; + for (size_t i = 0; i < decoded.Length(); i++) { + // Note that the third argument is the number of elements we want + // to copy (and swap) but not the number of bytes we want to copy. + char p[4]; + NativeEndian::copyAndSwapToLittleEndian(p, &decoded[i], 1); + prefixes.append(p, 4); + } + + aTableUpdate.NewPrefixes(4, prefixes); + + return NS_OK; +} + +nsresult +ProtocolParserProtobuf::ProcessEncodedRemoval(TableUpdateV4& aTableUpdate, + const ThreatEntrySet& aRemoval) +{ + if (!aRemoval.has_rice_indices()) { + PARSER_LOG(("* No rice encoded removal.")); + return NS_OK; + } + + nsTArray<uint32_t> decoded; + nsresult rv = DoRiceDeltaDecode(aRemoval.rice_indices(), decoded); + if (NS_FAILED(rv)) { + PARSER_LOG(("Failed to decode encoded removal indices.")); + return rv; + } + + // The encoded prefixes are always 4 bytes. + aTableUpdate.NewRemovalIndices(&decoded[0], decoded.Length()); + + return NS_OK; +} + +} // namespace safebrowsing +} // namespace mozilla diff --git a/toolkit/components/url-classifier/ProtocolParser.h b/toolkit/components/url-classifier/ProtocolParser.h new file mode 100644 index 000000000..ec1a695f4 --- /dev/null +++ b/toolkit/components/url-classifier/ProtocolParser.h @@ -0,0 +1,204 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProtocolParser_h__ +#define ProtocolParser_h__ + +#include "HashStore.h" +#include "nsICryptoHMAC.h" +#include "safebrowsing.pb.h" + +namespace mozilla { +namespace safebrowsing { + +/** + * Abstract base class for parsing update data in multiple formats. + */ +class ProtocolParser { +public: + struct ForwardedUpdate { + nsCString table; + nsCString url; + }; + + ProtocolParser(); + virtual ~ProtocolParser(); + + nsresult Status() const { return mUpdateStatus; } + + nsresult Init(nsICryptoHash* aHasher); + +#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES + nsCString GetRawTableUpdates() const { return mPending; } +#endif + + virtual void SetCurrentTable(const nsACString& aTable) = 0; + + void SetRequestedTables(const nsTArray<nsCString>& aRequestTables) + { + mRequestedTables = aRequestTables; + } + + nsresult Begin(); + virtual nsresult AppendStream(const nsACString& aData) = 0; + + uint32_t UpdateWaitSec() { return mUpdateWaitSec; } + + // Notify that the inbound data is ready for parsing if progressive + // parsing is not supported, for example in V4. + virtual void End() = 0; + + // Forget the table updates that were created by this pass. It + // becomes the caller's responsibility to free them. This is shitty. + TableUpdate *GetTableUpdate(const nsACString& aTable); + void ForgetTableUpdates() { mTableUpdates.Clear(); } + nsTArray<TableUpdate*> &GetTableUpdates() { return mTableUpdates; } + + // These are only meaningful to V2. Since they were originally public, + // moving them to ProtocolParserV2 requires a dymamic cast in the call + // sites. As a result, we will leave them until we remove support + // for V2 entirely.. + virtual const nsTArray<ForwardedUpdate> &Forwards() const { return mForwards; } + virtual bool ResetRequested() { return false; } + +protected: + virtual TableUpdate* CreateTableUpdate(const nsACString& aTableName) const = 0; + + nsCString mPending; + nsresult mUpdateStatus; + + // Keep track of updates to apply before passing them to the DBServiceWorkers. + nsTArray<TableUpdate*> mTableUpdates; + + nsTArray<ForwardedUpdate> mForwards; + nsCOMPtr<nsICryptoHash> mCryptoHash; + + // The table names that were requested from the client. + nsTArray<nsCString> mRequestedTables; + + // How long we should wait until the next update. + uint32_t mUpdateWaitSec; + +private: + void CleanupUpdates(); +}; + +/** + * Helpers to parse the "shavar", "digest256" and "simple" list formats. + */ +class ProtocolParserV2 final : public ProtocolParser { +public: + ProtocolParserV2(); + virtual ~ProtocolParserV2(); + + virtual void SetCurrentTable(const nsACString& aTable) override; + virtual nsresult AppendStream(const nsACString& aData) override; + virtual void End() override; + + // Update information. + virtual const nsTArray<ForwardedUpdate> &Forwards() const override { return mForwards; } + virtual bool ResetRequested() override { return mResetRequested; } + +private: + virtual TableUpdate* CreateTableUpdate(const nsACString& aTableName) const override; + + nsresult ProcessControl(bool* aDone); + nsresult ProcessExpirations(const nsCString& aLine); + nsresult ProcessChunkControl(const nsCString& aLine); + nsresult ProcessForward(const nsCString& aLine); + nsresult AddForward(const nsACString& aUrl); + nsresult ProcessChunk(bool* done); + // Remove this, it's only used for testing + nsresult ProcessPlaintextChunk(const nsACString& aChunk); + nsresult ProcessShaChunk(const nsACString& aChunk); + nsresult ProcessHostAdd(const Prefix& aDomain, uint8_t aNumEntries, + const nsACString& aChunk, uint32_t* aStart); + nsresult ProcessHostSub(const Prefix& aDomain, uint8_t aNumEntries, + const nsACString& aChunk, uint32_t* aStart); + nsresult ProcessHostAddComplete(uint8_t aNumEntries, const nsACString& aChunk, + uint32_t *aStart); + nsresult ProcessHostSubComplete(uint8_t numEntries, const nsACString& aChunk, + uint32_t* start); + // Digest chunks are very similar to shavar chunks, except digest chunks + // always contain the full hash, so there is no need for chunk data to + // contain prefix sizes. + nsresult ProcessDigestChunk(const nsACString& aChunk); + nsresult ProcessDigestAdd(const nsACString& aChunk); + nsresult ProcessDigestSub(const nsACString& aChunk); + bool NextLine(nsACString& aLine); + + enum ParserState { + PROTOCOL_STATE_CONTROL, + PROTOCOL_STATE_CHUNK + }; + ParserState mState; + + enum ChunkType { + // Types for shavar tables. + CHUNK_ADD, + CHUNK_SUB, + // Types for digest256 tables. digest256 tables differ in format from + // shavar tables since they only contain complete hashes. + CHUNK_ADD_DIGEST, + CHUNK_SUB_DIGEST + }; + + struct ChunkState { + ChunkType type; + uint32_t num; + uint32_t hashSize; + uint32_t length; + void Clear() { num = 0; hashSize = 0; length = 0; } + }; + ChunkState mChunkState; + + bool mResetRequested; + + // Updates to apply to the current table being parsed. + TableUpdateV2 *mTableUpdate; +}; + +// Helpers to parse the "proto" list format. +class ProtocolParserProtobuf final : public ProtocolParser { +public: + typedef FetchThreatListUpdatesResponse_ListUpdateResponse ListUpdateResponse; + typedef google::protobuf::RepeatedPtrField<ThreatEntrySet> ThreatEntrySetList; + +public: + ProtocolParserProtobuf(); + + virtual void SetCurrentTable(const nsACString& aTable) override; + virtual nsresult AppendStream(const nsACString& aData) override; + virtual void End() override; + +private: + virtual ~ProtocolParserProtobuf(); + + virtual TableUpdate* CreateTableUpdate(const nsACString& aTableName) const override; + + // For parsing update info. + nsresult ProcessOneResponse(const ListUpdateResponse& aResponse); + + nsresult ProcessAdditionOrRemoval(TableUpdateV4& aTableUpdate, + const ThreatEntrySetList& aUpdate, + bool aIsAddition); + + nsresult ProcessRawAddition(TableUpdateV4& aTableUpdate, + const ThreatEntrySet& aAddition); + + nsresult ProcessRawRemoval(TableUpdateV4& aTableUpdate, + const ThreatEntrySet& aRemoval); + + nsresult ProcessEncodedAddition(TableUpdateV4& aTableUpdate, + const ThreatEntrySet& aAddition); + + nsresult ProcessEncodedRemoval(TableUpdateV4& aTableUpdate, + const ThreatEntrySet& aRemoval); +}; + +} // namespace safebrowsing +} // namespace mozilla + +#endif diff --git a/toolkit/components/url-classifier/RiceDeltaDecoder.cpp b/toolkit/components/url-classifier/RiceDeltaDecoder.cpp new file mode 100644 index 000000000..66b0b3d6d --- /dev/null +++ b/toolkit/components/url-classifier/RiceDeltaDecoder.cpp @@ -0,0 +1,229 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "RiceDeltaDecoder.h" + +namespace { + +//////////////////////////////////////////////////////////////////////// +// BitBuffer is copied and modified from webrtc/base/bitbuffer.h +// + +/* + * Copyright 2015 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree (webrtc/base/bitbuffer.h/cc). An additional intellectual property + * rights grant can be found in the file PATENTS. All contributing + * project authors may be found in the AUTHORS file in the root of + * the source tree. + */ + +class BitBuffer { + public: + BitBuffer(const uint8_t* bytes, size_t byte_count); + + // The remaining bits in the byte buffer. + uint64_t RemainingBitCount() const; + + // Reads bit-sized values from the buffer. Returns false if there isn't enough + // data left for the specified bit count.. + bool ReadBits(uint32_t* val, size_t bit_count); + + // Peeks bit-sized values from the buffer. Returns false if there isn't enough + // data left for the specified number of bits. Doesn't move the current + // offset. + bool PeekBits(uint32_t* val, size_t bit_count); + + // Reads the exponential golomb encoded value at the current offset. + // Exponential golomb values are encoded as: + // 1) x = source val + 1 + // 2) In binary, write [countbits(x) - 1] 1s, then x + // To decode, we count the number of leading 1 bits, read that many + 1 bits, + // and increment the result by 1. + // Returns false if there isn't enough data left for the specified type, or if + // the value wouldn't fit in a uint32_t. + bool ReadExponentialGolomb(uint32_t* val); + + // Moves current position |bit_count| bits forward. Returns false if + // there aren't enough bits left in the buffer. + bool ConsumeBits(size_t bit_count); + + protected: + const uint8_t* const bytes_; + // The total size of |bytes_|. + size_t byte_count_; + // The current offset, in bytes, from the start of |bytes_|. + size_t byte_offset_; + // The current offset, in bits, into the current byte. + size_t bit_offset_; +}; + +} // end of unnamed namespace + +static void +ReverseByte(uint8_t& b) +{ + b = (b & 0xF0) >> 4 | (b & 0x0F) << 4; + b = (b & 0xCC) >> 2 | (b & 0x33) << 2; + b = (b & 0xAA) >> 1 | (b & 0x55) << 1; +} + +namespace mozilla { +namespace safebrowsing { + +RiceDeltaDecoder::RiceDeltaDecoder(uint8_t* aEncodedData, + size_t aEncodedDataSize) + : mEncodedData(aEncodedData) + , mEncodedDataSize(aEncodedDataSize) +{ +} + +bool +RiceDeltaDecoder::Decode(uint32_t aRiceParameter, + uint32_t aFirstValue, + uint32_t aNumEntries, + uint32_t* aDecodedData) +{ + // Reverse each byte before reading bits from the byte buffer. + for (size_t i = 0; i < mEncodedDataSize; i++) { + ReverseByte(mEncodedData[i]); + } + + BitBuffer bitBuffer(mEncodedData, mEncodedDataSize); + + // q = quotient + // r = remainder + // k = RICE parameter + const uint32_t k = aRiceParameter; + aDecodedData[0] = aFirstValue; + for (uint32_t i = 0; i < aNumEntries; i++) { + // Read the quotient of N. + uint32_t q; + if (!bitBuffer.ReadExponentialGolomb(&q)) { + LOG(("Encoded data underflow!")); + return false; + } + + // Read the remainder of N, one bit at a time. + uint32_t r = 0; + for (uint32_t j = 0; j < k; j++) { + uint32_t b = 0; + if (!bitBuffer.ReadBits(&b, 1)) { + // Insufficient bits. Just leave them as zeros. + break; + } + // Add the bit to the right position so that it's in Little Endian order. + r |= b << j; + } + + // Caculate N from q,r,k. + uint32_t N = (q << k) + r; + + // We start filling aDecodedData from [1]. + aDecodedData[i + 1] = N + aDecodedData[i]; + } + + return true; +} + +} // end of namespace mozilla +} // end of namespace safebrowsing + +namespace { +////////////////////////////////////////////////////////////////////////// +// The BitBuffer impl is copied and modified from webrtc/base/bitbuffer.cc +// + +// Returns the lowest (right-most) |bit_count| bits in |byte|. +uint8_t LowestBits(uint8_t byte, size_t bit_count) { + return byte & ((1 << bit_count) - 1); +} + +// Returns the highest (left-most) |bit_count| bits in |byte|, shifted to the +// lowest bits (to the right). +uint8_t HighestBits(uint8_t byte, size_t bit_count) { + MOZ_ASSERT(bit_count < 8u); + uint8_t shift = 8 - static_cast<uint8_t>(bit_count); + uint8_t mask = 0xFF << shift; + return (byte & mask) >> shift; +} + +BitBuffer::BitBuffer(const uint8_t* bytes, size_t byte_count) + : bytes_(bytes), byte_count_(byte_count), byte_offset_(), bit_offset_() { + MOZ_ASSERT(static_cast<uint64_t>(byte_count_) <= + std::numeric_limits<uint32_t>::max()); +} + +uint64_t BitBuffer::RemainingBitCount() const { + return (static_cast<uint64_t>(byte_count_) - byte_offset_) * 8 - bit_offset_; +} + +bool BitBuffer::PeekBits(uint32_t* val, size_t bit_count) { + if (!val || bit_count > RemainingBitCount() || bit_count > 32) { + return false; + } + const uint8_t* bytes = bytes_ + byte_offset_; + size_t remaining_bits_in_current_byte = 8 - bit_offset_; + uint32_t bits = LowestBits(*bytes++, remaining_bits_in_current_byte); + // If we're reading fewer bits than what's left in the current byte, just + // return the portion of this byte that we need. + if (bit_count < remaining_bits_in_current_byte) { + *val = HighestBits(bits, bit_offset_ + bit_count); + return true; + } + // Otherwise, subtract what we've read from the bit count and read as many + // full bytes as we can into bits. + bit_count -= remaining_bits_in_current_byte; + while (bit_count >= 8) { + bits = (bits << 8) | *bytes++; + bit_count -= 8; + } + // Whatever we have left is smaller than a byte, so grab just the bits we need + // and shift them into the lowest bits. + if (bit_count > 0) { + bits <<= bit_count; + bits |= HighestBits(*bytes, bit_count); + } + *val = bits; + return true; +} + +bool BitBuffer::ReadBits(uint32_t* val, size_t bit_count) { + return PeekBits(val, bit_count) && ConsumeBits(bit_count); +} + +bool BitBuffer::ConsumeBits(size_t bit_count) { + if (bit_count > RemainingBitCount()) { + return false; + } + + byte_offset_ += (bit_offset_ + bit_count) / 8; + bit_offset_ = (bit_offset_ + bit_count) % 8; + return true; +} + +bool BitBuffer::ReadExponentialGolomb(uint32_t* val) { + if (!val) { + return false; + } + + *val = 0; + + // Count the number of leading 0 bits by peeking/consuming them one at a time. + size_t one_bit_count = 0; + uint32_t peeked_bit; + while (PeekBits(&peeked_bit, 1) && peeked_bit == 1) { + one_bit_count++; + ConsumeBits(1); + } + if (!ConsumeBits(1)) { + return false; // The stream is incorrectly terminated at '1'. + } + + *val = one_bit_count; + return true; +} +} diff --git a/toolkit/components/url-classifier/RiceDeltaDecoder.h b/toolkit/components/url-classifier/RiceDeltaDecoder.h new file mode 100644 index 000000000..cf87cea88 --- /dev/null +++ b/toolkit/components/url-classifier/RiceDeltaDecoder.h @@ -0,0 +1,39 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef RICE_DELTA_DECODER_H +#define RICE_DELTA_DECODER_H + +namespace mozilla { +namespace safebrowsing { + +class RiceDeltaDecoder { +public: + // This decoder is tailored for safebrowsing v4, including the + // bit reading order and how the remainder part is interpreted. + // The caller just needs to feed the byte stream received from + // network directly. Note that the input buffer must be mutable + // since the decoder will do some pre-processing before decoding. + RiceDeltaDecoder(uint8_t* aEncodedData, size_t aEncodedDataSize); + + // @param aNumEntries The number of values to be decoded, not including + // the first value. + // @param aDecodedData A pre-allocated output buffer. Note that + // aDecodedData[0] will be filled with |aFirstValue| + // and the buffer length (in byte) should be + // ((aNumEntries + 1) * sizeof(uint32_t)). + bool Decode(uint32_t aRiceParameter, + uint32_t aFirstValue, + uint32_t aNumEntries, + uint32_t* aDecodedData); + +private: + uint8_t* mEncodedData; + size_t mEncodedDataSize; +}; + +} // namespace safebrowsing +} // namespace mozilla + +#endif // UPDATE_V4_DECODER_H diff --git a/toolkit/components/url-classifier/SafeBrowsing.jsm b/toolkit/components/url-classifier/SafeBrowsing.jsm new file mode 100644 index 000000000..b49be71fe --- /dev/null +++ b/toolkit/components/url-classifier/SafeBrowsing.jsm @@ -0,0 +1,429 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +this.EXPORTED_SYMBOLS = ["SafeBrowsing"]; + +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cu = Components.utils; + +Cu.import("resource://gre/modules/Services.jsm"); + +// Log only if browser.safebrowsing.debug is true +function log(...stuff) { + let logging = null; + try { + logging = Services.prefs.getBoolPref("browser.safebrowsing.debug"); + } catch(e) { + return; + } + if (!logging) { + return; + } + + var d = new Date(); + let msg = "SafeBrowsing: " + d.toTimeString() + ": " + stuff.join(" "); + dump(Services.urlFormatter.trimSensitiveURLs(msg) + "\n"); +} + +function getLists(prefName) { + log("getLists: " + prefName); + let pref = null; + try { + pref = Services.prefs.getCharPref(prefName); + } catch(e) { + return null; + } + // Splitting an empty string returns [''], we really want an empty array. + if (!pref) { + return []; + } + return pref.split(",") + .map(function(value) { return value.trim(); }); +} + +const tablePreferences = [ + "urlclassifier.phishTable", + "urlclassifier.malwareTable", + "urlclassifier.downloadBlockTable", + "urlclassifier.downloadAllowTable", + "urlclassifier.trackingTable", + "urlclassifier.trackingWhitelistTable", + "urlclassifier.blockedTable" +]; + +this.SafeBrowsing = { + + init: function() { + if (this.initialized) { + log("Already initialized"); + return; + } + + Services.prefs.addObserver("browser.safebrowsing", this, false); + Services.prefs.addObserver("privacy.trackingprotection", this, false); + Services.prefs.addObserver("urlclassifier", this, false); + + this.readPrefs(); + this.addMozEntries(); + + this.controlUpdateChecking(); + this.initialized = true; + + log("init() finished"); + }, + + registerTableWithURLs: function(listname) { + let listManager = Cc["@mozilla.org/url-classifier/listmanager;1"]. + getService(Ci.nsIUrlListManager); + + let providerName = this.listToProvider[listname]; + let provider = this.providers[providerName]; + + if (!providerName || !provider) { + log("No provider info found for " + listname); + log("Check browser.safebrowsing.provider.[google/mozilla].lists"); + return; + } + + listManager.registerTable(listname, providerName, provider.updateURL, provider.gethashURL); + }, + + registerTables: function() { + for (let i = 0; i < this.phishingLists.length; ++i) { + this.registerTableWithURLs(this.phishingLists[i]); + } + for (let i = 0; i < this.malwareLists.length; ++i) { + this.registerTableWithURLs(this.malwareLists[i]); + } + for (let i = 0; i < this.downloadBlockLists.length; ++i) { + this.registerTableWithURLs(this.downloadBlockLists[i]); + } + for (let i = 0; i < this.downloadAllowLists.length; ++i) { + this.registerTableWithURLs(this.downloadAllowLists[i]); + } + for (let i = 0; i < this.trackingProtectionLists.length; ++i) { + this.registerTableWithURLs(this.trackingProtectionLists[i]); + } + for (let i = 0; i < this.trackingProtectionWhitelists.length; ++i) { + this.registerTableWithURLs(this.trackingProtectionWhitelists[i]); + } + for (let i = 0; i < this.blockedLists.length; ++i) { + this.registerTableWithURLs(this.blockedLists[i]); + } + }, + + + initialized: false, + phishingEnabled: false, + malwareEnabled: false, + trackingEnabled: false, + blockedEnabled: false, + + phishingLists: [], + malwareLists: [], + downloadBlockLists: [], + downloadAllowLists: [], + trackingProtectionLists: [], + trackingProtectionWhitelists: [], + blockedLists: [], + + updateURL: null, + gethashURL: null, + + reportURL: null, + + getReportURL: function(kind, URI) { + let pref; + switch (kind) { + case "Phish": + pref = "browser.safebrowsing.reportPhishURL"; + break; + case "PhishMistake": + pref = "browser.safebrowsing.reportPhishMistakeURL"; + break; + case "MalwareMistake": + pref = "browser.safebrowsing.reportMalwareMistakeURL"; + break; + + default: + let err = "SafeBrowsing getReportURL() called with unknown kind: " + kind; + Components.utils.reportError(err); + throw err; + } + let reportUrl = Services.urlFormatter.formatURLPref(pref); + + let pageUri = URI.clone(); + + // Remove the query to avoid including potentially sensitive data + if (pageUri instanceof Ci.nsIURL) + pageUri.query = ''; + + reportUrl += encodeURIComponent(pageUri.asciiSpec); + + return reportUrl; + }, + + observe: function(aSubject, aTopic, aData) { + // skip nextupdatetime and lastupdatetime + if (aData.indexOf("lastupdatetime") >= 0 || aData.indexOf("nextupdatetime") >= 0) { + return; + } + this.readPrefs(); + }, + + readPrefs: function() { + log("reading prefs"); + + this.debug = Services.prefs.getBoolPref("browser.safebrowsing.debug"); + this.phishingEnabled = Services.prefs.getBoolPref("browser.safebrowsing.phishing.enabled"); + this.malwareEnabled = Services.prefs.getBoolPref("browser.safebrowsing.malware.enabled"); + this.trackingEnabled = Services.prefs.getBoolPref("privacy.trackingprotection.enabled") || Services.prefs.getBoolPref("privacy.trackingprotection.pbmode.enabled"); + this.blockedEnabled = Services.prefs.getBoolPref("browser.safebrowsing.blockedURIs.enabled"); + + [this.phishingLists, + this.malwareLists, + this.downloadBlockLists, + this.downloadAllowLists, + this.trackingProtectionLists, + this.trackingProtectionWhitelists, + this.blockedLists] = tablePreferences.map(getLists); + + this.updateProviderURLs(); + this.registerTables(); + + // XXX The listManager backend gets confused if this is called before the + // lists are registered. So only call it here when a pref changes, and not + // when doing initialization. I expect to refactor this later, so pardon the hack. + if (this.initialized) { + this.controlUpdateChecking(); + } + }, + + + updateProviderURLs: function() { + try { + var clientID = Services.prefs.getCharPref("browser.safebrowsing.id"); + } catch(e) { + clientID = Services.appinfo.name; + } + + log("initializing safe browsing URLs, client id", clientID); + + // Get the different providers + let branch = Services.prefs.getBranch("browser.safebrowsing.provider."); + let children = branch.getChildList("", {}); + this.providers = {}; + this.listToProvider = {}; + + for (let child of children) { + log("Child: " + child); + let prefComponents = child.split("."); + let providerName = prefComponents[0]; + this.providers[providerName] = {}; + } + + if (this.debug) { + let providerStr = ""; + Object.keys(this.providers).forEach(function(provider) { + if (providerStr === "") { + providerStr = provider; + } else { + providerStr += ", " + provider; + } + }); + log("Providers: " + providerStr); + } + + Object.keys(this.providers).forEach(function(provider) { + let updateURL = Services.urlFormatter.formatURLPref( + "browser.safebrowsing.provider." + provider + ".updateURL"); + let gethashURL = Services.urlFormatter.formatURLPref( + "browser.safebrowsing.provider." + provider + ".gethashURL"); + updateURL = updateURL.replace("SAFEBROWSING_ID", clientID); + gethashURL = gethashURL.replace("SAFEBROWSING_ID", clientID); + + log("Provider: " + provider + " updateURL=" + updateURL); + log("Provider: " + provider + " gethashURL=" + gethashURL); + + // Urls used to update DB + this.providers[provider].updateURL = updateURL; + this.providers[provider].gethashURL = gethashURL; + + // Get lists this provider manages + let lists = getLists("browser.safebrowsing.provider." + provider + ".lists"); + if (lists) { + lists.forEach(function(list) { + this.listToProvider[list] = provider; + }, this); + } else { + log("Update URL given but no lists managed for provider: " + provider); + } + }, this); + }, + + controlUpdateChecking: function() { + log("phishingEnabled:", this.phishingEnabled, "malwareEnabled:", + this.malwareEnabled, "trackingEnabled:", this.trackingEnabled, + "blockedEnabled:", this.blockedEnabled); + + let listManager = Cc["@mozilla.org/url-classifier/listmanager;1"]. + getService(Ci.nsIUrlListManager); + + for (let i = 0; i < this.phishingLists.length; ++i) { + if (this.phishingEnabled) { + listManager.enableUpdate(this.phishingLists[i]); + } else { + listManager.disableUpdate(this.phishingLists[i]); + } + } + for (let i = 0; i < this.malwareLists.length; ++i) { + if (this.malwareEnabled) { + listManager.enableUpdate(this.malwareLists[i]); + } else { + listManager.disableUpdate(this.malwareLists[i]); + } + } + for (let i = 0; i < this.downloadBlockLists.length; ++i) { + if (this.malwareEnabled) { + listManager.enableUpdate(this.downloadBlockLists[i]); + } else { + listManager.disableUpdate(this.downloadBlockLists[i]); + } + } + for (let i = 0; i < this.downloadAllowLists.length; ++i) { + if (this.malwareEnabled) { + listManager.enableUpdate(this.downloadAllowLists[i]); + } else { + listManager.disableUpdate(this.downloadAllowLists[i]); + } + } + for (let i = 0; i < this.trackingProtectionLists.length; ++i) { + if (this.trackingEnabled) { + listManager.enableUpdate(this.trackingProtectionLists[i]); + } else { + listManager.disableUpdate(this.trackingProtectionLists[i]); + } + } + for (let i = 0; i < this.trackingProtectionWhitelists.length; ++i) { + if (this.trackingEnabled) { + listManager.enableUpdate(this.trackingProtectionWhitelists[i]); + } else { + listManager.disableUpdate(this.trackingProtectionWhitelists[i]); + } + } + for (let i = 0; i < this.blockedLists.length; ++i) { + if (this.blockedEnabled) { + listManager.enableUpdate(this.blockedLists[i]); + } else { + listManager.disableUpdate(this.blockedLists[i]); + } + } + listManager.maybeToggleUpdateChecking(); + }, + + + addMozEntries: function() { + // Add test entries to the DB. + // XXX bug 779008 - this could be done by DB itself? + const phishURL = "itisatrap.org/firefox/its-a-trap.html"; + const malwareURL = "itisatrap.org/firefox/its-an-attack.html"; + const unwantedURL = "itisatrap.org/firefox/unwanted.html"; + const trackerURLs = [ + "trackertest.org/", + "itisatracker.org/", + ]; + const whitelistURL = "itisatrap.org/?resource=itisatracker.org"; + const blockedURL = "itisatrap.org/firefox/blocked.html"; + + const flashDenyURL = "flashblock.itisatrap.org/"; + const flashDenyExceptURL = "except.flashblock.itisatrap.org/"; + const flashAllowURL = "flashallow.itisatrap.org/"; + const flashAllowExceptURL = "except.flashallow.itisatrap.org/"; + const flashSubDocURL = "flashsubdoc.itisatrap.org/"; + const flashSubDocExceptURL = "except.flashsubdoc.itisatrap.org/"; + + let update = "n:1000\ni:test-malware-simple\nad:1\n" + + "a:1:32:" + malwareURL.length + "\n" + + malwareURL + "\n"; + update += "n:1000\ni:test-phish-simple\nad:1\n" + + "a:1:32:" + phishURL.length + "\n" + + phishURL + "\n"; + update += "n:1000\ni:test-unwanted-simple\nad:1\n" + + "a:1:32:" + unwantedURL.length + "\n" + + unwantedURL + "\n"; + update += "n:1000\ni:test-track-simple\n" + + "ad:" + trackerURLs.length + "\n"; + trackerURLs.forEach((trackerURL, i) => { + update += "a:" + (i + 1) + ":32:" + trackerURL.length + "\n" + + trackerURL + "\n"; + }); + update += "n:1000\ni:test-trackwhite-simple\nad:1\n" + + "a:1:32:" + whitelistURL.length + "\n" + + whitelistURL; + update += "n:1000\ni:test-block-simple\nad:1\n" + + "a:1:32:" + blockedURL.length + "\n" + + blockedURL; + update += "n:1000\ni:test-flash-simple\nad:1\n" + + "a:1:32:" + flashDenyURL.length + "\n" + + flashDenyURL; + update += "n:1000\ni:testexcept-flash-simple\nad:1\n" + + "a:1:32:" + flashDenyExceptURL.length + "\n" + + flashDenyExceptURL; + update += "n:1000\ni:test-flashallow-simple\nad:1\n" + + "a:1:32:" + flashAllowURL.length + "\n" + + flashAllowURL; + update += "n:1000\ni:testexcept-flashallow-simple\nad:1\n" + + "a:1:32:" + flashAllowExceptURL.length + "\n" + + flashAllowExceptURL; + update += "n:1000\ni:test-flashsubdoc-simple\nad:1\n" + + "a:1:32:" + flashSubDocURL.length + "\n" + + flashSubDocURL; + update += "n:1000\ni:testexcept-flashsubdoc-simple\nad:1\n" + + "a:1:32:" + flashSubDocExceptURL.length + "\n" + + flashSubDocExceptURL; + log("addMozEntries:", update); + + let db = Cc["@mozilla.org/url-classifier/dbservice;1"]. + getService(Ci.nsIUrlClassifierDBService); + + // nsIUrlClassifierUpdateObserver + let dummyListener = { + updateUrlRequested: function() { }, + streamFinished: function() { }, + // We notify observers when we're done in order to be able to make perf + // test results more consistent + updateError: function() { + Services.obs.notifyObservers(db, "mozentries-update-finished", "error"); + }, + updateSuccess: function() { + Services.obs.notifyObservers(db, "mozentries-update-finished", "success"); + } + }; + + try { + let tables = "test-malware-simple,test-phish-simple,test-unwanted-simple,test-track-simple,test-trackwhite-simple,test-block-simple,test-flash-simple,testexcept-flash-simple,test-flashallow-simple,testexcept-flashallow-simple,test-flashsubdoc-simple,testexcept-flashsubdoc-simple"; + db.beginUpdate(dummyListener, tables, ""); + db.beginStream("", ""); + db.updateStream(update); + db.finishStream(); + db.finishUpdate(); + } catch(ex) { + // beginUpdate will throw harmlessly if there's an existing update in progress, ignore failures. + log("addMozEntries failed!", ex); + Services.obs.notifyObservers(db, "mozentries-update-finished", "exception"); + } + }, + + addMozEntriesFinishedPromise: new Promise(resolve => { + let finished = (subject, topic, data) => { + Services.obs.removeObserver(finished, "mozentries-update-finished"); + if (data == "error") { + Cu.reportError("addMozEntries failed to update the db!"); + } + resolve(); + }; + Services.obs.addObserver(finished, "mozentries-update-finished", false); + }), +}; diff --git a/toolkit/components/url-classifier/VariableLengthPrefixSet.cpp b/toolkit/components/url-classifier/VariableLengthPrefixSet.cpp new file mode 100644 index 000000000..e9d6770d3 --- /dev/null +++ b/toolkit/components/url-classifier/VariableLengthPrefixSet.cpp @@ -0,0 +1,443 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "VariableLengthPrefixSet.h" +#include "nsUrlClassifierPrefixSet.h" +#include "nsPrintfCString.h" +#include "nsThreadUtils.h" +#include "mozilla/EndianUtils.h" +#include "mozilla/Telemetry.h" +#include "mozilla/Unused.h" +#include <algorithm> + +// MOZ_LOG=UrlClassifierPrefixSet:5 +static mozilla::LazyLogModule gUrlClassifierPrefixSetLog("UrlClassifierPrefixSet"); +#define LOG(args) MOZ_LOG(gUrlClassifierPrefixSetLog, mozilla::LogLevel::Debug, args) +#define LOG_ENABLED() MOZ_LOG_TEST(gUrlClassifierPrefixSetLog, mozilla::LogLevel::Debug) + +namespace mozilla { +namespace safebrowsing { + +#define PREFIX_SIZE_FIXED 4 + +NS_IMPL_ISUPPORTS(VariableLengthPrefixSet, nsIMemoryReporter) + +// Definition required due to std::max<>() +const uint32_t VariableLengthPrefixSet::MAX_BUFFER_SIZE; + +// This class will process prefix size between 4~32. But for 4 bytes prefixes, +// they will be passed to nsUrlClassifierPrefixSet because of better optimization. +VariableLengthPrefixSet::VariableLengthPrefixSet() + : mLock("VariableLengthPrefixSet.mLock") + , mMemoryReportPath() +{ + mFixedPrefixSet = new nsUrlClassifierPrefixSet(); +} + +NS_IMETHODIMP +VariableLengthPrefixSet::Init(const nsACString& aName) +{ + mMemoryReportPath = + nsPrintfCString( + "explicit/storage/prefix-set/%s", + (!aName.IsEmpty() ? PromiseFlatCString(aName).get() : "?!") + ); + + RegisterWeakMemoryReporter(this); + + return NS_OK; +} + +VariableLengthPrefixSet::~VariableLengthPrefixSet() +{ + UnregisterWeakMemoryReporter(this); +} + +NS_IMETHODIMP +VariableLengthPrefixSet::SetPrefixes(const PrefixStringMap& aPrefixMap) +{ + MutexAutoLock lock(mLock); + + // Prefix size should not less than 4-bytes or greater than 32-bytes + for (auto iter = aPrefixMap.ConstIter(); !iter.Done(); iter.Next()) { + if (iter.Key() < PREFIX_SIZE_FIXED || + iter.Key() > COMPLETE_SIZE) { + return NS_ERROR_FAILURE; + } + } + + // Clear old prefixSet before setting new one. + mFixedPrefixSet->SetPrefixes(nullptr, 0); + mVLPrefixSet.Clear(); + + // 4-bytes prefixes are handled by nsUrlClassifierPrefixSet. + nsCString* prefixes = aPrefixMap.Get(PREFIX_SIZE_FIXED); + if (prefixes) { + NS_ENSURE_TRUE(prefixes->Length() % PREFIX_SIZE_FIXED == 0, NS_ERROR_FAILURE); + + uint32_t numPrefixes = prefixes->Length() / PREFIX_SIZE_FIXED; + +#if MOZ_BIG_ENDIAN + const uint32_t* arrayPtr = reinterpret_cast<const uint32_t*>(prefixes->BeginReading()); +#else + FallibleTArray<uint32_t> array; + // Prefixes are lexicographically-sorted, so the interger array + // passed to nsUrlClassifierPrefixSet should also follow the same order. + // To make sure of that, we convert char array to integer with Big-Endian + // instead of casting to integer directly. + if (!array.SetCapacity(numPrefixes, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + const char* begin = prefixes->BeginReading(); + const char* end = prefixes->EndReading(); + + while (begin != end) { + array.AppendElement(BigEndian::readUint32(begin), fallible); + begin += sizeof(uint32_t); + } + + const uint32_t* arrayPtr = array.Elements(); +#endif + + nsresult rv = mFixedPrefixSet->SetPrefixes(arrayPtr, numPrefixes); + NS_ENSURE_SUCCESS(rv, rv); + } + + // 5~32 bytes prefixes are stored in mVLPrefixSet. + for (auto iter = aPrefixMap.ConstIter(); !iter.Done(); iter.Next()) { + // Skip 4bytes prefixes because it is already stored in mFixedPrefixSet. + if (iter.Key() == PREFIX_SIZE_FIXED) { + continue; + } + + mVLPrefixSet.Put(iter.Key(), new nsCString(*iter.Data())); + } + + return NS_OK; +} + +nsresult +VariableLengthPrefixSet::GetPrefixes(PrefixStringMap& aPrefixMap) +{ + MutexAutoLock lock(mLock); + + // 4-bytes prefixes are handled by nsUrlClassifierPrefixSet. + FallibleTArray<uint32_t> array; + nsresult rv = mFixedPrefixSet->GetPrefixesNative(array); + NS_ENSURE_SUCCESS(rv, rv); + + size_t count = array.Length(); + if (count) { + nsCString* prefixes = new nsCString(); + prefixes->SetLength(PREFIX_SIZE_FIXED * count); + + // Writing integer array to character array + uint32_t* begin = reinterpret_cast<uint32_t*>(prefixes->BeginWriting()); + for (uint32_t i = 0; i < count; i++) { + begin[i] = NativeEndian::swapToBigEndian(array[i]); + } + + aPrefixMap.Put(PREFIX_SIZE_FIXED, prefixes); + } + + // Copy variable-length prefix set + for (auto iter = mVLPrefixSet.ConstIter(); !iter.Done(); iter.Next()) { + aPrefixMap.Put(iter.Key(), new nsCString(*iter.Data())); + } + + return NS_OK; +} + +// It should never be the case that more than one hash prefixes match a given +// full hash. However, if that happens, this method returns any one of them. +// It does not guarantee which one of those will be returned. +NS_IMETHODIMP +VariableLengthPrefixSet::Matches(const nsACString& aFullHash, uint32_t* aLength) +{ + MutexAutoLock lock(mLock); + + // Only allow full-length hash to check if match any of the prefix + MOZ_ASSERT(aFullHash.Length() == COMPLETE_SIZE); + NS_ENSURE_ARG_POINTER(aLength); + + *aLength = 0; + + // Check if it matches 4-bytes prefixSet first + const uint32_t* hash = reinterpret_cast<const uint32_t*>(aFullHash.BeginReading()); + uint32_t value = BigEndian::readUint32(hash); + + bool found = false; + nsresult rv = mFixedPrefixSet->Contains(value, &found); + NS_ENSURE_SUCCESS(rv, rv); + + if (found) { + *aLength = PREFIX_SIZE_FIXED; + return NS_OK; + } + + for (auto iter = mVLPrefixSet.ConstIter(); !iter.Done(); iter.Next()) { + if (BinarySearch(aFullHash, *iter.Data(), iter.Key())) { + *aLength = iter.Key(); + return NS_OK; + } + } + + return NS_OK; +} + +NS_IMETHODIMP +VariableLengthPrefixSet::IsEmpty(bool* aEmpty) +{ + MutexAutoLock lock(mLock); + + NS_ENSURE_ARG_POINTER(aEmpty); + + mFixedPrefixSet->IsEmpty(aEmpty); + *aEmpty = *aEmpty && mVLPrefixSet.IsEmpty(); + + return NS_OK; +} + +NS_IMETHODIMP +VariableLengthPrefixSet::LoadFromFile(nsIFile* aFile) +{ + MutexAutoLock lock(mLock); + + NS_ENSURE_ARG_POINTER(aFile); + + Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_VLPS_FILELOAD_TIME> timer; + + nsCOMPtr<nsIInputStream> localInFile; + nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(localInFile), aFile, + PR_RDONLY | nsIFile::OS_READAHEAD); + NS_ENSURE_SUCCESS(rv, rv); + + // Calculate how big the file is, make sure our read buffer isn't bigger + // than the file itself which is just wasting memory. + int64_t fileSize; + rv = aFile->GetFileSize(&fileSize); + NS_ENSURE_SUCCESS(rv, rv); + + if (fileSize < 0 || fileSize > UINT32_MAX) { + return NS_ERROR_FAILURE; + } + + uint32_t bufferSize = std::min<uint32_t>(static_cast<uint32_t>(fileSize), + MAX_BUFFER_SIZE); + + // Convert to buffered stream + nsCOMPtr<nsIInputStream> in = NS_BufferInputStream(localInFile, bufferSize); + + rv = mFixedPrefixSet->LoadPrefixes(in); + NS_ENSURE_SUCCESS(rv, rv); + + rv = LoadPrefixes(in); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK;; +} + +NS_IMETHODIMP +VariableLengthPrefixSet::StoreToFile(nsIFile* aFile) +{ + NS_ENSURE_ARG_POINTER(aFile); + + MutexAutoLock lock(mLock); + + nsCOMPtr<nsIOutputStream> localOutFile; + nsresult rv = NS_NewLocalFileOutputStream(getter_AddRefs(localOutFile), aFile, + PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE); + NS_ENSURE_SUCCESS(rv, rv); + + uint32_t fileSize = 0; + // Preallocate the file storage + { + nsCOMPtr<nsIFileOutputStream> fos(do_QueryInterface(localOutFile)); + Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_VLPS_FALLOCATE_TIME> timer; + + fileSize += mFixedPrefixSet->CalculatePreallocateSize(); + fileSize += CalculatePreallocateSize(); + + Unused << fos->Preallocate(fileSize); + } + + // Convert to buffered stream + nsCOMPtr<nsIOutputStream> out = + NS_BufferOutputStream(localOutFile, std::min(fileSize, MAX_BUFFER_SIZE)); + + rv = mFixedPrefixSet->WritePrefixes(out); + NS_ENSURE_SUCCESS(rv, rv); + + rv = WritePrefixes(out); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +VariableLengthPrefixSet::LoadPrefixes(nsIInputStream* in) +{ + uint32_t magic; + uint32_t read; + + nsresult rv = in->Read(reinterpret_cast<char*>(&magic), sizeof(uint32_t), &read); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(read == sizeof(uint32_t), NS_ERROR_FAILURE); + + if (magic != PREFIXSET_VERSION_MAGIC) { + LOG(("Version magic mismatch, not loading")); + return NS_ERROR_FILE_CORRUPTED; + } + + mVLPrefixSet.Clear(); + + uint32_t count; + rv = in->Read(reinterpret_cast<char*>(&count), sizeof(uint32_t), &read); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(read == sizeof(uint32_t), NS_ERROR_FAILURE); + + for(;count > 0; count--) { + uint8_t prefixSize; + rv = in->Read(reinterpret_cast<char*>(&prefixSize), sizeof(uint8_t), &read); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(read == sizeof(uint8_t), NS_ERROR_FAILURE); + + uint32_t stringLength; + rv = in->Read(reinterpret_cast<char*>(&stringLength), sizeof(uint32_t), &read); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(read == sizeof(uint32_t), NS_ERROR_FAILURE); + + nsCString* vlPrefixes = new nsCString(); + if (!vlPrefixes->SetLength(stringLength, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + rv = in->Read(reinterpret_cast<char*>(vlPrefixes->BeginWriting()), stringLength, &read); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(read == stringLength, NS_ERROR_FAILURE); + + mVLPrefixSet.Put(prefixSize, vlPrefixes); + } + + return NS_OK; +} + +uint32_t +VariableLengthPrefixSet::CalculatePreallocateSize() +{ + uint32_t fileSize = 0; + + // Store how many prefix string. + fileSize += sizeof(uint32_t); + + for (auto iter = mVLPrefixSet.ConstIter(); !iter.Done(); iter.Next()) { + // Store prefix size, prefix string length, and prefix string. + fileSize += sizeof(uint8_t); + fileSize += sizeof(uint32_t); + fileSize += iter.Data()->Length(); + } + return fileSize; +} + +nsresult +VariableLengthPrefixSet::WritePrefixes(nsIOutputStream* out) +{ + uint32_t written; + uint32_t writelen = sizeof(uint32_t); + uint32_t magic = PREFIXSET_VERSION_MAGIC; + nsresult rv = out->Write(reinterpret_cast<char*>(&magic), writelen, &written); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE); + + uint32_t count = mVLPrefixSet.Count(); + rv = out->Write(reinterpret_cast<char*>(&count), writelen, &written); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE); + + // Store PrefixSize, Length of Prefix String and then Prefix String + for (auto iter = mVLPrefixSet.ConstIter(); !iter.Done(); iter.Next()) { + const nsCString& vlPrefixes = *iter.Data(); + + uint8_t prefixSize = iter.Key(); + writelen = sizeof(uint8_t); + rv = out->Write(reinterpret_cast<char*>(&prefixSize), writelen, &written); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE); + + uint32_t stringLength = vlPrefixes.Length(); + writelen = sizeof(uint32_t); + rv = out->Write(reinterpret_cast<char*>(&stringLength), writelen, &written); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE); + + rv = out->Write(const_cast<char*>(vlPrefixes.BeginReading()), + stringLength, &written); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(stringLength == written, NS_ERROR_FAILURE); + } + + return NS_OK; +} + +bool +VariableLengthPrefixSet::BinarySearch(const nsACString& aFullHash, + const nsACString& aPrefixes, + uint32_t aPrefixSize) +{ + const char* fullhash = aFullHash.BeginReading(); + const char* prefixes = aPrefixes.BeginReading(); + int32_t begin = 0, end = aPrefixes.Length() / aPrefixSize; + + while (end > begin) { + int32_t mid = (begin + end) >> 1; + int cmp = memcmp(fullhash, prefixes + mid*aPrefixSize, aPrefixSize); + if (cmp < 0) { + end = mid; + } else if (cmp > 0) { + begin = mid + 1; + } else { + return true; + } + } + return false; +} + +MOZ_DEFINE_MALLOC_SIZE_OF(UrlClassifierMallocSizeOf) + +NS_IMETHODIMP +VariableLengthPrefixSet::CollectReports(nsIHandleReportCallback* aHandleReport, + nsISupports* aData, bool aAnonymize) +{ + MOZ_ASSERT(NS_IsMainThread()); + + size_t amount = SizeOfIncludingThis(UrlClassifierMallocSizeOf); + + return aHandleReport->Callback( + EmptyCString(), mMemoryReportPath, KIND_HEAP, UNITS_BYTES, amount, + NS_LITERAL_CSTRING("Memory used by the variable-length prefix set for a URL classifier."), + aData); +} + +size_t +VariableLengthPrefixSet::SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf) +{ + MutexAutoLock lock(mLock); + + size_t n = 0; + n += aMallocSizeOf(this); + n += mFixedPrefixSet->SizeOfIncludingThis(moz_malloc_size_of) - aMallocSizeOf(mFixedPrefixSet); + + n += mVLPrefixSet.ShallowSizeOfExcludingThis(aMallocSizeOf); + for (auto iter = mVLPrefixSet.ConstIter(); !iter.Done(); iter.Next()) { + n += iter.Data()->SizeOfExcludingThisIfUnshared(aMallocSizeOf); + } + + return n; +} + +} // namespace safebrowsing +} // namespace mozilla diff --git a/toolkit/components/url-classifier/VariableLengthPrefixSet.h b/toolkit/components/url-classifier/VariableLengthPrefixSet.h new file mode 100644 index 000000000..eca214885 --- /dev/null +++ b/toolkit/components/url-classifier/VariableLengthPrefixSet.h @@ -0,0 +1,70 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef VariableLengthPrefixSet_h +#define VariableLengthPrefixSet_h + +#include "nsISupports.h" +#include "nsIMemoryReporter.h" +#include "Entries.h" +#include "nsTArray.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/Mutex.h" + +class nsUrlClassifierPrefixSet; + +namespace mozilla { +namespace safebrowsing { + +class VariableLengthPrefixSet final + : public nsIMemoryReporter +{ +public: + VariableLengthPrefixSet(); + + NS_IMETHOD Init(const nsACString& aName); + NS_IMETHOD SetPrefixes(const mozilla::safebrowsing::PrefixStringMap& aPrefixMap); + NS_IMETHOD GetPrefixes(mozilla::safebrowsing::PrefixStringMap& aPrefixMap); + NS_IMETHOD Matches(const nsACString& aFullHash, uint32_t* aLength); + NS_IMETHOD IsEmpty(bool* aEmpty); + NS_IMETHOD LoadFromFile(nsIFile* aFile); + NS_IMETHOD StoreToFile(nsIFile* aFile); + + size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf); + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIMEMORYREPORTER + +private: + virtual ~VariableLengthPrefixSet(); + + static const uint32_t MAX_BUFFER_SIZE = 64 * 1024; + static const uint32_t PREFIXSET_VERSION_MAGIC = 1; + + bool BinarySearch(const nsACString& aFullHash, + const nsACString& aPrefixes, + uint32_t aPrefixSize); + + uint32_t CalculatePreallocateSize(); + nsresult WritePrefixes(nsIOutputStream* out); + nsresult LoadPrefixes(nsIInputStream* in); + + // Lock to prevent races between the url-classifier thread (which does most + // of the operations) and the main thread (which does memory reporting). + // It should be held for all operations between Init() and destruction that + // touch this class's data members. + mozilla::Mutex mLock; + + RefPtr<nsUrlClassifierPrefixSet> mFixedPrefixSet; + mozilla::safebrowsing::PrefixStringMap mVLPrefixSet; + + nsCString mMemoryReportPath; +}; + +} // namespace safebrowsing +} // namespace mozilla + +#endif diff --git a/toolkit/components/url-classifier/chromium/README.txt b/toolkit/components/url-classifier/chromium/README.txt new file mode 100644 index 000000000..e5a0f4132 --- /dev/null +++ b/toolkit/components/url-classifier/chromium/README.txt @@ -0,0 +1,23 @@ +# Overview + +'safebrowsing.proto' is modified from [1] with the following line added: + +"package mozilla.safebrowsing;" + +to avoid naming pollution. We use this source file along with protobuf compiler (protoc) to generate safebrowsing.pb.h/cc for safebrowsing v4 update and hash completion. The current generated files are compiled by protoc 2.6.1 since the protobuf library in gecko is not upgraded to 3.0 yet. + +# Update + +If you want to update to the latest upstream version, + +1. Checkout the latest one in [2] +2. Use protoc to generate safebrowsing.pb.h and safebrowsing.pb.cc. For example, + +$ protoc -I=. --cpp_out="../protobuf/" safebrowsing.proto + +(Note that we should use protoc v2.6.1 [3] to compile. You can find the compiled protoc in [4] if you don't have one.) + +[1] https://chromium.googlesource.com/chromium/src.git/+/9c4485f1ce7cac7ae82f7a4ae36ccc663afe806c/components/safe_browsing_db/safebrowsing.proto +[2] https://chromium.googlesource.com/chromium/src.git/+/master/components/safe_browsing_db/safebrowsing.proto +[3] https://github.com/google/protobuf/releases/tag/v2.6.1 +[4] https://repo1.maven.org/maven2/com/google/protobuf/protoc diff --git a/toolkit/components/url-classifier/chromium/safebrowsing.proto b/toolkit/components/url-classifier/chromium/safebrowsing.proto new file mode 100644 index 000000000..d62155244 --- /dev/null +++ b/toolkit/components/url-classifier/chromium/safebrowsing.proto @@ -0,0 +1,473 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// This file includes Safe Browsing V4 API blacklist request and response +// protocol buffers. They should be kept in sync with the server implementation. + +syntax = "proto2"; + +option optimize_for = LITE_RUNTIME; + +package mozilla.safebrowsing; + +message ThreatInfo { + // The threat types to be checked. + repeated ThreatType threat_types = 1; + + // The platform types to be checked. + repeated PlatformType platform_types = 2; + + // The entry types to be checked. + repeated ThreatEntryType threat_entry_types = 4; + + // The threat entries to be checked. + repeated ThreatEntry threat_entries = 3; +} + +// A match when checking a threat entry in the Safe Browsing threat lists. +message ThreatMatch { + // The threat type matching this threat. + optional ThreatType threat_type = 1; + + // The platform type matching this threat. + optional PlatformType platform_type = 2; + + // The threat entry type matching this threat. + optional ThreatEntryType threat_entry_type = 6; + + // The threat matching this threat. + optional ThreatEntry threat = 3; + + // Optional metadata associated with this threat. + optional ThreatEntryMetadata threat_entry_metadata = 4; + + // The cache lifetime for the returned match. Clients must not cache this + // response for more than this duration to avoid false positives. + optional Duration cache_duration = 5; +} + +// Request to check entries against lists. +message FindThreatMatchesRequest { + // The client metadata. + optional ClientInfo client = 1; + + // The lists and entries to be checked for matches. + optional ThreatInfo threat_info = 2; +} + +// Response type for requests to find threat matches. +message FindThreatMatchesResponse { + // The threat list matches. + repeated ThreatMatch matches = 1; +} + +// Describes a Safe Browsing API update request. Clients can request updates for +// multiple lists in a single request. +message FetchThreatListUpdatesRequest { + // The client metadata. + optional ClientInfo client = 1; + + // A single list update request. + message ListUpdateRequest { + // The type of threat posed by entries present in the list. + optional ThreatType threat_type = 1; + + // The type of platform at risk by entries present in the list. + optional PlatformType platform_type = 2; + + // The types of entries present in the list. + optional ThreatEntryType threat_entry_type = 5; + + // The current state of the client for the requested list (the encrypted + // ClientState that was sent to the client from the previous update + // request). + optional bytes state = 3; + + // The constraints for this update. + message Constraints { + // The maximum size in number of entries. The update will not contain more + // entries than this value. This should be a power of 2 between 2**10 and + // 2**20. If zero, no update size limit is set. + optional int32 max_update_entries = 1; + + // Sets the maxmimum number of entries that the client is willing to have + // in the local database. This should be a power of 2 between 2**10 and + // 2**20. If zero, no database size limit is set. + optional int32 max_database_entries = 2; + + // Requests the list for a specific geographic location. If not set the + // server may pick that value based on the user's IP address. Expects ISO + // 3166-1 alpha-2 format. + optional string region = 3; + + // The compression types supported by the client. + repeated CompressionType supported_compressions = 4; + } + + // The constraints associated with this request. + optional Constraints constraints = 4; + } + + // The requested threat list updates. + repeated ListUpdateRequest list_update_requests = 3; +} + +// Response type for threat list update requests. +message FetchThreatListUpdatesResponse { + // An update to an individual list. + message ListUpdateResponse { + // The threat type for which data is returned. + optional ThreatType threat_type = 1; + + // The format of the threats. + optional ThreatEntryType threat_entry_type = 2; + + // The platform type for which data is returned. + optional PlatformType platform_type = 3; + + // The type of response sent to the client. + enum ResponseType { + // Unknown. + RESPONSE_TYPE_UNSPECIFIED = 0; + + // Partial updates are applied to the client's existing local database. + PARTIAL_UPDATE = 1; + + // Full updates replace the client's entire local database. This means + // that either the client was seriously out-of-date or the client is + // believed to be corrupt. + FULL_UPDATE = 2; + } + + // The type of response. This may indicate that an action is required by the + // client when the response is received. + optional ResponseType response_type = 4; + + // A set of entries to add to a local threat type's list. Repeated to allow + // for a combination of compressed and raw data to be sent in a single + // response. + repeated ThreatEntrySet additions = 5; + + // A set of entries to remove from a local threat type's list. Repeated for + // the same reason as above. + repeated ThreatEntrySet removals = 6; + + // The new client state, in encrypted format. Opaque to clients. + optional bytes new_client_state = 7; + + // The expected SHA256 hash of the client state; that is, of the sorted list + // of all hashes present in the database after applying the provided update. + // If the client state doesn't match the expected state, the client must + // disregard this update and retry later. + optional Checksum checksum = 8; + } + + // The list updates requested by the clients. + repeated ListUpdateResponse list_update_responses = 1; + + // The minimum duration the client must wait before issuing any update + // request. If this field is not set clients may update as soon as they want. + optional Duration minimum_wait_duration = 2; +} + +// Request to return full hashes matched by the provided hash prefixes. +message FindFullHashesRequest { + // The client metadata. + optional ClientInfo client = 1; + + // The current client states for each of the client's local threat lists. + repeated bytes client_states = 2; + + // The lists and hashes to be checked. + optional ThreatInfo threat_info = 3; +} + +// Response type for requests to find full hashes. +message FindFullHashesResponse { + // The full hashes that matched the requested prefixes. + repeated ThreatMatch matches = 1; + + // The minimum duration the client must wait before issuing any find hashes + // request. If this field is not set, clients can issue a request as soon as + // they want. + optional Duration minimum_wait_duration = 2; + + // For requested entities that did not match the threat list, how long to + // cache the response. + optional Duration negative_cache_duration = 3; +} + +// A hit comprised of multiple resources; one is the threat list entry that was +// encountered by the client, while others give context as to how the client +// arrived at the unsafe entry. +message ThreatHit { + // The threat type reported. + optional ThreatType threat_type = 1; + + // The platform type reported. + optional PlatformType platform_type = 2; + + // The threat entry responsible for the hit. Full hash should be reported for + // hash-based hits. + optional ThreatEntry entry = 3; + + // Types of resources reported by the client as part of a single hit. + enum ThreatSourceType { + // Unknown. + THREAT_SOURCE_TYPE_UNSPECIFIED = 0; + // The URL that matched the threat list (for which GetFullHash returned a + // valid hash). + MATCHING_URL = 1; + // The final top-level URL of the tab that the client was browsing when the + // match occurred. + TAB_URL = 2; + // A redirect URL that was fetched before hitting the final TAB_URL. + TAB_REDIRECT = 3; + } + + // A single resource related to a threat hit. + message ThreatSource { + // The URL of the resource. + optional string url = 1; + + // The type of source reported. + optional ThreatSourceType type = 2; + + // The remote IP of the resource in ASCII format. Either IPv4 or IPv6. + optional string remote_ip = 3; + + // Referrer of the resource. Only set if the referrer is available. + optional string referrer = 4; + } + + // The resources related to the threat hit. + repeated ThreatSource resources = 4; +} + +// Types of threats. +enum ThreatType { + // Unknown. + THREAT_TYPE_UNSPECIFIED = 0; + + // Malware threat type. + MALWARE_THREAT = 1; + + // Social engineering threat type. + SOCIAL_ENGINEERING_PUBLIC = 2; + + // Unwanted software threat type. + UNWANTED_SOFTWARE = 3; + + // Potentially harmful application threat type. + POTENTIALLY_HARMFUL_APPLICATION = 4; + + // Social engineering threat type for internal use. + SOCIAL_ENGINEERING = 5; + + // API abuse threat type. + API_ABUSE = 6; +} + +// Types of platforms. +enum PlatformType { + // Unknown platform. + PLATFORM_TYPE_UNSPECIFIED = 0; + + // Threat posed to Windows. + WINDOWS_PLATFORM = 1; + + // Threat posed to Linux. + LINUX_PLATFORM = 2; + + // Threat posed to Android. + // This cannot be ANDROID because that symbol is defined for android builds + // here: build/config/android/BUILD.gn line21. + ANDROID_PLATFORM = 3; + + // Threat posed to OSX. + OSX_PLATFORM = 4; + + // Threat posed to iOS. + IOS_PLATFORM = 5; + + // Threat posed to at least one of the defined platforms. + ANY_PLATFORM = 6; + + // Threat posed to all defined platforms. + ALL_PLATFORMS = 7; + + // Threat posed to Chrome. + CHROME_PLATFORM = 8; +} + +// The client metadata associated with Safe Browsing API requests. +message ClientInfo { + // A client ID that (hopefully) uniquely identifies the client implementation + // of the Safe Browsing API. + optional string client_id = 1; + + // The version of the client implementation. + optional string client_version = 2; +} + +// The expected state of a client's local database. +message Checksum { + // The SHA256 hash of the client state; that is, of the sorted list of all + // hashes present in the database. + optional bytes sha256 = 1; +} + +// The ways in which threat entry sets can be compressed. +enum CompressionType { + // Unknown. + COMPRESSION_TYPE_UNSPECIFIED = 0; + + // Raw, uncompressed data. + RAW = 1; + + // Rice-Golomb encoded data. + RICE = 2; +} + +// An individual threat; for example, a malicious URL or its hash +// representation. Only one of these fields should be set. +message ThreatEntry { + // A variable-length SHA256 hash with size between 4 and 32 bytes inclusive. + optional bytes hash = 1; + + // A URL. + optional string url = 2; +} + +// Types of entries that pose threats. Threat lists are collections of entries +// of a single type. +enum ThreatEntryType { + // Unspecified. + THREAT_ENTRY_TYPE_UNSPECIFIED = 0; + + // A host-suffix/path-prefix URL expression; for example, "foo.bar.com/baz/". + URL = 1; + + // An executable program. + EXECUTABLE = 2; + + // An IP range. + IP_RANGE = 3; +} + +// A set of threats that should be added or removed from a client's local +// database. +message ThreatEntrySet { + // The compression type for the entries in this set. + optional CompressionType compression_type = 1; + + // At most one of the following fields should be set. + + // The raw SHA256-formatted entries. + optional RawHashes raw_hashes = 2; + + // The raw removal indices for a local list. + optional RawIndices raw_indices = 3; + + // The encoded 4-byte prefixes of SHA256-formatted entries, using a + // Golomb-Rice encoding. + optional RiceDeltaEncoding rice_hashes = 4; + + // The encoded local, lexicographically-sorted list indices, using a + // Golomb-Rice encoding. Used for sending compressed removal indicies. + optional RiceDeltaEncoding rice_indices = 5; +} + +// A set of raw indicies to remove from a local list. +message RawIndices { + // The indicies to remove from a lexicographically-sorted local list. + repeated int32 indices = 1; +} + +// The uncompressed threat entries in hash format of a particular prefix length. +// Hashes can be anywhere from 4 to 32 bytes in size. A large majority are 4 +// bytes, but some hashes are lengthened if they collide with the hash of a +// popular URL. +// +// Used for sending ThreatEntrySet to clients that do not support compression, +// or when sending non-4-byte hashes to clients that do support compression. +message RawHashes { + // The number of bytes for each prefix encoded below. This field can be + // anywhere from 4 (shortest prefix) to 32 (full SHA256 hash). + optional int32 prefix_size = 1; + + // The hashes, all concatenated into one long string. Each hash has a prefix + // size of |prefix_size| above. Hashes are sorted in lexicographic order. + optional bytes raw_hashes = 2; +} + +// The Rice-Golomb encoded data. Used for sending compressed 4-byte hashes or +// compressed removal indices. +message RiceDeltaEncoding { + // The offset of the first entry in the encoded data, or, if only a single + // integer was encoded, that single integer's value. + optional int64 first_value = 1; + + // The Golomb-Rice parameter which is a number between 2 and 28. This field + // is missing (that is, zero) if num_entries is zero. + optional int32 rice_parameter = 2; + + // The number of entries that are delta encoded in the encoded data. If only a + // single integer was encoded, this will be zero and the single value will be + // stored in first_value. + optional int32 num_entries = 3; + + // The encoded deltas that are encoded using the Golomb-Rice coder. + optional bytes encoded_data = 4; +} + +// The metadata associated with a specific threat entry. The client is expected +// to know the metadata key/value pairs associated with each threat type. +message ThreatEntryMetadata { + // A single metadata entry. + message MetadataEntry { + // The metadata entry key. + optional bytes key = 1; + + // The metadata entry value. + optional bytes value = 2; + } + + // The metadata entries. + repeated MetadataEntry entries = 1; +} + +// Describes an individual threat list. A list is defined by three parameters: +// the type of threat posed, the type of platform targeted by the threat, and +// the type of entries in the list. +message ThreatListDescriptor { + // The threat type posed by the list's entries. + optional ThreatType threat_type = 1; + + // The platform type targeted by the list's entries. + optional PlatformType platform_type = 2; + + // The entry types contained in the list. + optional ThreatEntryType threat_entry_type = 3; +} + +// A collection of lists available for download. +message ListThreatListsResponse { + // The lists available for download. + repeated ThreatListDescriptor threat_lists = 1; +} + +message Duration { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. + optional int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + optional int32 nanos = 2; +} diff --git a/toolkit/components/url-classifier/content/listmanager.js b/toolkit/components/url-classifier/content/listmanager.js new file mode 100644 index 000000000..68325bec8 --- /dev/null +++ b/toolkit/components/url-classifier/content/listmanager.js @@ -0,0 +1,601 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +var Cu = Components.utils; +Cu.import("resource://gre/modules/Services.jsm"); + +// This is the only implementation of nsIUrlListManager. +// A class that manages lists, namely white and black lists for +// phishing or malware protection. The ListManager knows how to fetch, +// update, and store lists. +// +// There is a single listmanager for the whole application. +// +// TODO more comprehensive update tests, for example add unittest check +// that the listmanagers tables are properly written on updates + +// Lower and upper limits on the server-provided polling frequency +const minDelayMs = 5 * 60 * 1000; +const maxDelayMs = 24 * 60 * 60 * 1000; + +// Log only if browser.safebrowsing.debug is true +this.log = function log(...stuff) { + var prefs_ = new G_Preferences(); + var debug = prefs_.getPref("browser.safebrowsing.debug"); + if (!debug) { + return; + } + + var d = new Date(); + let msg = "listmanager: " + d.toTimeString() + ": " + stuff.join(" "); + msg = Services.urlFormatter.trimSensitiveURLs(msg); + Services.console.logStringMessage(msg); + dump(msg + "\n"); +} + +this.QueryAdapter = function QueryAdapter(callback) { + this.callback_ = callback; +}; + +QueryAdapter.prototype.handleResponse = function(value) { + this.callback_.handleEvent(value); +} + +/** + * A ListManager keeps track of black and white lists and knows + * how to update them. + * + * @constructor + */ +this.PROT_ListManager = function PROT_ListManager() { + log("Initializing list manager"); + this.prefs_ = new G_Preferences(); + this.updateInterval = this.prefs_.getPref("urlclassifier.updateinterval", 30 * 60) * 1000; + + // A map of tableNames to objects of type + // { updateUrl: <updateUrl>, gethashUrl: <gethashUrl> } + this.tablesData = {}; + // A map of updateUrls to maps of tables requiring updates, e.g. + // { safebrowsing-update-url: { goog-phish-shavar: true, + // goog-malware-shavar: true } + this.needsUpdate_ = {}; + + this.observerServiceObserver_ = new G_ObserverServiceObserver( + 'quit-application', + BindToObject(this.shutdown_, this), + true /*only once*/); + + // A map of updateUrls to single-use G_Alarms. An entry exists if and only if + // there is at least one table with updates enabled for that url. G_Alarms + // are reset when enabling/disabling updates or on update callbacks (update + // success, update failure, download error). + this.updateCheckers_ = {}; + this.requestBackoffs_ = {}; + this.dbService_ = Cc["@mozilla.org/url-classifier/dbservice;1"] + .getService(Ci.nsIUrlClassifierDBService); + + + this.hashCompleter_ = Cc["@mozilla.org/url-classifier/hashcompleter;1"] + .getService(Ci.nsIUrlClassifierHashCompleter); +} + +/** + * xpcom-shutdown callback + * Delete all of our data tables which seem to leak otherwise. + */ +PROT_ListManager.prototype.shutdown_ = function() { + for (var name in this.tablesData) { + delete this.tablesData[name]; + } +} + +/** + * Register a new table table + * @param tableName - the name of the table + * @param updateUrl - the url for updating the table + * @param gethashUrl - the url for fetching hash completions + * @returns true if the table could be created; false otherwise + */ +PROT_ListManager.prototype.registerTable = function(tableName, + providerName, + updateUrl, + gethashUrl) { + log("registering " + tableName + " with " + updateUrl); + if (!updateUrl) { + log("Can't register table " + tableName + " without updateUrl"); + return false; + } + this.tablesData[tableName] = {}; + this.tablesData[tableName].updateUrl = updateUrl; + this.tablesData[tableName].gethashUrl = gethashUrl; + this.tablesData[tableName].provider = providerName; + + // Keep track of all of our update URLs. + if (!this.needsUpdate_[updateUrl]) { + this.needsUpdate_[updateUrl] = {}; + + // Using the V4 backoff algorithm for both V2 and V4. See bug 1273398. + this.requestBackoffs_[updateUrl] = new RequestBackoffV4( + 4 /* num requests */, + 60*60*1000 /* request time, 60 min */); + } + this.needsUpdate_[updateUrl][tableName] = false; + + return true; +} + +PROT_ListManager.prototype.getGethashUrl = function(tableName) { + if (this.tablesData[tableName] && this.tablesData[tableName].gethashUrl) { + return this.tablesData[tableName].gethashUrl; + } + return ""; +} + +/** + * Enable updates for some tables + * @param tables - an array of table names that need updating + */ +PROT_ListManager.prototype.enableUpdate = function(tableName) { + var table = this.tablesData[tableName]; + if (table) { + log("Enabling table updates for " + tableName); + this.needsUpdate_[table.updateUrl][tableName] = true; + } +} + +/** + * Returns true if any table associated with the updateUrl requires updates. + * @param updateUrl - the updateUrl + */ +PROT_ListManager.prototype.updatesNeeded_ = function(updateUrl) { + let updatesNeeded = false; + for (var tableName in this.needsUpdate_[updateUrl]) { + if (this.needsUpdate_[updateUrl][tableName]) { + updatesNeeded = true; + } + } + return updatesNeeded; +} + +/** + * Disables updates for some tables + * @param tables - an array of table names that no longer need updating + */ +PROT_ListManager.prototype.disableUpdate = function(tableName) { + var table = this.tablesData[tableName]; + if (table) { + log("Disabling table updates for " + tableName); + this.needsUpdate_[table.updateUrl][tableName] = false; + if (!this.updatesNeeded_(table.updateUrl) && + this.updateCheckers_[table.updateUrl]) { + this.updateCheckers_[table.updateUrl].cancel(); + this.updateCheckers_[table.updateUrl] = null; + } + } +} + +/** + * Determine if we have some tables that need updating. + */ +PROT_ListManager.prototype.requireTableUpdates = function() { + for (var name in this.tablesData) { + // Tables that need updating even if other tables don't require it + if (this.needsUpdate_[this.tablesData[name].updateUrl][name]) { + return true; + } + } + + return false; +} + +/** + * Acts as a nsIUrlClassifierCallback for getTables. + */ +PROT_ListManager.prototype.kickoffUpdate_ = function (onDiskTableData) +{ + this.startingUpdate_ = false; + var initialUpdateDelay = 3000; + // Add a fuzz of 0-1 minutes for both v2 and v4 according to Bug 1305478. + initialUpdateDelay += Math.floor(Math.random() * (1 * 60 * 1000)); + + // If the user has never downloaded tables, do the check now. + log("needsUpdate: " + JSON.stringify(this.needsUpdate_, undefined, 2)); + for (var updateUrl in this.needsUpdate_) { + // If we haven't already kicked off updates for this updateUrl, set a + // non-repeating timer for it. The timer delay will be reset either on + // updateSuccess to this.updateInterval, or backed off on downloadError. + // Don't set the updateChecker unless at least one table has updates + // enabled. + if (this.updatesNeeded_(updateUrl) && !this.updateCheckers_[updateUrl]) { + let provider = null; + Object.keys(this.tablesData).forEach(function(table) { + if (this.tablesData[table].updateUrl === updateUrl) { + let newProvider = this.tablesData[table].provider; + if (provider) { + if (newProvider !== provider) { + log("Multiple tables for the same updateURL have a different provider?!"); + } + } else { + provider = newProvider; + } + } + }, this); + log("Initializing update checker for " + updateUrl + + " provided by " + provider); + + // Use the initialUpdateDelay + fuzz unless we had previous updates + // and the server told us when to try again. + let updateDelay = initialUpdateDelay; + let targetPref = "browser.safebrowsing.provider." + provider + ".nextupdatetime"; + let nextUpdate = this.prefs_.getPref(targetPref); + if (nextUpdate) { + updateDelay = Math.min(maxDelayMs, Math.max(0, nextUpdate - Date.now())); + log("Next update at " + nextUpdate); + } + log("Next update " + updateDelay + "ms from now"); + + // Set the last update time to verify if data is still valid. + let freshnessPref = "browser.safebrowsing.provider." + provider + ".lastupdatetime"; + let freshness = this.prefs_.getPref(freshnessPref); + if (freshness) { + Object.keys(this.tablesData).forEach(function(table) { + if (this.tablesData[table].provider === provider) { + this.dbService_.setLastUpdateTime(table, freshness); + }}, this); + } + + this.updateCheckers_[updateUrl] = + new G_Alarm(BindToObject(this.checkForUpdates, this, updateUrl), + updateDelay, false /* repeating */); + } else { + log("No updates needed or already initialized for " + updateUrl); + } + } +} + +PROT_ListManager.prototype.stopUpdateCheckers = function() { + log("Stopping updates"); + for (var updateUrl in this.updateCheckers_) { + if (this.updateCheckers_[updateUrl]) { + this.updateCheckers_[updateUrl].cancel(); + this.updateCheckers_[updateUrl] = null; + } + } +} + +/** + * Determine if we have any tables that require updating. Different + * Wardens may call us with new tables that need to be updated. + */ +PROT_ListManager.prototype.maybeToggleUpdateChecking = function() { + // We update tables if we have some tables that want updates. If there + // are no tables that want to be updated - we dont need to check anything. + if (this.requireTableUpdates()) { + log("Starting managing lists"); + + // Get the list of existing tables from the DBService before making any + // update requests. + if (!this.startingUpdate_) { + this.startingUpdate_ = true; + // check the current state of tables in the database + this.dbService_.getTables(BindToObject(this.kickoffUpdate_, this)); + } + } else { + log("Stopping managing lists (if currently active)"); + this.stopUpdateCheckers(); // Cancel pending updates + } +} + +/** + * Provides an exception free way to look up the data in a table. We + * use this because at certain points our tables might not be loaded, + * and querying them could throw. + * + * @param table String Name of the table that we want to consult + * @param key Principal being used to lookup the database + * @param callback nsIUrlListManagerCallback (ie., Function) given false or the + * value in the table corresponding to key. If the table name does not + * exist, we return false, too. + */ +PROT_ListManager.prototype.safeLookup = function(key, callback) { + try { + log("safeLookup: " + key); + var cb = new QueryAdapter(callback); + this.dbService_.lookup(key, + BindToObject(cb.handleResponse, cb), + true); + } catch(e) { + log("safeLookup masked failure for key " + key + ": " + e); + callback.handleEvent(""); + } +} + +/** + * Updates our internal tables from the update server + * + * @param updateUrl: request updates for tables associated with that url, or + * for all tables if the url is empty. + */ +PROT_ListManager.prototype.checkForUpdates = function(updateUrl) { + log("checkForUpdates with " + updateUrl); + // See if we've triggered the request backoff logic. + if (!updateUrl) { + return false; + } + if (!this.requestBackoffs_[updateUrl] || + !this.requestBackoffs_[updateUrl].canMakeRequest()) { + log("Can't make update request"); + return false; + } + // Grab the current state of the tables from the database + this.dbService_.getTables(BindToObject(this.makeUpdateRequest_, this, + updateUrl)); + return true; +} + +/** + * Method that fires the actual HTTP update request. + * First we reset any tables that have disappeared. + * @param tableData List of table data already in the database, in the form + * tablename;<chunk ranges>\n + */ +PROT_ListManager.prototype.makeUpdateRequest_ = function(updateUrl, tableData) { + log("this.tablesData: " + JSON.stringify(this.tablesData, undefined, 2)); + log("existing chunks: " + tableData + "\n"); + // Disallow blank updateUrls + if (!updateUrl) { + return; + } + // An object of the form + // { tableList: comma-separated list of tables to request, + // tableNames: map of tables that need updating, + // request: list of tables and existing chunk ranges from tableData + // } + var streamerMap = { tableList: null, + tableNames: {}, + requestPayload: "", + isPostRequest: true }; + + let useProtobuf = false; + let onceThru = false; + for (var tableName in this.tablesData) { + // Skip tables not matching this update url + if (this.tablesData[tableName].updateUrl != updateUrl) { + continue; + } + + // Check if |updateURL| is for 'proto'. (only v4 uses protobuf for now.) + // We use the table name 'goog-*-proto' and an additional provider "google4" + // to describe the v4 settings. + let isCurTableProto = tableName.endsWith('-proto'); + if (!onceThru) { + useProtobuf = isCurTableProto; + onceThru = true; + } else if (useProtobuf !== isCurTableProto) { + log('ERROR: Cannot mix "proto" tables with other types ' + + 'within the same provider.'); + } + + if (this.needsUpdate_[this.tablesData[tableName].updateUrl][tableName]) { + streamerMap.tableNames[tableName] = true; + } + if (!streamerMap.tableList) { + streamerMap.tableList = tableName; + } else { + streamerMap.tableList += "," + tableName; + } + } + + if (useProtobuf) { + let tableArray = []; + Object.keys(streamerMap.tableNames).forEach(aTableName => { + if (streamerMap.tableNames[aTableName]) { + tableArray.push(aTableName); + } + }); + + // Build the <tablename, stateBase64> mapping. + let tableState = {}; + tableData.split("\n").forEach(line => { + let p = line.indexOf(";"); + if (-1 === p) { + return; + } + let tableName = line.substring(0, p); + let metadata = line.substring(p + 1).split(":"); + let stateBase64 = metadata[0]; + log(tableName + " ==> " + stateBase64); + tableState[tableName] = stateBase64; + }); + + // The state is a byte stream which server told us from the + // last table update. The state would be used to do the partial + // update and the empty string means the table has + // never been downloaded. See Bug 1287058 for supporting + // partial update. + let stateArray = []; + tableArray.forEach(listName => { + stateArray.push(tableState[listName] || ""); + }); + + log("stateArray: " + stateArray); + + let urlUtils = Cc["@mozilla.org/url-classifier/utils;1"] + .getService(Ci.nsIUrlClassifierUtils); + + streamerMap.requestPayload = urlUtils.makeUpdateRequestV4(tableArray, + stateArray, + tableArray.length); + streamerMap.isPostRequest = false; + } else { + // Build the request. For each table already in the database, include the + // chunk data from the database + var lines = tableData.split("\n"); + for (var i = 0; i < lines.length; i++) { + var fields = lines[i].split(";"); + var name = fields[0]; + if (streamerMap.tableNames[name]) { + streamerMap.requestPayload += lines[i] + "\n"; + delete streamerMap.tableNames[name]; + } + } + // For each requested table that didn't have chunk data in the database, + // request it fresh + for (let tableName in streamerMap.tableNames) { + streamerMap.requestPayload += tableName + ";\n"; + } + + streamerMap.isPostRequest = true; + } + + log("update request: " + JSON.stringify(streamerMap, undefined, 2) + "\n"); + + // Don't send an empty request. + if (streamerMap.requestPayload.length > 0) { + this.makeUpdateRequestForEntry_(updateUrl, streamerMap.tableList, + streamerMap.requestPayload, + streamerMap.isPostRequest); + } else { + // We were disabled between kicking off getTables and now. + log("Not sending empty request"); + } +} + +PROT_ListManager.prototype.makeUpdateRequestForEntry_ = function(updateUrl, + tableList, + requestPayload, + isPostRequest) { + log("makeUpdateRequestForEntry_: requestPayload " + requestPayload + + " update: " + updateUrl + " tablelist: " + tableList + "\n"); + var streamer = Cc["@mozilla.org/url-classifier/streamupdater;1"] + .getService(Ci.nsIUrlClassifierStreamUpdater); + + this.requestBackoffs_[updateUrl].noteRequest(); + + if (!streamer.downloadUpdates( + tableList, + requestPayload, + isPostRequest, + updateUrl, + BindToObject(this.updateSuccess_, this, tableList, updateUrl), + BindToObject(this.updateError_, this, tableList, updateUrl), + BindToObject(this.downloadError_, this, tableList, updateUrl))) { + // Our alarm gets reset in one of the 3 callbacks. + log("pending update, queued request until later"); + } +} + +/** + * Callback function if the update request succeeded. + * @param waitForUpdate String The number of seconds that the client should + * wait before requesting again. + */ +PROT_ListManager.prototype.updateSuccess_ = function(tableList, updateUrl, + waitForUpdateSec) { + log("update success for " + tableList + " from " + updateUrl + ": " + + waitForUpdateSec + "\n"); + + // The time unit below are all milliseconds if not specified. + + var delay = 0; + if (waitForUpdateSec) { + delay = parseInt(waitForUpdateSec, 10) * 1000; + } + // As long as the delay is something sane (5 min to 1 day), update + // our delay time for requesting updates. We always use a non-repeating + // timer since the delay is set differently at every callback. + if (delay > maxDelayMs) { + log("Ignoring delay from server (too long), waiting " + + maxDelayMs + "ms"); + delay = maxDelayMs; + } else if (delay < minDelayMs) { + log("Ignoring delay from server (too short), waiting " + + this.updateInterval + "ms"); + delay = this.updateInterval; + } else { + log("Waiting " + delay + "ms"); + } + this.updateCheckers_[updateUrl] = + new G_Alarm(BindToObject(this.checkForUpdates, this, updateUrl), + delay, false); + + // Let the backoff object know that we completed successfully. + this.requestBackoffs_[updateUrl].noteServerResponse(200); + + // Set last update time for provider + // Get the provider for these tables, check for consistency + let tables = tableList.split(","); + let provider = null; + for (let table of tables) { + let newProvider = this.tablesData[table].provider; + if (provider) { + if (newProvider !== provider) { + log("Multiple tables for the same updateURL have a different provider?!"); + } + } else { + provider = newProvider; + } + } + + // Store the last update time (needed to know if the table is "fresh") + // and the next update time (to know when to update next). + let lastUpdatePref = "browser.safebrowsing.provider." + provider + ".lastupdatetime"; + let now = Date.now(); + log("Setting last update of " + provider + " to " + now); + this.prefs_.setPref(lastUpdatePref, now.toString()); + + let nextUpdatePref = "browser.safebrowsing.provider." + provider + ".nextupdatetime"; + let targetTime = now + delay; + log("Setting next update of " + provider + " to " + targetTime + + " (" + delay + "ms from now)"); + this.prefs_.setPref(nextUpdatePref, targetTime.toString()); +} + +/** + * Callback function if the update request succeeded. + * @param result String The error code of the failure + */ +PROT_ListManager.prototype.updateError_ = function(table, updateUrl, result) { + log("update error for " + table + " from " + updateUrl + ": " + result + "\n"); + // There was some trouble applying the updates. Don't try again for at least + // updateInterval milliseconds. + this.updateCheckers_[updateUrl] = + new G_Alarm(BindToObject(this.checkForUpdates, this, updateUrl), + this.updateInterval, false); +} + +/** + * Callback function when the download failed + * @param status String http status or an empty string if connection refused. + */ +PROT_ListManager.prototype.downloadError_ = function(table, updateUrl, status) { + log("download error for " + table + ": " + status + "\n"); + // If status is empty, then we assume that we got an NS_CONNECTION_REFUSED + // error. In this case, we treat this is a http 500 error. + if (!status) { + status = 500; + } + status = parseInt(status, 10); + this.requestBackoffs_[updateUrl].noteServerResponse(status); + var delay = this.updateInterval; + if (this.requestBackoffs_[updateUrl].isErrorStatus(status)) { + // Schedule an update for when our backoff is complete + delay = this.requestBackoffs_[updateUrl].nextRequestDelay(); + } else { + log("Got non error status for error callback?!"); + } + this.updateCheckers_[updateUrl] = + new G_Alarm(BindToObject(this.checkForUpdates, this, updateUrl), + delay, false); + +} + +PROT_ListManager.prototype.QueryInterface = function(iid) { + if (iid.equals(Ci.nsISupports) || + iid.equals(Ci.nsIUrlListManager) || + iid.equals(Ci.nsITimerCallback)) + return this; + + throw Components.results.NS_ERROR_NO_INTERFACE; +} diff --git a/toolkit/components/url-classifier/content/moz/alarm.js b/toolkit/components/url-classifier/content/moz/alarm.js new file mode 100644 index 000000000..7de067546 --- /dev/null +++ b/toolkit/components/url-classifier/content/moz/alarm.js @@ -0,0 +1,157 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +// An Alarm fires a callback after a certain amount of time, or at +// regular intervals. It's a convenient replacement for +// setTimeout/Interval when you don't want to bind to a specific +// window. +// +// The ConditionalAlarm is an Alarm that cancels itself if its callback +// returns a value that type-converts to true. +// +// Example: +// +// function foo() { dump('hi'); }; +// new G_Alarm(foo, 10*1000); // Fire foo in 10 seconds +// new G_Alarm(foo, 10*1000, true /*repeat*/); // Fire foo every 10 seconds +// new G_Alarm(foo, 10*1000, true, 7); // Fire foo every 10 seconds +// // seven times +// new G_ConditionalAlarm(foo, 1000, true); // Fire every sec until foo()==true +// +// // Fire foo every 10 seconds until foo returns true or until it fires seven +// // times, whichever happens first. +// new G_ConditionalAlarm(foo, 10*1000, true /*repeating*/, 7); +// +// TODO: maybe pass an isFinal flag to the callback if they opted to +// set maxTimes and this is the last iteration? + + +/** + * Set an alarm to fire after a given amount of time, or at specific + * intervals. + * + * @param callback Function to call when the alarm fires + * @param delayMS Number indicating the length of the alarm period in ms + * @param opt_repeating Boolean indicating whether this should fire + * periodically + * @param opt_maxTimes Number indicating a maximum number of times to + * repeat (obviously only useful when opt_repeating==true) + */ +this.G_Alarm = +function G_Alarm(callback, delayMS, opt_repeating, opt_maxTimes) { + this.debugZone = "alarm"; + this.callback_ = callback; + this.repeating_ = !!opt_repeating; + this.timer_ = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + var type = opt_repeating ? + this.timer_.TYPE_REPEATING_SLACK : + this.timer_.TYPE_ONE_SHOT; + this.maxTimes_ = opt_maxTimes ? opt_maxTimes : null; + this.nTimes_ = 0; + + this.observerServiceObserver_ = new G_ObserverServiceObserver( + 'xpcom-shutdown', + BindToObject(this.cancel, this)); + + // Ask the timer to use nsITimerCallback (.notify()) when ready + this.timer_.initWithCallback(this, delayMS, type); +} + +/** + * Cancel this timer + */ +G_Alarm.prototype.cancel = function() { + if (!this.timer_) { + return; + } + + this.timer_.cancel(); + // Break circular reference created between this.timer_ and the G_Alarm + // instance (this) + this.timer_ = null; + this.callback_ = null; + + // We don't need the shutdown observer anymore + this.observerServiceObserver_.unregister(); +} + +/** + * Invoked by the timer when it fires + * + * @param timer Reference to the nsITimer which fired (not currently + * passed along) + */ +G_Alarm.prototype.notify = function(timer) { + // fire callback and save results + var ret = this.callback_(); + + // If they've given us a max number of times to fire, enforce it + this.nTimes_++; + if (this.repeating_ && + typeof this.maxTimes_ == "number" + && this.nTimes_ >= this.maxTimes_) { + this.cancel(); + } else if (!this.repeating_) { + // Clear out the callback closure for TYPE_ONE_SHOT timers + this.cancel(); + } + // We don't cancel/cleanup timers that repeat forever until either + // xpcom-shutdown occurs or cancel() is called explicitly. + + return ret; +} + +G_Alarm.prototype.setDelay = function(delay) { + this.timer_.delay = delay; +} + +/** + * XPCOM cruft + */ +G_Alarm.prototype.QueryInterface = function(iid) { + if (iid.equals(Components.interfaces.nsISupports) || + iid.equals(Components.interfaces.nsITimerCallback)) + return this; + + throw Components.results.NS_ERROR_NO_INTERFACE; +} + + +/** + * An alarm with the additional property that it cancels itself if its + * callback returns true. + * + * For parameter documentation, see G_Alarm + */ +this.G_ConditionalAlarm = +function G_ConditionalAlarm(callback, delayMS, opt_repeating, opt_maxTimes) { + G_Alarm.call(this, callback, delayMS, opt_repeating, opt_maxTimes); + this.debugZone = "conditionalalarm"; +} + +G_ConditionalAlarm.inherits = function(parentCtor) { + var tempCtor = function(){}; + tempCtor.prototype = parentCtor.prototype; + this.superClass_ = parentCtor.prototype; + this.prototype = new tempCtor(); +} + +G_ConditionalAlarm.inherits(G_Alarm); + +/** + * Invoked by the timer when it fires + * + * @param timer Reference to the nsITimer which fired (not currently + * passed along) + */ +G_ConditionalAlarm.prototype.notify = function(timer) { + // Call G_Alarm::notify + var rv = G_Alarm.prototype.notify.call(this, timer); + + if (this.repeating_ && rv) { + G_Debug(this, "Callback of a repeating alarm returned true; cancelling."); + this.cancel(); + } +} diff --git a/toolkit/components/url-classifier/content/moz/cryptohasher.js b/toolkit/components/url-classifier/content/moz/cryptohasher.js new file mode 100644 index 000000000..a1294aa93 --- /dev/null +++ b/toolkit/components/url-classifier/content/moz/cryptohasher.js @@ -0,0 +1,176 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +// A very thin wrapper around nsICryptoHash. It's not strictly +// necessary, but makes the code a bit cleaner and gives us the +// opportunity to verify that our implementations give the results that +// we expect, for example if we have to interoperate with a server. +// +// The digest* methods reset the state of the hasher, so it's +// necessary to call init() explicitly after them. +// +// Works only in Firefox 1.5+. +// +// IMPORTANT NOTE: Due to https://bugzilla.mozilla.org/show_bug.cgi?id=321024 +// you cannot use the cryptohasher before app-startup. The symptom of doing +// so is a segfault in NSS. + +/** + * Instantiate a new hasher. You must explicitly call init() before use! + */ +this.G_CryptoHasher = +function G_CryptoHasher() { + this.debugZone = "cryptohasher"; + this.hasher_ = null; +} + +G_CryptoHasher.algorithms = { + MD2: Ci.nsICryptoHash.MD2, + MD5: Ci.nsICryptoHash.MD5, + SHA1: Ci.nsICryptoHash.SHA1, + SHA256: Ci.nsICryptoHash.SHA256, + SHA384: Ci.nsICryptoHash.SHA384, + SHA512: Ci.nsICryptoHash.SHA512, +}; + +/** + * Initialize the hasher. This function must be called after every call + * to one of the digest* methods. + * + * @param algorithm Constant from G_CryptoHasher.algorithms specifying the + * algorithm this hasher will use + */ +G_CryptoHasher.prototype.init = function(algorithm) { + var validAlgorithm = false; + for (var alg in G_CryptoHasher.algorithms) + if (algorithm == G_CryptoHasher.algorithms[alg]) + validAlgorithm = true; + + if (!validAlgorithm) + throw new Error("Invalid algorithm: " + algorithm); + + this.hasher_ = Cc["@mozilla.org/security/hash;1"] + .createInstance(Ci.nsICryptoHash); + this.hasher_.init(algorithm); +} + +/** + * Update the hash's internal state with input given in a string. Can be + * called multiple times for incrementeal hash updates. + * + * @param input String containing data to hash. + */ +G_CryptoHasher.prototype.updateFromString = function(input) { + if (!this.hasher_) + throw new Error("You must initialize the hasher first!"); + + var stream = Cc['@mozilla.org/io/string-input-stream;1'] + .createInstance(Ci.nsIStringInputStream); + stream.setData(input, input.length); + this.updateFromStream(stream); +} + +/** + * Update the hash's internal state with input given in an array. Can be + * called multiple times for incremental hash updates. + * + * @param input Array containing data to hash. + */ +G_CryptoHasher.prototype.updateFromArray = function(input) { + if (!this.hasher_) + throw new Error("You must initialize the hasher first!"); + + this.hasher_.update(input, input.length); +} + +/** + * Update the hash's internal state with input given in a stream. Can be + * called multiple times from incremental hash updates. + */ +G_CryptoHasher.prototype.updateFromStream = function(stream) { + if (!this.hasher_) + throw new Error("You must initialize the hasher first!"); + + if (stream.available()) + this.hasher_.updateFromStream(stream, stream.available()); +} + +/** + * @returns The hash value as a string (sequence of 8-bit values) + */ +G_CryptoHasher.prototype.digestRaw = function() { + var digest = this.hasher_.finish(false /* not b64 encoded */); + this.hasher_ = null; + return digest; +} + +/** + * @returns The hash value as a base64-encoded string + */ +G_CryptoHasher.prototype.digestBase64 = function() { + var digest = this.hasher_.finish(true /* b64 encoded */); + this.hasher_ = null; + return digest; +} + +/** + * @returns The hash value as a hex-encoded string + */ +G_CryptoHasher.prototype.digestHex = function() { + var raw = this.digestRaw(); + return this.toHex_(raw); +} + +/** + * Converts a sequence of values to a hex-encoded string. The input is a + * a string, so you can stick 16-bit values in each character. + * + * @param str String to conver to hex. (Often this is just a sequence of + * 16-bit values) + * + * @returns String containing the hex representation of the input + */ +G_CryptoHasher.prototype.toHex_ = function(str) { + var hexchars = '0123456789ABCDEF'; + var hexrep = new Array(str.length * 2); + + for (var i = 0; i < str.length; ++i) { + hexrep[i * 2] = hexchars.charAt((str.charCodeAt(i) >> 4) & 15); + hexrep[i * 2 + 1] = hexchars.charAt(str.charCodeAt(i) & 15); + } + return hexrep.join(''); +} + +#ifdef DEBUG +/** + * Lame unittest function + */ +this.TEST_G_CryptoHasher = function TEST_G_CryptoHasher() { + if (G_GDEBUG) { + var z = "cryptohasher UNITTEST"; + G_debugService.enableZone(z); + + G_Debug(z, "Starting"); + + var md5 = function(str) { + var hasher = new G_CryptoHasher(); + hasher.init(G_CryptoHasher.algorithms.MD5); + hasher.updateFromString(str); + return hasher.digestHex().toLowerCase(); + }; + + // test vectors from: http://www.faqs.org/rfcs/rfc1321.html + var vectors = {"": "d41d8cd98f00b204e9800998ecf8427e", + "a": "0cc175b9c0f1b6a831c399e269772661", + "abc": "900150983cd24fb0d6963f7d28e17f72", + "message digest": "f96b697d7cb7938d525a2f31aaf161d0", + "abcdefghijklmnopqrstuvwxyz": "c3fcd3d76192e4007dfb496cca67e13b", + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789": "d174ab98d277d9f5a5611c2c9f419d9f", + "12345678901234567890123456789012345678901234567890123456789012345678901234567890": "57edf4a22be3c955ac49da2e2107b67a"}; + + G_Debug(z, "PASSED"); + } +} +#endif diff --git a/toolkit/components/url-classifier/content/moz/debug.js b/toolkit/components/url-classifier/content/moz/debug.js new file mode 100644 index 000000000..ed4c11793 --- /dev/null +++ b/toolkit/components/url-classifier/content/moz/debug.js @@ -0,0 +1,867 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifdef DEBUG + +// Generic logging/debugging functionality that: +// +// (*) when disabled compiles to no-ops at worst (for calls to the service) +// and to nothing at best (calls to G_Debug() and similar are compiled +// away when you use a jscompiler that strips dead code) +// +// (*) has dynamically configurable/creatable debugging "zones" enabling +// selective logging +// +// (*) hides its plumbing so that all calls in different zones are uniform, +// so you can drop files using this library into other apps that use it +// without any configuration +// +// (*) can be controlled programmatically or via preferences. The +// preferences that control the service and its zones are under +// the preference branch "safebrowsing-debug-service." +// +// (*) outputs function call traces when the "loggifier" zone is enabled +// +// (*) can write output to logfiles so that you can get a call trace +// from someone who is having a problem +// +// Example: +// +// var G_GDEBUG = true // Enable this module +// var G_debugService = new G_DebugService(); // in global context +// +// // You can use it with arbitrary primitive first arguement +// G_Debug("myzone", "Yo yo yo"); // outputs: [myzone] Yo yo yo\n +// +// // But it's nice to use it with an object; it will probe for the zone name +// function Obj() { +// this.debugZone = "someobj"; +// } +// Obj.prototype.foo = function() { +// G_Debug(this, "foo called"); +// } +// (new Obj).foo(); // outputs: [someobj] foo called\n +// +// G_debugService.loggifier.loggify(Obj.prototype); // enable call tracing +// +// // En/disable specific zones programmatically (you can also use preferences) +// G_debugService.enableZone("somezone"); +// G_debugService.disableZone("someotherzone"); +// G_debugService.enableAllZones(); +// +// // We also have asserts and errors: +// G_Error(this, "Some error occurred"); // will throw +// G_Assert(this, (x > 3), "x not greater than three!"); // will throw +// +// See classes below for more methods. +// +// TODO add code to set prefs when not found to the default value of a tristate +// TODO add error level support +// TODO add ability to turn off console output +// +// -------> TO START DEBUGGING: set G_GDEBUG to true + +// These are the functions code will typically call. Everything is +// wrapped in if's so we can compile it away when G_GDEBUG is false. + + +if (typeof G_GDEBUG == "undefined") { + throw new Error("G_GDEBUG constant must be set before loading debug.js"); +} + + +/** + * Write out a debugging message. + * + * @param who The thingy to convert into a zone name corresponding to the + * zone to which this message belongs + * @param msg Message to output + */ +this.G_Debug = function G_Debug(who, msg) { + if (G_GDEBUG) { + G_GetDebugZone(who).debug(msg); + } +} + +/** + * Debugs loudly + */ +this.G_DebugL = function G_DebugL(who, msg) { + if (G_GDEBUG) { + var zone = G_GetDebugZone(who); + + if (zone.zoneIsEnabled()) { + G_debugService.dump( + "\n************************************************************\n"); + + G_Debug(who, msg); + + G_debugService.dump( + "************************************************************\n\n"); + } + } +} + +/** + * Write out a call tracing message + * + * @param who The thingy to convert into a zone name corresponding to the + * zone to which this message belongs + * @param msg Message to output + */ +this.G_TraceCall = function G_TraceCall(who, msg) { + if (G_GDEBUG) { + if (G_debugService.callTracingEnabled()) { + G_debugService.dump(msg + "\n"); + } + } +} + +/** + * Write out an error (and throw) + * + * @param who The thingy to convert into a zone name corresponding to the + * zone to which this message belongs + * @param msg Message to output + */ +this.G_Error = function G_Error(who, msg) { + if (G_GDEBUG) { + G_GetDebugZone(who).error(msg); + } +} + +/** + * Assert something as true and signal an error if it's not + * + * @param who The thingy to convert into a zone name corresponding to the + * zone to which this message belongs + * @param condition Boolean condition to test + * @param msg Message to output + */ +this.G_Assert = function G_Assert(who, condition, msg) { + if (G_GDEBUG) { + G_GetDebugZone(who).assert(condition, msg); + } +} + +/** + * Helper function that takes input and returns the DebugZone + * corresponding to it. + * + * @param who Arbitrary input that will be converted into a zone name. Most + * likely an object that has .debugZone property, or a string. + * @returns The DebugZone object corresponding to the input + */ +this.G_GetDebugZone = function G_GetDebugZone(who) { + if (G_GDEBUG) { + var zone = "?"; + + if (who && who.debugZone) { + zone = who.debugZone; + } else if (typeof who == "string") { + zone = who; + } + + return G_debugService.getZone(zone); + } +} + +// Classes that implement the functionality. + +/** + * A debug "zone" is a string derived from arbitrary types (but + * typically derived from another string or an object). All debugging + * messages using a particular zone can be enabled or disabled + * independent of other zones. This enables you to turn on/off logging + * of particular objects or modules. This object implements a single + * zone and the methods required to use it. + * + * @constructor + * @param service Reference to the DebugService object we use for + * registration + * @param prefix String indicating the unique prefix we should use + * when creating preferences to control this zone + * @param zone String indicating the name of the zone + */ +this.G_DebugZone = function G_DebugZone(service, prefix, zone) { + if (G_GDEBUG) { + this.debugService_ = service; + this.prefix_ = prefix; + this.zone_ = zone; + this.zoneEnabledPrefName_ = prefix + ".zone." + this.zone_; + this.settings_ = new G_DebugSettings(); + } +} + +/** + * @returns Boolean indicating if this zone is enabled + */ +G_DebugZone.prototype.zoneIsEnabled = function() { + if (G_GDEBUG) { + var explicit = this.settings_.getSetting(this.zoneEnabledPrefName_, null); + + if (explicit !== null) { + return explicit; + } else { + return this.debugService_.allZonesEnabled(); + } + } +} + +/** + * Enable this logging zone + */ +G_DebugZone.prototype.enableZone = function() { + if (G_GDEBUG) { + this.settings_.setDefault(this.zoneEnabledPrefName_, true); + } +} + +/** + * Disable this logging zone + */ +G_DebugZone.prototype.disableZone = function() { + if (G_GDEBUG) { + this.settings_.setDefault(this.zoneEnabledPrefName_, false); + } +} + +/** + * Write a debugging message to this zone + * + * @param msg String of message to write + */ +G_DebugZone.prototype.debug = function(msg) { + if (G_GDEBUG) { + if (this.zoneIsEnabled()) { + this.debugService_.dump("[" + this.zone_ + "] " + msg + "\n"); + } + } +} + +/** + * Write an error to this zone and throw + * + * @param msg String of error to write + */ +G_DebugZone.prototype.error = function(msg) { + if (G_GDEBUG) { + this.debugService_.dump("[" + this.zone_ + "] " + msg + "\n"); + throw new Error(msg); + debugger; + } +} + +/** + * Assert something as true and error if it is not + * + * @param condition Boolean condition to test + * @param msg String of message to write if is false + */ +G_DebugZone.prototype.assert = function(condition, msg) { + if (G_GDEBUG) { + if (condition !== true) { + G_Error(this.zone_, "ASSERT FAILED: " + msg); + } + } +} + + +/** + * The debug service handles auto-registration of zones, namespacing + * the zones preferences, and various global settings such as whether + * all zones are enabled. + * + * @constructor + * @param opt_prefix Optional string indicating the unique prefix we should + * use when creating preferences + */ +this.G_DebugService = function G_DebugService(opt_prefix) { + if (G_GDEBUG) { + this.prefix_ = opt_prefix ? opt_prefix : "safebrowsing-debug-service"; + this.consoleEnabledPrefName_ = this.prefix_ + ".alsologtoconsole"; + this.allZonesEnabledPrefName_ = this.prefix_ + ".enableallzones"; + this.callTracingEnabledPrefName_ = this.prefix_ + ".trace-function-calls"; + this.logFileEnabledPrefName_ = this.prefix_ + ".logfileenabled"; + this.logFileErrorLevelPrefName_ = this.prefix_ + ".logfile-errorlevel"; + this.zones_ = {}; + + this.loggifier = new G_Loggifier(); + this.settings_ = new G_DebugSettings(); + } +} + +// Error levels for reporting console messages to the log. +G_DebugService.ERROR_LEVEL_INFO = "INFO"; +G_DebugService.ERROR_LEVEL_WARNING = "WARNING"; +G_DebugService.ERROR_LEVEL_EXCEPTION = "EXCEPTION"; + + +/** + * @returns Boolean indicating if we should send messages to the jsconsole + */ +G_DebugService.prototype.alsoDumpToConsole = function() { + if (G_GDEBUG) { + return this.settings_.getSetting(this.consoleEnabledPrefName_, false); + } +} + +/** + * @returns whether to log output to a file as well as the console. + */ +G_DebugService.prototype.logFileIsEnabled = function() { + if (G_GDEBUG) { + return this.settings_.getSetting(this.logFileEnabledPrefName_, false); + } +} + +/** + * Turns on file logging. dump() output will also go to the file specified by + * setLogFile() + */ +G_DebugService.prototype.enableLogFile = function() { + if (G_GDEBUG) { + this.settings_.setDefault(this.logFileEnabledPrefName_, true); + } +} + +/** + * Turns off file logging + */ +G_DebugService.prototype.disableLogFile = function() { + if (G_GDEBUG) { + this.settings_.setDefault(this.logFileEnabledPrefName_, false); + } +} + +/** + * @returns an nsIFile instance pointing to the current log file location + */ +G_DebugService.prototype.getLogFile = function() { + if (G_GDEBUG) { + return this.logFile_; + } +} + +/** + * Sets a new log file location + */ +G_DebugService.prototype.setLogFile = function(file) { + if (G_GDEBUG) { + this.logFile_ = file; + } +} + +/** + * Enables sending messages to the jsconsole + */ +G_DebugService.prototype.enableDumpToConsole = function() { + if (G_GDEBUG) { + this.settings_.setDefault(this.consoleEnabledPrefName_, true); + } +} + +/** + * Disables sending messages to the jsconsole + */ +G_DebugService.prototype.disableDumpToConsole = function() { + if (G_GDEBUG) { + this.settings_.setDefault(this.consoleEnabledPrefName_, false); + } +} + +/** + * @param zone Name of the zone to get + * @returns The DebugZone object corresopnding to input. If not such + * zone exists, a new one is created and returned + */ +G_DebugService.prototype.getZone = function(zone) { + if (G_GDEBUG) { + if (!this.zones_[zone]) + this.zones_[zone] = new G_DebugZone(this, this.prefix_, zone); + + return this.zones_[zone]; + } +} + +/** + * @param zone Zone to enable debugging for + */ +G_DebugService.prototype.enableZone = function(zone) { + if (G_GDEBUG) { + var toEnable = this.getZone(zone); + toEnable.enableZone(); + } +} + +/** + * @param zone Zone to disable debugging for + */ +G_DebugService.prototype.disableZone = function(zone) { + if (G_GDEBUG) { + var toDisable = this.getZone(zone); + toDisable.disableZone(); + } +} + +/** + * @returns Boolean indicating whether debugging is enabled for all zones + */ +G_DebugService.prototype.allZonesEnabled = function() { + if (G_GDEBUG) { + return this.settings_.getSetting(this.allZonesEnabledPrefName_, false); + } +} + +/** + * Enables all debugging zones + */ +G_DebugService.prototype.enableAllZones = function() { + if (G_GDEBUG) { + this.settings_.setDefault(this.allZonesEnabledPrefName_, true); + } +} + +/** + * Disables all debugging zones + */ +G_DebugService.prototype.disableAllZones = function() { + if (G_GDEBUG) { + this.settings_.setDefault(this.allZonesEnabledPrefName_, false); + } +} + +/** + * @returns Boolean indicating whether call tracing is enabled + */ +G_DebugService.prototype.callTracingEnabled = function() { + if (G_GDEBUG) { + return this.settings_.getSetting(this.callTracingEnabledPrefName_, false); + } +} + +/** + * Enables call tracing + */ +G_DebugService.prototype.enableCallTracing = function() { + if (G_GDEBUG) { + this.settings_.setDefault(this.callTracingEnabledPrefName_, true); + } +} + +/** + * Disables call tracing + */ +G_DebugService.prototype.disableCallTracing = function() { + if (G_GDEBUG) { + this.settings_.setDefault(this.callTracingEnabledPrefName_, false); + } +} + +/** + * Gets the minimum error that will be reported to the log. + */ +G_DebugService.prototype.getLogFileErrorLevel = function() { + if (G_GDEBUG) { + var level = this.settings_.getSetting(this.logFileErrorLevelPrefName_, + G_DebugService.ERROR_LEVEL_EXCEPTION); + + return level.toUpperCase(); + } +} + +/** + * Sets the minimum error level that will be reported to the log. + */ +G_DebugService.prototype.setLogFileErrorLevel = function(level) { + if (G_GDEBUG) { + // normalize case just to make it slightly easier to not screw up. + level = level.toUpperCase(); + + if (level != G_DebugService.ERROR_LEVEL_INFO && + level != G_DebugService.ERROR_LEVEL_WARNING && + level != G_DebugService.ERROR_LEVEL_EXCEPTION) { + throw new Error("Invalid error level specified: {" + level + "}"); + } + + this.settings_.setDefault(this.logFileErrorLevelPrefName_, level); + } +} + +/** + * Internal dump() method + * + * @param msg String of message to dump + */ +G_DebugService.prototype.dump = function(msg) { + if (G_GDEBUG) { + dump(msg); + + if (this.alsoDumpToConsole()) { + try { + var console = Components.classes['@mozilla.org/consoleservice;1'] + .getService(Components.interfaces.nsIConsoleService); + console.logStringMessage(msg); + } catch(e) { + dump("G_DebugZone ERROR: COULD NOT DUMP TO CONSOLE\n"); + } + } + + this.maybeDumpToFile(msg); + } +} + +/** + * Writes the specified message to the log file, if file logging is enabled. + */ +G_DebugService.prototype.maybeDumpToFile = function(msg) { + if (this.logFileIsEnabled() && this.logFile_) { + + /* try to get the correct line end character for this platform */ + if (!this._LINE_END_CHAR) + this._LINE_END_CHAR = + Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULRuntime) + .OS == "WINNT" ? "\r\n" : "\n"; + if (this._LINE_END_CHAR != "\n") + msg = msg.replace(/\n/g, this._LINE_END_CHAR); + + try { + var stream = Cc["@mozilla.org/network/file-output-stream;1"] + .createInstance(Ci.nsIFileOutputStream); + stream.init(this.logFile_, + 0x02 | 0x08 | 0x10 /* PR_WRONLY | PR_CREATE_FILE | PR_APPEND */ + -1 /* default perms */, 0 /* no special behavior */); + stream.write(msg, msg.length); + } finally { + stream.close(); + } + } +} + +/** + * Implements nsIConsoleListener.observe(). Gets called when an error message + * gets reported to the console and sends it to the log file as well. + */ +G_DebugService.prototype.observe = function(consoleMessage) { + if (G_GDEBUG) { + var errorLevel = this.getLogFileErrorLevel(); + + // consoleMessage can be either nsIScriptError or nsIConsoleMessage. The + // latter does not have things like line number, etc. So we special case + // it first. + if (!(consoleMessage instanceof Ci.nsIScriptError)) { + // Only report these messages if the error level is INFO. + if (errorLevel == G_DebugService.ERROR_LEVEL_INFO) { + this.maybeDumpToFile(G_DebugService.ERROR_LEVEL_INFO + ": " + + consoleMessage.message + "\n"); + } + + return; + } + + // We make a local copy of these fields because writing to it doesn't seem + // to work. + var flags = consoleMessage.flags; + var sourceName = consoleMessage.sourceName; + var lineNumber = consoleMessage.lineNumber; + + // Sometimes, a scripterror instance won't have any flags set. We + // default to exception. + if (!flags) { + flags = Ci.nsIScriptError.exceptionFlag; + } + + // Default the filename and line number if they aren't set. + if (!sourceName) { + sourceName = "<unknown>"; + } + + if (!lineNumber) { + lineNumber = "<unknown>"; + } + + // Report the error in the log file. + if (flags & Ci.nsIScriptError.warningFlag) { + // Only report warnings if the error level is warning or better. + if (errorLevel == G_DebugService.ERROR_LEVEL_WARNING || + errorLevel == G_DebugService.ERROR_LEVEL_INFO) { + this.reportScriptError_(consoleMessage.message, + sourceName, + lineNumber, + G_DebugService.ERROR_LEVEL_WARNING); + } + } else if (flags & Ci.nsIScriptError.exceptionFlag) { + // Always report exceptions. + this.reportScriptError_(consoleMessage.message, + sourceName, + lineNumber, + G_DebugService.ERROR_LEVEL_EXCEPTION); + } + } +} + +/** + * Private helper to report an nsIScriptError instance to the log/console. + */ +G_DebugService.prototype.reportScriptError_ = function(message, sourceName, + lineNumber, label) { + message = "\n------------------------------------------------------------\n" + + label + ": " + message + + "\nlocation: " + sourceName + ", " + "line: " + lineNumber + + "\n------------------------------------------------------------\n\n"; + + dump(message); + this.maybeDumpToFile(message); +} + + + +/** + * A class that instruments methods so they output a call trace, + * including the values of their actual parameters and return value. + * This code is mostly stolen from Aaron Boodman's original + * implementation in clobber utils. + * + * Note that this class uses the "loggifier" debug zone, so you'll see + * a complete call trace when that zone is enabled. + * + * @constructor + */ +this.G_Loggifier = function G_Loggifier() { + if (G_GDEBUG) { + // Careful not to loggify ourselves! + this.mark_(this); + } +} + +/** + * Marks an object as having been loggified. Loggification is not + * idempotent :) + * + * @param obj Object to be marked + */ +G_Loggifier.prototype.mark_ = function(obj) { + if (G_GDEBUG) { + obj.__loggified_ = true; + } +} + +/** + * @param obj Object to be examined + * @returns Boolean indicating if the object has been loggified + */ +G_Loggifier.prototype.isLoggified = function(obj) { + if (G_GDEBUG) { + return !!obj.__loggified_; + } +} + +/** + * Attempt to extract the class name from the constructor definition. + * Assumes the object was created using new. + * + * @param constructor String containing the definition of a constructor, + * for example what you'd get by examining obj.constructor + * @returns Name of the constructor/object if it could be found, else "???" + */ +G_Loggifier.prototype.getFunctionName_ = function(constructor) { + if (G_GDEBUG) { + return constructor.name || "???"; + } +} + +/** + * Wraps all the methods in an object so that call traces are + * automatically outputted. + * + * @param obj Object to loggify. SHOULD BE THE PROTOTYPE OF A USER-DEFINED + * object. You can get into trouble if you attempt to + * loggify something that isn't, for example the Window. + * + * Any additional parameters are considered method names which should not be + * loggified. + * + * Usage: + * G_debugService.loggifier.loggify(MyClass.prototype, + * "firstMethodNotToLog", + * "secondMethodNotToLog", + * ... etc ...); + */ +G_Loggifier.prototype.loggify = function(obj) { + if (G_GDEBUG) { + if (!G_debugService.callTracingEnabled()) { + return; + } + + if (typeof window != "undefined" && obj == window || + this.isLoggified(obj)) // Don't go berserk! + return; + + var zone = G_GetDebugZone(obj); + if (!zone || !zone.zoneIsEnabled()) { + return; + } + + this.mark_(obj); + + // Helper function returns an instrumented version of + // objName.meth, with "this" bound properly. (BTW, because we're + // in a conditional here, functions will only be defined as + // they're encountered during execution, so declare this helper + // before using it.) + + let wrap = function (meth, objName, methName) { + return function() { + + // First output the call along with actual parameters + var args = new Array(arguments.length); + var argsString = ""; + for (var i = 0; i < args.length; i++) { + args[i] = arguments[i]; + argsString += (i == 0 ? "" : ", "); + + if (typeof args[i] == "function") { + argsString += "[function]"; + } else { + argsString += args[i]; + } + } + + G_TraceCall(this, "> " + objName + "." + methName + "(" + + argsString + ")"); + + // Then run the function, capturing the return value and throws + try { + var retVal = meth.apply(this, arguments); + var reportedRetVal = retVal; + + if (typeof reportedRetVal == "undefined") + reportedRetVal = "void"; + else if (reportedRetVal === "") + reportedRetVal = "\"\" (empty string)"; + } catch (e) { + if (e && !e.__logged) { + G_TraceCall(this, "Error: " + e.message + ". " + + e.fileName + ": " + e.lineNumber); + try { + e.__logged = true; + } catch (e2) { + // Sometimes we can't add the __logged flag because it's an + // XPC wrapper + throw e; + } + } + + throw e; // Re-throw! + } + + // And spit it out already + G_TraceCall( + this, + "< " + objName + "." + methName + ": " + reportedRetVal); + + return retVal; + }; + }; + + var ignoreLookup = {}; + + if (arguments.length > 1) { + for (var i = 1; i < arguments.length; i++) { + ignoreLookup[arguments[i]] = true; + } + } + + // Wrap each method of obj + for (var p in obj) { + // Work around bug in Firefox. In ffox typeof RegExp is "function", + // so make sure this really is a function. Bug as of FFox 1.5b2. + if (typeof obj[p] == "function" && obj[p].call && !ignoreLookup[p]) { + var objName = this.getFunctionName_(obj.constructor); + obj[p] = wrap(obj[p], objName, p); + } + } + } +} + + +/** + * Simple abstraction around debug settings. The thing with debug settings is + * that we want to be able to specify a default in the application's startup, + * but have that default be overridable by the user via their prefs. + * + * To generalize this, we package up a dictionary of defaults with the + * preferences tree. If a setting isn't in the preferences tree, then we grab it + * from the defaults. + */ +this.G_DebugSettings = function G_DebugSettings() { + this.defaults_ = {}; + this.prefs_ = new G_Preferences(); +} + +/** + * Returns the value of a settings, optionally defaulting to a given value if it + * doesn't exist. If no default is specified, the default is |undefined|. + */ +G_DebugSettings.prototype.getSetting = function(name, opt_default) { + var override = this.prefs_.getPref(name, null); + + if (override !== null) { + return override; + } else if (typeof this.defaults_[name] != "undefined") { + return this.defaults_[name]; + } else { + return opt_default; + } +} + +/** + * Sets the default value for a setting. If the user doesn't override it with a + * preference, this is the value which will be returned by getSetting(). + */ +G_DebugSettings.prototype.setDefault = function(name, val) { + this.defaults_[name] = val; +} + +var G_debugService = new G_DebugService(); // Instantiate us! + +if (G_GDEBUG) { + G_debugService.enableAllZones(); +} + +#else + +// Stubs for the debugging aids scattered through this component. +// They will be expanded if you compile yourself a debug build. + +this.G_Debug = function G_Debug(who, msg) { } +this.G_Assert = function G_Assert(who, condition, msg) { } +this.G_Error = function G_Error(who, msg) { } +this.G_debugService = { + alsoDumpToConsole: () => {}, + logFileIsEnabled: () => {}, + enableLogFile: () => {}, + disableLogFile: () => {}, + getLogFile: () => {}, + setLogFile: () => {}, + enableDumpToConsole: () => {}, + disableDumpToConsole: () => {}, + getZone: () => {}, + enableZone: () => {}, + disableZone: () => {}, + allZonesEnabled: () => {}, + enableAllZones: () => {}, + disableAllZones: () => {}, + callTracingEnabled: () => {}, + enableCallTracing: () => {}, + disableCallTracing: () => {}, + getLogFileErrorLevel: () => {}, + setLogFileErrorLevel: () => {}, + dump: () => {}, + maybeDumpToFile: () => {}, + observe: () => {}, + reportScriptError_: () => {} +}; + +#endif diff --git a/toolkit/components/url-classifier/content/moz/lang.js b/toolkit/components/url-classifier/content/moz/lang.js new file mode 100644 index 000000000..804a6e973 --- /dev/null +++ b/toolkit/components/url-classifier/content/moz/lang.js @@ -0,0 +1,82 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +/** + * lang.js - Some missing JavaScript language features + */ + +/** + * Partially applies a function to a particular "this object" and zero or + * more arguments. The result is a new function with some arguments of the first + * function pre-filled and the value of |this| "pre-specified". + * + * Remaining arguments specified at call-time are appended to the pre- + * specified ones. + * + * Usage: + * var barMethBound = BindToObject(myFunction, myObj, "arg1", "arg2"); + * barMethBound("arg3", "arg4"); + * + * @param fn {string} Reference to the function to be bound + * + * @param self {object} Specifies the object which |this| should point to + * when the function is run. If the value is null or undefined, it will default + * to the global object. + * + * @returns {function} A partially-applied form of the speficied function. + */ +this.BindToObject = function BindToObject(fn, self, opt_args) { + var boundargs = fn.boundArgs_ || []; + boundargs = boundargs.concat(Array.slice(arguments, 2, arguments.length)); + + if (fn.boundSelf_) + self = fn.boundSelf_; + if (fn.boundFn_) + fn = fn.boundFn_; + + var newfn = function() { + // Combine the static args and the new args into one big array + var args = boundargs.concat(Array.slice(arguments)); + return fn.apply(self, args); + } + + newfn.boundArgs_ = boundargs; + newfn.boundSelf_ = self; + newfn.boundFn_ = fn; + + return newfn; +} + +/** + * Inherit the prototype methods from one constructor into another. + * + * Usage: + * + * function ParentClass(a, b) { } + * ParentClass.prototype.foo = function(a) { } + * + * function ChildClass(a, b, c) { + * ParentClass.call(this, a, b); + * } + * + * ChildClass.inherits(ParentClass); + * + * var child = new ChildClass("a", "b", "see"); + * child.foo(); // works + * + * In addition, a superclass' implementation of a method can be invoked + * as follows: + * + * ChildClass.prototype.foo = function(a) { + * ChildClass.superClass_.foo.call(this, a); + * // other code + * }; + */ +Function.prototype.inherits = function(parentCtor) { + var tempCtor = function(){}; + tempCtor.prototype = parentCtor.prototype; + this.superClass_ = parentCtor.prototype; + this.prototype = new tempCtor(); +} diff --git a/toolkit/components/url-classifier/content/moz/observer.js b/toolkit/components/url-classifier/content/moz/observer.js new file mode 100644 index 000000000..a9d22ee21 --- /dev/null +++ b/toolkit/components/url-classifier/content/moz/observer.js @@ -0,0 +1,145 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +// A couple of classes to simplify creating observers. +// +// Example1: +// +// function doSomething() { ... } +// var observer = new G_ObserverWrapper(topic, doSomething); +// someObj.addObserver(topic, observer); +// +// Example2: +// +// function doSomething() { ... } +// new G_ObserverServiceObserver("profile-after-change", +// doSomething, +// true /* run only once */); + + +/** + * This class abstracts the admittedly simple boilerplate required of + * an nsIObserver. It saves you the trouble of implementing the + * indirection of your own observe() function. + * + * @param topic String containing the topic the observer will filter for + * + * @param observeFunction Reference to the function to call when the + * observer fires + * + * @constructor + */ +this.G_ObserverWrapper = function G_ObserverWrapper(topic, observeFunction) { + this.debugZone = "observer"; + this.topic_ = topic; + this.observeFunction_ = observeFunction; +} + +/** + * XPCOM + */ +G_ObserverWrapper.prototype.QueryInterface = function(iid) { + if (iid.equals(Ci.nsISupports) || iid.equals(Ci.nsIObserver)) + return this; + throw Components.results.NS_ERROR_NO_INTERFACE; +} + +/** + * Invoked by the thingy being observed + */ +G_ObserverWrapper.prototype.observe = function(subject, topic, data) { + if (topic == this.topic_) + this.observeFunction_(subject, topic, data); +} + + +/** + * This class abstracts the admittedly simple boilerplate required of + * observing an observerservice topic. It implements the indirection + * required, and automatically registers to hear the topic. + * + * @param topic String containing the topic the observer will filter for + * + * @param observeFunction Reference to the function to call when the + * observer fires + * + * @param opt_onlyOnce Boolean indicating if the observer should unregister + * after it has fired + * + * @constructor + */ +this.G_ObserverServiceObserver = +function G_ObserverServiceObserver(topic, observeFunction, opt_onlyOnce) { + this.debugZone = "observerserviceobserver"; + this.topic_ = topic; + this.observeFunction_ = observeFunction; + this.onlyOnce_ = !!opt_onlyOnce; + + this.observer_ = new G_ObserverWrapper(this.topic_, + BindToObject(this.observe_, this)); + this.observerService_ = Cc["@mozilla.org/observer-service;1"] + .getService(Ci.nsIObserverService); + this.observerService_.addObserver(this.observer_, this.topic_, false); +} + +/** + * Unregister the observer from the observerservice + */ +G_ObserverServiceObserver.prototype.unregister = function() { + this.observerService_.removeObserver(this.observer_, this.topic_); + this.observerService_ = null; +} + +/** + * Invoked by the observerservice + */ +G_ObserverServiceObserver.prototype.observe_ = function(subject, topic, data) { + this.observeFunction_(subject, topic, data); + if (this.onlyOnce_) + this.unregister(); +} + +#ifdef DEBUG +this.TEST_G_Observer = function TEST_G_Observer() { + if (G_GDEBUG) { + + var z = "observer UNITTEST"; + G_debugService.enableZone(z); + + G_Debug(z, "Starting"); + + var regularObserverRan = 0; + var observerServiceObserverRan = 0; + + let regularObserver = function () { + regularObserverRan++; + }; + + let observerServiceObserver = function () { + observerServiceObserverRan++; + }; + + var service = Cc["@mozilla.org/observer-service;1"] + .getService(Ci.nsIObserverService); + var topic = "google-observer-test"; + + var o1 = new G_ObserverWrapper(topic, regularObserver); + service.addObserver(o1, topic, false); + + new G_ObserverServiceObserver(topic, + observerServiceObserver, true /* once */); + + // Notifications happen synchronously, so this is easy + service.notifyObservers(null, topic, null); + service.notifyObservers(null, topic, null); + + G_Assert(z, regularObserverRan == 2, "Regular observer broken"); + G_Assert(z, observerServiceObserverRan == 1, "ObsServObs broken"); + + service.removeObserver(o1, topic); + G_Debug(z, "PASSED"); + } +} +#endif diff --git a/toolkit/components/url-classifier/content/moz/preferences.js b/toolkit/components/url-classifier/content/moz/preferences.js new file mode 100644 index 000000000..30105ab34 --- /dev/null +++ b/toolkit/components/url-classifier/content/moz/preferences.js @@ -0,0 +1,276 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +// Class for manipulating preferences. Aside from wrapping the pref +// service, useful functionality includes: +// +// - abstracting prefobserving so that you can observe preferences +// without implementing nsIObserver +// +// - getters that return a default value when the pref doesn't exist +// (instead of throwing) +// +// - get-and-set getters +// +// Example: +// +// var p = new PROT_Preferences(); +// dump(p.getPref("some-true-pref")); // shows true +// dump(p.getPref("no-such-pref", true)); // shows true +// dump(p.getPref("no-such-pref", null)); // shows null +// +// function observe(prefThatChanged) { +// dump("Pref changed: " + prefThatChanged); +// }; +// +// p.addObserver("somepref", observe); +// p.setPref("somepref", true); // dumps +// p.removeObserver("somepref", observe); +// +// TODO: should probably have the prefobserver pass in the new and old +// values + +// TODO(tc): Maybe remove this class and just call natively since we're no +// longer an extension. + +/** + * A class that wraps the preferences service. + * + * @param opt_startPoint A starting point on the prefs tree to resolve + * names passed to setPref and getPref. + * + * @param opt_useDefaultPranch Set to true to work against the default + * preferences tree instead of the profile one. + * + * @constructor + */ +this.G_Preferences = +function G_Preferences(opt_startPoint, opt_getDefaultBranch) { + this.debugZone = "prefs"; + this.observers_ = {}; + this.getDefaultBranch_ = !!opt_getDefaultBranch; + + this.startPoint_ = opt_startPoint || null; +} + +G_Preferences.setterMap_ = { "string": "setCharPref", + "boolean": "setBoolPref", + "number": "setIntPref" }; + +G_Preferences.getterMap_ = {}; +G_Preferences.getterMap_[Ci.nsIPrefBranch.PREF_STRING] = "getCharPref"; +G_Preferences.getterMap_[Ci.nsIPrefBranch.PREF_BOOL] = "getBoolPref"; +G_Preferences.getterMap_[Ci.nsIPrefBranch.PREF_INT] = "getIntPref"; + +G_Preferences.prototype.__defineGetter__('prefs_', function() { + var prefs; + var prefSvc = Cc["@mozilla.org/preferences-service;1"] + .getService(Ci.nsIPrefService); + + if (this.getDefaultBranch_) { + prefs = prefSvc.getDefaultBranch(this.startPoint_); + } else { + prefs = prefSvc.getBranch(this.startPoint_); + } + + // QI to prefs in case we want to add observers + prefs.QueryInterface(Ci.nsIPrefBranchInternal); + return prefs; +}); + +/** + * Stores a key/value in a user preference. Valid types for val are string, + * boolean, and number. Complex values are not yet supported (but feel free to + * add them!). + */ +G_Preferences.prototype.setPref = function(key, val) { + var datatype = typeof(val); + + if (datatype == "number" && (val % 1 != 0)) { + throw new Error("Cannot store non-integer numbers in preferences."); + } + + var meth = G_Preferences.setterMap_[datatype]; + + if (!meth) { + throw new Error("Pref datatype {" + datatype + "} not supported."); + } + + return this.prefs_[meth](key, val); +} + +/** + * Retrieves a user preference. Valid types for the value are the same as for + * setPref. If the preference is not found, opt_default will be returned + * instead. + */ +G_Preferences.prototype.getPref = function(key, opt_default) { + var type = this.prefs_.getPrefType(key); + + // zero means that the specified pref didn't exist + if (type == Ci.nsIPrefBranch.PREF_INVALID) { + return opt_default; + } + + var meth = G_Preferences.getterMap_[type]; + + if (!meth) { + throw new Error("Pref datatype {" + type + "} not supported."); + } + + // If a pref has been cleared, it will have a valid type but won't + // be gettable, so this will throw. + try { + return this.prefs_[meth](key); + } catch(e) { + return opt_default; + } +} + +/** + * Delete a preference. + * + * @param which Name of preference to obliterate + */ +G_Preferences.prototype.clearPref = function(which) { + try { + // This throws if the pref doesn't exist, which is fine because a + // nonexistent pref is cleared + this.prefs_.clearUserPref(which); + } catch(e) {} +} + +/** + * Add an observer for a given pref. + * + * @param which String containing the pref to listen to + * @param callback Function to be called when the pref changes. This + * function will receive a single argument, a string + * holding the preference name that changed + */ +G_Preferences.prototype.addObserver = function(which, callback) { + // Need to store the observer we create so we can eventually unregister it + if (!this.observers_[which]) + this.observers_[which] = { callbacks: [], observers: [] }; + + /* only add an observer if the callback hasn't been registered yet */ + if (this.observers_[which].callbacks.indexOf(callback) == -1) { + var observer = new G_PreferenceObserver(callback); + this.observers_[which].callbacks.push(callback); + this.observers_[which].observers.push(observer); + this.prefs_.addObserver(which, observer, false /* strong reference */); + } +} + +/** + * Remove an observer for a given pref. + * + * @param which String containing the pref to stop listening to + * @param callback Function to remove as an observer + */ +G_Preferences.prototype.removeObserver = function(which, callback) { + var ix = this.observers_[which].callbacks.indexOf(callback); + G_Assert(this, ix != -1, "Tried to unregister a nonexistent observer"); + this.observers_[which].callbacks.splice(ix, 1); + var observer = this.observers_[which].observers.splice(ix, 1)[0]; + this.prefs_.removeObserver(which, observer); +} + +/** + * Remove all preference observers registered through this object. + */ +G_Preferences.prototype.removeAllObservers = function() { + for (var which in this.observers_) { + for (var observer of this.observers_[which].observers) { + this.prefs_.removeObserver(which, observer); + } + } + this.observers_ = {}; +} + +/** + * Helper class that knows how to observe preference changes and + * invoke a callback when they do + * + * @constructor + * @param callback Function to call when the preference changes + */ +this.G_PreferenceObserver = +function G_PreferenceObserver(callback) { + this.debugZone = "prefobserver"; + this.callback_ = callback; +} + +/** + * Invoked by the pref system when a preference changes. Passes the + * message along to the callback. + * + * @param subject The nsIPrefBranch that changed + * @param topic String "nsPref:changed" (aka + * NS_PREFBRANCH_PREFCHANGE_OBSERVER_ID -- but where does it + * live???) + * @param data Name of the pref that changed + */ +G_PreferenceObserver.prototype.observe = function(subject, topic, data) { + G_Debug(this, "Observed pref change: " + data); + this.callback_(data); +} + +/** + * XPCOM cruft + * + * @param iid Interface id of the interface the caller wants + */ +G_PreferenceObserver.prototype.QueryInterface = function(iid) { + if (iid.equals(Ci.nsISupports) || + iid.equals(Ci.nsIObserver) || + iid.equals(Ci.nsISupportsWeakReference)) + return this; + throw Components.results.NS_ERROR_NO_INTERFACE; +} + +#ifdef DEBUG +// UNITTESTS +this.TEST_G_Preferences = function TEST_G_Preferences() { + if (G_GDEBUG) { + var z = "preferences UNITTEST"; + G_debugService.enableZone(z); + G_Debug(z, "Starting"); + + var p = new G_Preferences(); + + var testPref = "test-preferences-unittest"; + var noSuchPref = "test-preferences-unittest-aypabtu"; + + // Used to test observing + var observeCount = 0; + let observe = function (prefChanged) { + G_Assert(z, prefChanged == testPref, "observer broken"); + observeCount++; + }; + + // Test setting, getting, and observing + p.addObserver(testPref, observe); + p.setPref(testPref, true); + G_Assert(z, p.getPref(testPref), "get or set broken"); + G_Assert(z, observeCount == 1, "observer adding not working"); + + p.removeObserver(testPref, observe); + + p.setPref(testPref, false); + G_Assert(z, observeCount == 1, "observer removal not working"); + G_Assert(z, !p.getPref(testPref), "get broken"); + + // Remember to clean up the prefs we've set, and test removing prefs + // while we're at it + p.clearPref(noSuchPref); + G_Assert(z, !p.getPref(noSuchPref, false), "clear broken"); + + p.clearPref(testPref); + + G_Debug(z, "PASSED"); + } +} +#endif diff --git a/toolkit/components/url-classifier/content/moz/protocol4.js b/toolkit/components/url-classifier/content/moz/protocol4.js new file mode 100644 index 000000000..a75f6b531 --- /dev/null +++ b/toolkit/components/url-classifier/content/moz/protocol4.js @@ -0,0 +1,133 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +// A helper class that knows how to parse from and serialize to +// protocol4. This is a simple, historical format used by some Google +// interfaces, for example the Toolbar (i.e., ancient services). +// +// Protocol4 consists of a newline-separated sequence of name/value +// pairs (strings). Each line consists of the name, the value length, +// and the value itself, all separated by colons. Example: +// +// foo:6:barbaz\n +// fritz:33:issickofdynamicallytypedlanguages\n + + +/** + * This class knows how to serialize/deserialize maps to/from their + * protocol4 representation. + * + * @constructor + */ +this.G_Protocol4Parser = function G_Protocol4Parser() { + this.debugZone = "protocol4"; + + this.protocol4RegExp_ = new RegExp("([^:]+):\\d+:(.*)$"); + this.newlineRegExp_ = new RegExp("(\\r)?\\n"); +} + +/** + * Create a map from a protocol4 string. Silently skips invalid lines. + * + * @param text String holding the protocol4 representation + * + * @returns Object as an associative array with keys and values + * given in text. The empty object is returned if none + * are parsed. + */ +G_Protocol4Parser.prototype.parse = function(text) { + + var response = {}; + if (!text) + return response; + + // Responses are protocol4: (repeated) name:numcontentbytes:content\n + var lines = text.split(this.newlineRegExp_); + for (var i = 0; i < lines.length; i++) + if (this.protocol4RegExp_.exec(lines[i])) + response[RegExp.$1] = RegExp.$2; + + return response; +} + +/** + * Create a protocol4 string from a map (object). Throws an error on + * an invalid input. + * + * @param map Object as an associative array with keys and values + * given as strings. + * + * @returns text String holding the protocol4 representation + */ +G_Protocol4Parser.prototype.serialize = function(map) { + if (typeof map != "object") + throw new Error("map must be an object"); + + var text = ""; + for (var key in map) { + if (typeof map[key] != "string") + throw new Error("Keys and values must be strings"); + + text += key + ":" + map[key].length + ":" + map[key] + "\n"; + } + + return text; +} + +#ifdef DEBUG +/** + * Cheesey unittests + */ +this.TEST_G_Protocol4Parser = function TEST_G_Protocol4Parser() { + if (G_GDEBUG) { + var z = "protocol4 UNITTEST"; + G_debugService.enableZone(z); + + G_Debug(z, "Starting"); + + var p = new G_Protocol4Parser(); + + let isEmpty = function (map) { + for (var key in map) + return false; + return true; + }; + + G_Assert(z, isEmpty(p.parse(null)), "Parsing null broken"); + G_Assert(z, isEmpty(p.parse("")), "Parsing nothing broken"); + + var t = "foo:3:bar"; + G_Assert(z, p.parse(t)["foo"] === "bar", "Parsing one line broken"); + + t = "foo:3:bar\n"; + G_Assert(z, p.parse(t)["foo"] === "bar", "Parsing line with lf broken"); + + t = "foo:3:bar\r\n"; + G_Assert(z, p.parse(t)["foo"] === "bar", "Parsing with crlf broken"); + + + t = "foo:3:bar\nbar:3:baz\r\nbom:3:yaz\n"; + G_Assert(z, p.parse(t)["foo"] === "bar", "First in multiline"); + G_Assert(z, p.parse(t)["bar"] === "baz", "Second in multiline"); + G_Assert(z, p.parse(t)["bom"] === "yaz", "Third in multiline"); + G_Assert(z, p.parse(t)[""] === undefined, "Nonexistent in multiline"); + + // Test serialization + + var original = { + "1": "1", + "2": "2", + "foobar": "baz", + "hello there": "how are you?" , + }; + var deserialized = p.parse(p.serialize(original)); + for (var key in original) + G_Assert(z, original[key] === deserialized[key], + "Trouble (de)serializing " + key); + + G_Debug(z, "PASSED"); + } +} +#endif diff --git a/toolkit/components/url-classifier/content/multi-querier.js b/toolkit/components/url-classifier/content/multi-querier.js new file mode 100644 index 000000000..f79db8154 --- /dev/null +++ b/toolkit/components/url-classifier/content/multi-querier.js @@ -0,0 +1,137 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +/** + * This class helps us batch a series of async calls to the db. + * If any of the tokens is in the database, we fire callback with + * true as a param. If all the tokens are not in the database, + * we fire callback with false as a param. + * This is an "Abstract" base class. Subclasses need to supply + * the condition_ method. + * + * @param tokens Array of strings to lookup in the db + * @param tableName String name of the table + * @param callback Function callback function that takes true if the condition + * passes. + */ +this.MultiQuerier = +function MultiQuerier(tokens, tableName, callback) { + this.tokens_ = tokens; + this.tableName_ = tableName; + this.callback_ = callback; + this.dbservice_ = Cc["@mozilla.org/url-classifier/dbservice;1"] + .getService(Ci.nsIUrlClassifierDBService); + // We put the current token in this variable. + this.key_ = null; +} + +/** + * Run the remaining tokens against the db. + */ +MultiQuerier.prototype.run = function() { + if (this.tokens_.length == 0) { + this.callback_.handleEvent(false); + this.dbservice_ = null; + this.callback_ = null; + return; + } + + this.key_ = this.tokens_.pop(); + G_Debug(this, "Looking up " + this.key_ + " in " + this.tableName_); + this.dbservice_.exists(this.tableName_, this.key_, + BindToObject(this.result_, this)); +} + +/** + * Callback from the db. If the returned value passes the this.condition_ + * test, go ahead and call the main callback. + */ +MultiQuerier.prototype.result_ = function(value) { + if (this.condition_(value)) { + this.callback_.handleEvent(true) + this.dbservice_ = null; + this.callback_ = null; + } else { + this.run(); + } +} + +// Subclasses must override this. +MultiQuerier.prototype.condition_ = function(value) { + throw "MultiQuerier is an abstract base class"; +} + + +/** + * Concrete MultiQuerier that stops if the key exists in the db. + */ +this.ExistsMultiQuerier = +function ExistsMultiQuerier(tokens, tableName, callback) { + MultiQuerier.call(this, tokens, tableName, callback); + this.debugZone = "existsMultiQuerier"; +} + +ExistsMultiQuerier.inherits = function(parentCtor) { + var tempCtor = function(){}; + tempCtor.prototype = parentCtor.prototype; + this.superClass_ = parentCtor.prototype; + this.prototype = new tempCtor(); +} +ExistsMultiQuerier.inherits(MultiQuerier); + +ExistsMultiQuerier.prototype.condition_ = function(value) { + return value.length > 0; +} + + +/** + * Concrete MultiQuerier that looks up a key, decrypts it, then + * checks the the resulting regular expressions for a match. + * @param tokens Array of hosts + */ +this.EnchashMultiQuerier = +function EnchashMultiQuerier(tokens, tableName, callback, url) { + MultiQuerier.call(this, tokens, tableName, callback); + this.url_ = url; + this.enchashDecrypter_ = new PROT_EnchashDecrypter(); + this.debugZone = "enchashMultiQuerier"; +} + +EnchashMultiQuerier.inherits = function(parentCtor) { + var tempCtor = function(){}; + tempCtor.prototype = parentCtor.prototype; + this.superClass_ = parentCtor.prototype; + this.prototype = new tempCtor(); +} +EnchashMultiQuerier.inherits(MultiQuerier); + +EnchashMultiQuerier.prototype.run = function() { + if (this.tokens_.length == 0) { + this.callback_.handleEvent(false); + this.dbservice_ = null; + this.callback_ = null; + return; + } + var host = this.tokens_.pop(); + this.key_ = host; + var lookupKey = this.enchashDecrypter_.getLookupKey(host); + this.dbservice_.exists(this.tableName_, lookupKey, + BindToObject(this.result_, this)); +} + +EnchashMultiQuerier.prototype.condition_ = function(encryptedValue) { + if (encryptedValue.length > 0) { + // We have encrypted regular expressions for this host. Let's + // decrypt them and see if we have a match. + var decrypted = this.enchashDecrypter_.decryptData(encryptedValue, + this.key_); + var res = this.enchashDecrypter_.parseRegExps(decrypted); + for (var j = 0; j < res.length; j++) { + if (res[j].test(this.url_)) { + return true; + } + } + } + return false; +} diff --git a/toolkit/components/url-classifier/content/request-backoff.js b/toolkit/components/url-classifier/content/request-backoff.js new file mode 100644 index 000000000..17e815cf1 --- /dev/null +++ b/toolkit/components/url-classifier/content/request-backoff.js @@ -0,0 +1,116 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This implements logic for stopping requests if the server starts to return +// too many errors. If we get MAX_ERRORS errors in ERROR_PERIOD minutes, we +// back off for TIMEOUT_INCREMENT minutes. If we get another error +// immediately after we restart, we double the timeout and add +// TIMEOUT_INCREMENT minutes, etc. +// +// This is similar to the logic used by the search suggestion service. + +// HTTP responses that count as an error. We also include any 5xx response +// as an error. +this.HTTP_FOUND = 302; +this.HTTP_SEE_OTHER = 303; +this.HTTP_TEMPORARY_REDIRECT = 307; + +/** + * @param maxErrors Number of times to request before backing off. + * @param retryIncrement Time (ms) for each retry before backing off. + * @param maxRequests Number the number of requests needed to trigger backoff + * @param requestPeriod Number time (ms) in which maxRequests have to occur to + * trigger the backoff behavior (0 to disable maxRequests) + * @param timeoutIncrement Number time (ms) the starting timeout period + * we double this time for consecutive errors + * @param maxTimeout Number time (ms) maximum timeout period + */ +this.RequestBackoff = +function RequestBackoff(maxErrors, retryIncrement, + maxRequests, requestPeriod, + timeoutIncrement, maxTimeout) { + this.MAX_ERRORS_ = maxErrors; + this.RETRY_INCREMENT_ = retryIncrement; + this.MAX_REQUESTS_ = maxRequests; + this.REQUEST_PERIOD_ = requestPeriod; + this.TIMEOUT_INCREMENT_ = timeoutIncrement; + this.MAX_TIMEOUT_ = maxTimeout; + + // Queue of ints keeping the time of all requests + this.requestTimes_ = []; + + this.numErrors_ = 0; + this.errorTimeout_ = 0; + this.nextRequestTime_ = 0; +} + +/** + * Reset the object for reuse. This deliberately doesn't clear requestTimes_. + */ +RequestBackoff.prototype.reset = function() { + this.numErrors_ = 0; + this.errorTimeout_ = 0; + this.nextRequestTime_ = 0; +} + +/** + * Check to see if we can make a request. + */ +RequestBackoff.prototype.canMakeRequest = function() { + var now = Date.now(); + if (now < this.nextRequestTime_) { + return false; + } + + return (this.requestTimes_.length < this.MAX_REQUESTS_ || + (now - this.requestTimes_[0]) > this.REQUEST_PERIOD_); +} + +RequestBackoff.prototype.noteRequest = function() { + var now = Date.now(); + this.requestTimes_.push(now); + + // We only care about keeping track of MAX_REQUESTS + if (this.requestTimes_.length > this.MAX_REQUESTS_) + this.requestTimes_.shift(); +} + +RequestBackoff.prototype.nextRequestDelay = function() { + return Math.max(0, this.nextRequestTime_ - Date.now()); +} + +/** + * Notify this object of the last server response. If it's an error, + */ +RequestBackoff.prototype.noteServerResponse = function(status) { + if (this.isErrorStatus(status)) { + this.numErrors_++; + + if (this.numErrors_ < this.MAX_ERRORS_) + this.errorTimeout_ = this.RETRY_INCREMENT_; + else if (this.numErrors_ == this.MAX_ERRORS_) + this.errorTimeout_ = this.TIMEOUT_INCREMENT_; + else + this.errorTimeout_ *= 2; + + this.errorTimeout_ = Math.min(this.errorTimeout_, this.MAX_TIMEOUT_); + this.nextRequestTime_ = Date.now() + this.errorTimeout_; + } else { + // Reset error timeout, allow requests to go through. + this.reset(); + } +} + +/** + * We consider 302, 303, 307, 4xx, and 5xx http responses to be errors. + * @param status Number http status + * @return Boolean true if we consider this http status an error + */ +RequestBackoff.prototype.isErrorStatus = function(status) { + return ((400 <= status && status <= 599) || + HTTP_FOUND == status || + HTTP_SEE_OTHER == status || + HTTP_TEMPORARY_REDIRECT == status); +} + diff --git a/toolkit/components/url-classifier/content/trtable.js b/toolkit/components/url-classifier/content/trtable.js new file mode 100644 index 000000000..c58a80c9a --- /dev/null +++ b/toolkit/components/url-classifier/content/trtable.js @@ -0,0 +1,169 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// XXX: This should all be moved into the dbservice class so it happens +// in the background thread. + +/** + * Abstract base class for a lookup table. + * @construction + */ +this.UrlClassifierTable = function UrlClassifierTable() { + this.debugZone = "urlclassifier-table"; + this.name = ''; + this.needsUpdate = false; + this.enchashDecrypter_ = new PROT_EnchashDecrypter(); + this.wrappedJSObject = this; +} + +UrlClassifierTable.prototype.QueryInterface = function(iid) { + if (iid.equals(Components.interfaces.nsISupports) || + iid.equals(Components.interfaces.nsIUrlClassifierTable)) + return this; + + throw Components.results.NS_ERROR_NO_INTERFACE; +} + +/** + * Subclasses need to implement this method. + */ +UrlClassifierTable.prototype.exists = function(url, callback) { + throw Components.results.NS_ERROR_NOT_IMPLEMENTED; +} + +///////////////////////////////////////////////////////////////////// +// Url table implementation +this.UrlClassifierTableUrl = function UrlClassifierTableUrl() { + UrlClassifierTable.call(this); +} + +UrlClassifierTableUrl.inherits = function(parentCtor) { + var tempCtor = function(){}; + tempCtor.prototype = parentCtor.prototype; + this.superClass_ = parentCtor.prototype; + this.prototype = new tempCtor(); +} +UrlClassifierTableUrl.inherits(UrlClassifierTable); + +/** + * Look up a URL in a URL table + */ +UrlClassifierTableUrl.prototype.exists = function(url, callback) { + // nsIUrlClassifierUtils.canonicalizeURL is the old way of canonicalizing a + // URL. Unfortunately, it doesn't normalize numeric domains so alternate IP + // formats (hex, octal, etc) won't trigger a match. + // this.enchashDecrypter_.getCanonicalUrl does the right thing and + // normalizes a URL to 4 decimal numbers, but the update server may still be + // giving us encoded IP addresses. So to be safe, we check both cases. + var urlUtils = Cc["@mozilla.org/url-classifier/utils;1"] + .getService(Ci.nsIUrlClassifierUtils); + var oldCanonicalized = urlUtils.canonicalizeURL(url); + var canonicalized = this.enchashDecrypter_.getCanonicalUrl(url); + G_Debug(this, "Looking up: " + url + " (" + oldCanonicalized + " and " + + canonicalized + ")"); + (new ExistsMultiQuerier([oldCanonicalized, canonicalized], + this.name, + callback)).run(); +} + +///////////////////////////////////////////////////////////////////// +// Domain table implementation + +this.UrlClassifierTableDomain = function UrlClassifierTableDomain() { + UrlClassifierTable.call(this); + this.debugZone = "urlclassifier-table-domain"; + this.ioService_ = Cc["@mozilla.org/network/io-service;1"] + .getService(Ci.nsIIOService); +} + +UrlClassifierTableDomain.inherits = function(parentCtor) { + var tempCtor = function(){}; + tempCtor.prototype = parentCtor.prototype; + this.superClass_ = parentCtor.prototype; + this.prototype = new tempCtor(); +} +UrlClassifierTableDomain.inherits(UrlClassifierTable); + +/** + * Look up a URL in a domain table + * We also try to lookup domain + first path component (e.g., + * www.mozilla.org/products). + * + * @returns Boolean true if the url domain is in the table + */ +UrlClassifierTableDomain.prototype.exists = function(url, callback) { + var canonicalized = this.enchashDecrypter_.getCanonicalUrl(url); + var urlObj = this.ioService_.newURI(canonicalized, null, null); + var host = ''; + try { + host = urlObj.host; + } catch (e) { } + var hostComponents = host.split("."); + + // Try to get the path of the URL. Pseudo urls (like wyciwyg:) throw + // errors when trying to convert to an nsIURL so we wrap in a try/catch + // block. + var path = "" + try { + urlObj.QueryInterface(Ci.nsIURL); + path = urlObj.filePath; + } catch (e) { } + + var pathComponents = path.split("/"); + + // We don't have a good way map from hosts to domains, so we instead try + // each possibility. Could probably optimize to start at the second dot? + var possible = []; + for (var i = 0; i < hostComponents.length - 1; i++) { + host = hostComponents.slice(i).join("."); + possible.push(host); + + // The path starts with a "/", so we are interested in the second path + // component if it is available + if (pathComponents.length >= 2 && pathComponents[1].length > 0) { + host = host + "/" + pathComponents[1]; + possible.push(host); + } + } + + // Run the possible domains against the db. + (new ExistsMultiQuerier(possible, this.name, callback)).run(); +} + +///////////////////////////////////////////////////////////////////// +// Enchash table implementation + +this.UrlClassifierTableEnchash = function UrlClassifierTableEnchash() { + UrlClassifierTable.call(this); + this.debugZone = "urlclassifier-table-enchash"; +} + +UrlClassifierTableEnchash.inherits = function(parentCtor) { + var tempCtor = function(){}; + tempCtor.prototype = parentCtor.prototype; + this.superClass_ = parentCtor.prototype; + this.prototype = new tempCtor(); +} +UrlClassifierTableEnchash.inherits(UrlClassifierTable); + +/** + * Look up a URL in an enchashDB. We try all sub domains (up to MAX_DOTS). + */ +UrlClassifierTableEnchash.prototype.exists = function(url, callback) { + url = this.enchashDecrypter_.getCanonicalUrl(url); + var host = this.enchashDecrypter_.getCanonicalHost(url, + PROT_EnchashDecrypter.MAX_DOTS); + + var possible = []; + for (var i = 0; i < PROT_EnchashDecrypter.MAX_DOTS + 1; i++) { + possible.push(host); + + var index = host.indexOf("."); + if (index == -1) + break; + host = host.substring(index + 1); + } + // Run the possible domains against the db. + (new EnchashMultiQuerier(possible, this.name, callback, url)).run(); +} diff --git a/toolkit/components/url-classifier/content/wireformat.js b/toolkit/components/url-classifier/content/wireformat.js new file mode 100644 index 000000000..a24b120e6 --- /dev/null +++ b/toolkit/components/url-classifier/content/wireformat.js @@ -0,0 +1,230 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +// A class that serializes and deserializes opaque key/value string to +// string maps to/from maps (trtables). It knows how to create +// trtables from the serialized format, so it also understands +// meta-information like the name of the table and the table's +// version. See docs for the protocol description. +// +// TODO: wireformatreader: if you have multiple updates for one table +// in a call to deserialize, the later ones will be merged +// (all but the last will be ignored). To fix, merge instead +// of replace when you have an existing table, and only do so once. +// TODO must have blank line between successive types -- problem? +// TODO doesn't tolerate blank lines very well +// +// Maybe: These classes could use a LOT more cleanup, but it's not a +// priority at the moment. For example, the tablesData/Known +// maps should be combined into a single object, the parser +// for a given type should be separate from the version info, +// and there should be synchronous interfaces for testing. + + +/** + * A class that knows how to serialize and deserialize meta-information. + * This meta information is the table name and version number, and + * in its serialized form looks like the first line below: + * + * [name-of-table X.Y update?] + * ...key/value pairs to add or delete follow... + * <blank line ends the table> + * + * The X.Y is the version number and the optional "update" token means + * that the table is a differential from the curent table the extension + * has. Its absence means that this is a full, new table. + */ +this.PROT_VersionParser = +function PROT_VersionParser(type, opt_major, opt_minor, opt_requireMac) { + this.debugZone = "versionparser"; + this.type = type; + this.major = 0; + this.minor = 0; + + this.badHeader = false; + + // Should the wireformatreader compute a mac? + this.mac = false; + this.macval = ""; + this.macFailed = false; + this.requireMac = !!opt_requireMac; + + this.update = false; + this.needsUpdate = false; // used by ListManager to determine update policy + // Used by ListerManager to see if we have read data for this table from + // disk. Once we read a table from disk, we are not going to do so again + // but instead update remotely if necessary. + this.didRead = false; + if (opt_major) + this.major = parseInt(opt_major); + if (opt_minor) + this.minor = parseInt(opt_minor); +} + +/** Import the version information from another VersionParser + * @params version a version parser object + */ +PROT_VersionParser.prototype.ImportVersion = function(version) { + this.major = version.major; + this.minor = version.minor; + + this.mac = version.mac; + this.macFailed = version.macFailed; + this.macval = version.macval; + // Don't set requireMac, since we create vparsers from scratch and doesn't + // know about it +} + +/** + * Creates a string like [goog-white-black 1.1] from internal information + * + * @returns String + */ +PROT_VersionParser.prototype.toString = function() { + var s = "[" + this.type + " " + this.major + "." + this.minor + "]"; + return s; +} + +/** + * Creates a string like 1.123 with the version number. This is the + * format we store in prefs. + * @return String + */ +PROT_VersionParser.prototype.versionString = function() { + return this.major + "." + this.minor; +} + +/** + * Creates a string like 1:1 from internal information used for + * fetching updates from the server. Called by the listmanager. + * + * @returns String + */ +PROT_VersionParser.prototype.toUrl = function() { + return this.major + ":" + this.minor; +} + +/** + * Process the old format, [type major.minor [update]] + * + * @returns true if the string could be parsed, false otherwise + */ +PROT_VersionParser.prototype.processOldFormat_ = function(line) { + if (line[0] != '[' || line.slice(-1) != ']') + return false; + + var description = line.slice(1, -1); + + // Get the type name and version number of this table + var tokens = description.split(" "); + this.type = tokens[0]; + var majorminor = tokens[1].split("."); + this.major = parseInt(majorminor[0]); + this.minor = parseInt(majorminor[1]); + if (isNaN(this.major) || isNaN(this.minor)) + return false; + + if (tokens.length >= 3) { + this.update = tokens[2] == "update"; + } + + return true; +} + +/** + * Takes a string like [name-of-table 1.1 [update]][mac=MAC] and figures out the + * type and corresponding version numbers. + * @returns true if the string could be parsed, false otherwise + */ +PROT_VersionParser.prototype.fromString = function(line) { + G_Debug(this, "Calling fromString with line: " + line); + if (line[0] != '[' || line.slice(-1) != ']') + return false; + + // There could be two [][], so take care of it + var secondBracket = line.indexOf('[', 1); + var firstPart = null; + var secondPart = null; + + if (secondBracket != -1) { + firstPart = line.substring(0, secondBracket); + secondPart = line.substring(secondBracket); + G_Debug(this, "First part: " + firstPart + " Second part: " + secondPart); + } else { + firstPart = line; + G_Debug(this, "Old format: " + firstPart); + } + + if (!this.processOldFormat_(firstPart)) + return false; + + if (secondPart && !this.processOptTokens_(secondPart)) + return false; + + return true; +} + +/** + * Process optional tokens + * + * @param line A string [token1=val1 token2=val2...] + * @returns true if the string could be parsed, false otherwise + */ +PROT_VersionParser.prototype.processOptTokens_ = function(line) { + if (line[0] != '[' || line.slice(-1) != ']') + return false; + var description = line.slice(1, -1); + // Get the type name and version number of this table + var tokens = description.split(" "); + + for (var i = 0; i < tokens.length; i++) { + G_Debug(this, "Processing optional token: " + tokens[i]); + var tokenparts = tokens[i].split("="); + switch(tokenparts[0]){ + case "mac": + this.mac = true; + if (tokenparts.length < 2) { + G_Debug(this, "Found mac flag but not mac value!"); + return false; + } + // The mac value may have "=" in it, so we can't just use tokenparts[1]. + // Instead, just take the rest of tokens[i] after the first "=" + this.macval = tokens[i].substr(tokens[i].indexOf("=")+1); + break; + default: + G_Debug(this, "Found unrecognized token: " + tokenparts[0]); + break; + } + } + + return true; +} + +#ifdef DEBUG +this.TEST_PROT_WireFormat = function TEST_PROT_WireFormat() { + if (G_GDEBUG) { + var z = "versionparser UNITTEST"; + G_Debug(z, "Starting"); + + var vp = new PROT_VersionParser("dummy"); + G_Assert(z, vp.fromString("[foo-bar-url 1.234]"), + "failed to parse old format"); + G_Assert(z, "foo-bar-url" == vp.type, "failed to parse type"); + G_Assert(z, "1" == vp.major, "failed to parse major"); + G_Assert(z, "234" == vp.minor, "failed to parse minor"); + + vp = new PROT_VersionParser("dummy"); + G_Assert(z, vp.fromString("[foo-bar-url 1.234][mac=567]"), + "failed to parse new format"); + G_Assert(z, "foo-bar-url" == vp.type, "failed to parse type"); + G_Assert(z, "1" == vp.major, "failed to parse major"); + G_Assert(z, "234" == vp.minor, "failed to parse minor"); + G_Assert(z, true == vp.mac, "failed to parse mac"); + G_Assert(z, "567" == vp.macval, "failed to parse macval"); + + G_Debug(z, "PASSED"); + } +} +#endif diff --git a/toolkit/components/url-classifier/content/xml-fetcher.js b/toolkit/components/url-classifier/content/xml-fetcher.js new file mode 100644 index 000000000..39b116e00 --- /dev/null +++ b/toolkit/components/url-classifier/content/xml-fetcher.js @@ -0,0 +1,126 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// A simple class that encapsulates a request. You'll notice the +// style here is different from the rest of the extension; that's +// because this was re-used from really old code we had. At some +// point it might be nice to replace this with something better +// (e.g., something that has explicit onerror handler, ability +// to set headers, and so on). + +/** + * Because we might be in a component, we can't just assume that + * XMLHttpRequest exists. So we use this tiny factory function to wrap the + * XPCOM version. + * + * @return XMLHttpRequest object + */ +this.PROT_NewXMLHttpRequest = function PROT_NewXMLHttpRequest() { + var Cc = Components.classes; + var Ci = Components.interfaces; + var request = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"] + .createInstance(Ci.nsIXMLHttpRequest); + // Need the following so we get onerror/load/progresschange + request.QueryInterface(Ci.nsIJSXMLHttpRequest); + return request; +} + +/** + * A helper class that does HTTP GETs and calls back a function with + * the content it receives. Asynchronous, so uses a closure for the + * callback. + * + * Note, that XMLFetcher is only used for SafeBrowsing, therefore + * we inherit from nsILoadContext, so we can use the callbacks on the + * channel to separate the safebrowsing cookie based on a reserved + * appId. + * @constructor + */ +this.PROT_XMLFetcher = function PROT_XMLFetcher() { + this.debugZone = "xmlfetcher"; + this._request = PROT_NewXMLHttpRequest(); + // implements nsILoadContext + this.appId = Ci.nsIScriptSecurityManager.SAFEBROWSING_APP_ID; + this.isInIsolatedMozBrowserElement = false; + this.usePrivateBrowsing = false; + this.isContent = false; +} + +PROT_XMLFetcher.prototype = { + /** + * Function that will be called back upon fetch completion. + */ + _callback: null, + + + /** + * Fetches some content. + * + * @param page URL to fetch + * @param callback Function to call back when complete. + */ + get: function(page, callback) { + this._request.abort(); // abort() is asynchronous, so + this._request = PROT_NewXMLHttpRequest(); + this._callback = callback; + var asynchronous = true; + this._request.loadInfo.originAttributes = { + appId: this.appId, + inIsolatedMozBrowser: this.isInIsolatedMozBrowserElement + }; + this._request.open("GET", page, asynchronous); + this._request.channel.notificationCallbacks = this; + + // Create a closure + var self = this; + this._request.addEventListener("readystatechange", function() { + self.readyStateChange(self); + }, false); + + this._request.send(null); + }, + + cancel: function() { + this._request.abort(); + this._request = null; + }, + + /** + * Called periodically by the request to indicate some state change. 4 + * means content has been received. + */ + readyStateChange: function(fetcher) { + if (fetcher._request.readyState != 4) + return; + + // If the request fails, on trunk we get status set to + // NS_ERROR_NOT_AVAILABLE. On 1.8.1 branch we get an exception + // forwarded from nsIHttpChannel::GetResponseStatus. To be consistent + // between branch and trunk, we send back NS_ERROR_NOT_AVAILABLE for + // http failures. + var responseText = null; + var status = Components.results.NS_ERROR_NOT_AVAILABLE; + try { + G_Debug(this, "xml fetch status code: \"" + + fetcher._request.status + "\""); + status = fetcher._request.status; + responseText = fetcher._request.responseText; + } catch(e) { + G_Debug(this, "Caught exception trying to read xmlhttprequest " + + "status/response."); + G_Debug(this, e); + } + if (fetcher._callback) + fetcher._callback(responseText, status); + }, + + // nsIInterfaceRequestor + getInterface: function(iid) { + return this.QueryInterface(iid); + }, + + QueryInterface: XPCOMUtils.generateQI([Ci.nsIInterfaceRequestor, + Ci.nsISupports, + Ci.nsILoadContext]) +}; diff --git a/toolkit/components/url-classifier/moz.build b/toolkit/components/url-classifier/moz.build new file mode 100644 index 000000000..d8856ee4a --- /dev/null +++ b/toolkit/components/url-classifier/moz.build @@ -0,0 +1,86 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +TEST_DIRS += ['tests'] + +XPIDL_SOURCES += [ + 'nsIUrlClassifierDBService.idl', + 'nsIUrlClassifierHashCompleter.idl', + 'nsIUrlClassifierPrefixSet.idl', + 'nsIUrlClassifierStreamUpdater.idl', + 'nsIUrlClassifierUtils.idl', + 'nsIUrlListManager.idl', +] + +XPIDL_MODULE = 'url-classifier' + +# Disable RTTI in google protocol buffer +DEFINES['GOOGLE_PROTOBUF_NO_RTTI'] = True + +UNIFIED_SOURCES += [ + 'ChunkSet.cpp', + 'Classifier.cpp', + 'LookupCache.cpp', + 'LookupCacheV4.cpp', + 'nsCheckSummedOutputStream.cpp', + 'nsUrlClassifierDBService.cpp', + 'nsUrlClassifierProxies.cpp', + 'nsUrlClassifierUtils.cpp', + 'protobuf/safebrowsing.pb.cc', + 'ProtocolParser.cpp', + 'RiceDeltaDecoder.cpp', +] + +# define conflicting LOG() macros +SOURCES += [ + 'nsUrlClassifierPrefixSet.cpp', + 'nsUrlClassifierStreamUpdater.cpp', + 'VariableLengthPrefixSet.cpp', +] + +# contains variables that conflict with LookupCache.cpp +SOURCES += [ + 'HashStore.cpp', +] + +EXTRA_COMPONENTS += [ + 'nsURLClassifier.manifest', + 'nsUrlClassifierHashCompleter.js', +] + +# Same as JS components that are run through the pre-processor. +EXTRA_PP_COMPONENTS += [ + 'nsUrlClassifierLib.js', + 'nsUrlClassifierListManager.js', +] + +EXTRA_JS_MODULES += [ + 'SafeBrowsing.jsm', +] + +EXPORTS += [ + 'Entries.h', + 'LookupCache.h', + 'LookupCacheV4.h', + 'nsUrlClassifierPrefixSet.h', + 'protobuf/safebrowsing.pb.h', + 'VariableLengthPrefixSet.h', +] + +FINAL_LIBRARY = 'xul' + +LOCAL_INCLUDES += [ + '../build', + '/ipc/chromium/src', +] + +CXXFLAGS += CONFIG['SQLITE_CFLAGS'] + +if CONFIG['GNU_CXX']: + CXXFLAGS += ['-Wno-error=shadow'] + +if CONFIG['NIGHTLY_BUILD'] or CONFIG['MOZ_DEBUG']: + DEFINES['MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES'] = True diff --git a/toolkit/components/url-classifier/nsCheckSummedOutputStream.cpp b/toolkit/components/url-classifier/nsCheckSummedOutputStream.cpp new file mode 100644 index 000000000..68f9f1f6f --- /dev/null +++ b/toolkit/components/url-classifier/nsCheckSummedOutputStream.cpp @@ -0,0 +1,59 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsILocalFile.h" +#include "nsCRT.h" +#include "nsIFile.h" +#include "nsISupportsImpl.h" +#include "nsCheckSummedOutputStream.h" + +//////////////////////////////////////////////////////////////////////////////// +// nsCheckSummedOutputStream + +NS_IMPL_ISUPPORTS_INHERITED(nsCheckSummedOutputStream, + nsSafeFileOutputStream, + nsISafeOutputStream, + nsIOutputStream, + nsIFileOutputStream) + +NS_IMETHODIMP +nsCheckSummedOutputStream::Init(nsIFile* file, int32_t ioFlags, int32_t perm, + int32_t behaviorFlags) +{ + nsresult rv; + mHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mHash->Init(nsICryptoHash::MD5); + NS_ENSURE_SUCCESS(rv, rv); + + return nsSafeFileOutputStream::Init(file, ioFlags, perm, behaviorFlags); +} + +NS_IMETHODIMP +nsCheckSummedOutputStream::Finish() +{ + nsresult rv = mHash->Finish(false, mCheckSum); + NS_ENSURE_SUCCESS(rv, rv); + + uint32_t written; + rv = nsSafeFileOutputStream::Write(reinterpret_cast<const char*>(mCheckSum.BeginReading()), + mCheckSum.Length(), &written); + NS_ASSERTION(written == mCheckSum.Length(), "Error writing stream checksum"); + NS_ENSURE_SUCCESS(rv, rv); + + return nsSafeFileOutputStream::Finish(); +} + +NS_IMETHODIMP +nsCheckSummedOutputStream::Write(const char *buf, uint32_t count, uint32_t *result) +{ + nsresult rv = mHash->Update(reinterpret_cast<const uint8_t*>(buf), count); + NS_ENSURE_SUCCESS(rv, rv); + + return nsSafeFileOutputStream::Write(buf, count, result); +} + +//////////////////////////////////////////////////////////////////////////////// diff --git a/toolkit/components/url-classifier/nsCheckSummedOutputStream.h b/toolkit/components/url-classifier/nsCheckSummedOutputStream.h new file mode 100644 index 000000000..c2fe26b5f --- /dev/null +++ b/toolkit/components/url-classifier/nsCheckSummedOutputStream.h @@ -0,0 +1,55 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsCheckSummedOutputStream_h__ +#define nsCheckSummedOutputStream_h__ + +#include "nsILocalFile.h" +#include "nsIFile.h" +#include "nsIOutputStream.h" +#include "nsICryptoHash.h" +#include "nsNetCID.h" +#include "nsString.h" +#include "../../../netwerk/base/nsFileStreams.h" +#include "nsToolkitCompsCID.h" + +class nsCheckSummedOutputStream : public nsSafeFileOutputStream +{ +public: + NS_DECL_ISUPPORTS_INHERITED + + // Size of MD5 hash in bytes + static const uint32_t CHECKSUM_SIZE = 16; + + nsCheckSummedOutputStream() {} + + NS_IMETHOD Finish() override; + NS_IMETHOD Write(const char *buf, uint32_t count, uint32_t *result) override; + NS_IMETHOD Init(nsIFile* file, int32_t ioFlags, int32_t perm, int32_t behaviorFlags) override; + +protected: + virtual ~nsCheckSummedOutputStream() { nsSafeFileOutputStream::Close(); } + + nsCOMPtr<nsICryptoHash> mHash; + nsCString mCheckSum; +}; + +// returns a file output stream which can be QI'ed to nsIFileOutputStream. +inline nsresult +NS_NewCheckSummedOutputStream(nsIOutputStream **result, + nsIFile *file, + int32_t ioFlags = -1, + int32_t perm = -1, + int32_t behaviorFlags = 0) +{ + nsCOMPtr<nsIFileOutputStream> out = new nsCheckSummedOutputStream(); + nsresult rv = out->Init(file, ioFlags, perm, behaviorFlags); + if (NS_SUCCEEDED(rv)) { + out.forget(result); + } + return rv; +} + +#endif diff --git a/toolkit/components/url-classifier/nsIUrlClassifierDBService.idl b/toolkit/components/url-classifier/nsIUrlClassifierDBService.idl new file mode 100644 index 000000000..498d9717e --- /dev/null +++ b/toolkit/components/url-classifier/nsIUrlClassifierDBService.idl @@ -0,0 +1,232 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +%{C++ +#include "Entries.h" +#include "LookupCache.h" +class nsUrlClassifierLookupResult; +%} +[ptr] native ResultArray(nsTArray<mozilla::safebrowsing::LookupResult>); +[ptr] native CacheCompletionArray(nsTArray<mozilla::safebrowsing::CacheResult>); +[ptr] native PrefixArray(mozilla::safebrowsing::PrefixArray); + +interface nsIUrlClassifierHashCompleter; +interface nsIPrincipal; + +// Interface for JS function callbacks +[scriptable, function, uuid(4ca27b6b-a674-4b3d-ab30-d21e2da2dffb)] +interface nsIUrlClassifierCallback : nsISupports { + void handleEvent(in ACString value); +}; + +/** + * The nsIUrlClassifierUpdateObserver interface is implemented by + * clients streaming updates to the url-classifier (usually + * nsUrlClassifierStreamUpdater. + */ +[scriptable, uuid(9fa11561-5816-4e1b-bcc9-b629ca05cce6)] +interface nsIUrlClassifierUpdateObserver : nsISupports { + /** + * The update requested a new URL whose contents should be downloaded + * and sent to the classifier as a new stream. + * + * @param url The url that was requested. + * @param table The table name that this URL's contents will be associated + * with. This should be passed back to beginStream(). + */ + void updateUrlRequested(in ACString url, + in ACString table); + + /** + * A stream update has completed. + * + * @param status The state of the update process. + * @param delay The amount of time the updater should wait to fetch the + * next URL in ms. + */ + void streamFinished(in nsresult status, in unsigned long delay); + + /* The update has encountered an error and should be cancelled */ + void updateError(in nsresult error); + + /** + * The update has completed successfully. + * + * @param requestedTimeout The number of seconds that the caller should + * wait before trying to update again. + **/ + void updateSuccess(in unsigned long requestedTimeout); +}; + +/** + * This is a proxy class that is instantiated and called from the JS thread. + * It provides async methods for querying and updating the database. As the + * methods complete, they call the callback function. + */ +[scriptable, uuid(7a258022-6765-11e5-b379-b37b1f2354be)] +interface nsIUrlClassifierDBService : nsISupports +{ + /** + * Looks up a URI in the specified tables. + * + * @param principal: The principal containing the URI to search. + * @param c: The callback will be called with a comma-separated list + * of tables to which the key belongs. + */ + void lookup(in nsIPrincipal principal, + in ACString tables, + in nsIUrlClassifierCallback c); + + /** + * Lists the tables along with their meta info in the following format: + * + * tablename;[metadata]\n + * tablename2;[metadata]\n + * + * For v2 tables, the metadata is the chunks info such as + * + * goog-phish-shavar;a:10,14,30-40s:56,67 + * goog-unwanted-shavar;a:1-3,5 + * + * For v4 tables, base64 encoded state is currently the only info in the + * metadata (can be extended whenever necessary). For exmaple, + * + * goog-phish-proto;Cg0IARAGGAEiAzAwMTABEKqTARoCGAjT1gDD:oCGAjT1gDD\n + * goog-malware-proto;Cg0IAhAGGAEiAzAwMTABENCQARoCGAjx5Yty:BENCQARoCGAj\n + * + * Note that the metadata is colon-separated. + * + */ + void getTables(in nsIUrlClassifierCallback c); + + /** + * Set the nsIUrlClassifierCompleter object for a given table. This + * object will be used to request complete versions of partial + * hashes. + */ + void setHashCompleter(in ACString tableName, + in nsIUrlClassifierHashCompleter completer); + + /** + * Set the last update time for the given table. We use this to + * remember freshness past restarts. Time is in milliseconds since epoch. + */ + void setLastUpdateTime(in ACString tableName, + in unsigned long long lastUpdateTime); + + /** + * Forget the results that were used in the last DB update. + */ + void clearLastResults(); + + //////////////////////////////////////////////////////////////////////////// + // Incremental update methods. + // + // An update to the database has the following steps: + // + // 1) The update process is started with beginUpdate(). The client + // passes an nsIUrlClassifierUpdateObserver object which will be + // notified as the update is processed by the dbservice. + // 2) The client sends an initial update stream to the dbservice, + // using beginStream/updateStream/finishStream. + // 3) While reading this initial update stream, the dbservice may + // request additional streams from the client as requested by the + // update stream. + // 4) For each additional update stream, the client feeds the + // contents to the dbservice using beginStream/updateStream/endStream. + // 5) Once all streams have been processed, the client calls + // finishUpdate. When the dbservice has finished processing + // all streams, it will notify the observer that the update process + // is complete. + + /** + * Begin an update process. Will throw NS_ERROR_NOT_AVAILABLE if there + * is already an update in progress. + * + * @param updater The update observer tied to this update. + * @param tables A comma-separated list of tables included in this update. + */ + void beginUpdate(in nsIUrlClassifierUpdateObserver updater, + in ACString tables); + + /** + * Begin a stream update. This should be called once per url being + * fetched. + * + * @param table The table the contents of this stream will be associated + * with, or empty for the initial stream. + */ + void beginStream(in ACString table); + + /** + * Update the table incrementally. + */ + void updateStream(in ACString updateChunk); + + // It would be nice to have an updateFromStream method to round out the + // interface, but it's tricky because of XPCOM proxies. + + /** + * Finish an individual stream update. Must be called for every + * beginStream() call, before the next beginStream() or finishUpdate(). + * + * The update observer's streamFinished will be called once the + * stream has been processed. + */ + void finishStream(); + + /** + * Finish an incremental update. This will attempt to commit any + * pending changes and resets the update interface. + * + * The update observer's updateSucceeded or updateError methods + * will be called when the update has been processed. + */ + void finishUpdate(); + + /** + * Cancel an incremental update. This rolls back any pending changes. + * and resets the update interface. + * + * The update observer's updateError method will be called when the + * update has been rolled back. + */ + void cancelUpdate(); + + /** + * Reset the url-classifier database. This call will delete the existing + * database, emptying all tables. Mostly intended for use in unit tests. + */ + void resetDatabase(); + + /** + * Reload he url-classifier database. This will empty all cache for + * completions from gethash, and reload it from database. Mostly intended + * for use in tests. + */ + void reloadDatabase(); +}; + +/** + * This is an internal helper interface for communication between the + * main thread and the dbservice worker thread. It is called for each + * lookup to provide a set of possible results, which the main thread + * may need to expand using an nsIUrlClassifierCompleter. + */ +[uuid(b903dc8f-dff1-42fe-894b-36e7a59bb801)] +interface nsIUrlClassifierLookupCallback : nsISupports +{ + /** + * The lookup process is complete. + * + * @param results + * If this parameter is null, there were no results found. + * If not, it contains an array of nsUrlClassifierEntry objects + * with possible matches. The callee is responsible for freeing + * this array. + */ + void lookupComplete(in ResultArray results); +}; diff --git a/toolkit/components/url-classifier/nsIUrlClassifierHashCompleter.idl b/toolkit/components/url-classifier/nsIUrlClassifierHashCompleter.idl new file mode 100644 index 000000000..a3a8ab617 --- /dev/null +++ b/toolkit/components/url-classifier/nsIUrlClassifierHashCompleter.idl @@ -0,0 +1,65 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +/** + * This interface is implemented by nsIUrlClassifierHashCompleter clients. + */ +[scriptable, uuid(da16de40-df26-414d-bde7-c4faf4504868)] +interface nsIUrlClassifierHashCompleterCallback : nsISupports +{ + /** + * A complete hash has been found that matches the partial hash. + * This method may be called 0-n times for a given + * nsIUrlClassifierCompleter::complete() call. + * + * @param hash + * The 128-bit hash that was discovered. + * @param table + * The name of the table that this hash belongs to. + * @param chunkId + * The database chunk that this hash belongs to. + */ + void completion(in ACString hash, + in ACString table, + in uint32_t chunkId); + + /** + * The completion is complete. This method is called once per + * nsIUrlClassifierCompleter::complete() call, after all completion() + * calls are finished. + * + * @param status + * NS_OK if the request completed successfully, or an error code. + */ + void completionFinished(in nsresult status); +}; + +/** + * Clients updating the url-classifier database have the option of sending + * partial (32-bit) hashes of URL fragments to be blacklisted. If the + * url-classifier encounters one of these truncated hashes, it will ask an + * nsIUrlClassifierCompleter instance to asynchronously provide the complete + * hash, along with some associated metadata. + * This is only ever used for testing and should absolutely be deleted (I + * think). + */ +[scriptable, uuid(231fb2ad-ea8a-4e63-a331-eafc3b434811)] +interface nsIUrlClassifierHashCompleter : nsISupports +{ + /** + * Request a completed hash from the given gethash url. + * + * @param partialHash + * The 32-bit hash encountered by the url-classifier. + * @param gethashUrl + * The gethash url to use. + * @param callback + * An nsIUrlClassifierCompleterCallback instance. + */ + void complete(in ACString partialHash, + in ACString gethashUrl, + in nsIUrlClassifierHashCompleterCallback callback); +}; diff --git a/toolkit/components/url-classifier/nsIUrlClassifierPrefixSet.idl b/toolkit/components/url-classifier/nsIUrlClassifierPrefixSet.idl new file mode 100644 index 000000000..7e1a527d7 --- /dev/null +++ b/toolkit/components/url-classifier/nsIUrlClassifierPrefixSet.idl @@ -0,0 +1,29 @@ +/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" +#include "nsIFile.idl" + +// Note that the PrefixSet name is historical and we do properly support +// duplicated values, so it's really a Prefix Trie. +// All methods are thread-safe. +[scriptable, uuid(3d8579f0-75fa-4e00-ba41-38661d5b5d17)] +interface nsIUrlClassifierPrefixSet : nsISupports +{ + // Initialize the PrefixSet. Give it a name for memory reporting. + void init(in ACString aName); + // Fills the PrefixSet with the given array of prefixes. + // Can send an empty Array to clear the tree. + // Requires array to be sorted. + void setPrefixes([const, array, size_is(aLength)] in unsigned long aPrefixes, + in unsigned long aLength); + void getPrefixes(out unsigned long aCount, + [array, size_is(aCount), retval] out unsigned long aPrefixes); + // Do a lookup in the PrefixSet, return whether the value is present. + boolean contains(in unsigned long aPrefix); + boolean isEmpty(); + void loadFromFile(in nsIFile aFile); + void storeToFile(in nsIFile aFile); +}; diff --git a/toolkit/components/url-classifier/nsIUrlClassifierStreamUpdater.idl b/toolkit/components/url-classifier/nsIUrlClassifierStreamUpdater.idl new file mode 100644 index 000000000..50844d0e0 --- /dev/null +++ b/toolkit/components/url-classifier/nsIUrlClassifierStreamUpdater.idl @@ -0,0 +1,39 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" +#include "nsIUrlClassifierDBService.idl" + +/** + * This is a class to manage large table updates from the server. Rather than + * downloading the whole update and then updating the sqlite database, we + * update tables as the data is streaming in. + */ +[scriptable, uuid(e1797597-f4d6-4dd3-a1e1-745ad352cd80)] +interface nsIUrlClassifierStreamUpdater : nsISupports +{ + /** + * Try to download updates from updateUrl. If an update is already in + * progress, queues the requested update. This is used in nsIUrlListManager + * as well as in testing. + * @param aRequestTables Comma-separated list of tables included in this + * update. + * @param aRequestPayload The payload for the request. + * @param aIsPostRequest Whether the request should be sent by POST method. + * Should be 'true' for v2 usage. + * @param aUpdateUrl The plaintext url from which to request updates. + * @param aSuccessCallback Called after a successful update. + * @param aUpdateErrorCallback Called for problems applying the update + * @param aDownloadErrorCallback Called if we get an http error or a + * connection refused error. + */ + boolean downloadUpdates(in ACString aRequestTables, + in ACString aRequestPayload, + in boolean aIsPostRequest, + in ACString aUpdateUrl, + in nsIUrlClassifierCallback aSuccessCallback, + in nsIUrlClassifierCallback aUpdateErrorCallback, + in nsIUrlClassifierCallback aDownloadErrorCallback); +}; diff --git a/toolkit/components/url-classifier/nsIUrlClassifierTable.idl b/toolkit/components/url-classifier/nsIUrlClassifierTable.idl new file mode 100644 index 000000000..123069556 --- /dev/null +++ b/toolkit/components/url-classifier/nsIUrlClassifierTable.idl @@ -0,0 +1,31 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" +#include "nsIUrlListManager.idl" + +// A map that contains a string keys mapped to string values. + +[scriptable, uuid(fd1f8334-1859-472d-b01f-4ac6b1121ce4)] +interface nsIUrlClassifierTable : nsISupports +{ + /** + * The name used to identify this table + */ + attribute ACString name; + + /** + * Set to false if we don't want to update this table. + */ + attribute boolean needsUpdate; + + /** + * In the simple case, exists just looks up the string in the + * table and call the callback after the query returns with true or + * false. It's possible that something more complex happens + * (e.g., canonicalize the url). + */ + void exists(in ACString key, in nsIUrlListManagerCallback cb); +}; diff --git a/toolkit/components/url-classifier/nsIUrlClassifierUtils.idl b/toolkit/components/url-classifier/nsIUrlClassifierUtils.idl new file mode 100644 index 000000000..fa872ec27 --- /dev/null +++ b/toolkit/components/url-classifier/nsIUrlClassifierUtils.idl @@ -0,0 +1,74 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" +/** + * Some utility methods used by the url classifier. + */ + +interface nsIURI; + +[scriptable, uuid(e4f0e59c-b922-48b0-a7b6-1735c1f96fed)] +interface nsIUrlClassifierUtils : nsISupports +{ + /** + * Get the lookup string for a given URI. This normalizes the hostname, + * url-decodes the string, and strips off the protocol. + * + * @param uri URI to get the lookup key for. + * + * @returns String containing the canonicalized URI. + */ + ACString getKeyForURI(in nsIURI uri); + + /** + * Get the provider by table name. + * + * @param tableName The table name that we want to lookup + * + * @returns the provider name that the given table belongs. + */ + ACString getProvider(in ACString tableName); + + /** + * Get the protocol version for the given provider. + * + * @param provider String the provider name. e.g. "google" + * + * @returns String to indicate the protocol version. e.g. "2.2" + */ + ACString getProtocolVersion(in ACString provider); + + /** + * Convert threat type to list name. + * + * @param Integer to indicate threat type. + * + * @returns The list names separated by ','. For example, + * 'goog-phish-proto,test-phish-proto'. + */ + ACString convertThreatTypeToListNames(in uint32_t threatType); + + /** + * Convert list name to threat type. + * + * @param The list name. + * + * @returns The threat type in integer. + */ + uint32_t convertListNameToThreatType(in ACString listName); + + /** + * Make update request for given lists and their states. + * + * @param aListNames An array of list name represented in string. + * @param aState An array of states (encoded in base64 format) for each list. + * @param aCount The array length of aList and aState. + * + * @returns A base64url encoded string. + */ + ACString makeUpdateRequestV4([array, size_is(aCount)] in string aListNames, + [array, size_is(aCount)] in string aStatesBase64, + in uint32_t aCount); +}; diff --git a/toolkit/components/url-classifier/nsIUrlListManager.idl b/toolkit/components/url-classifier/nsIUrlListManager.idl new file mode 100644 index 000000000..112c567dc --- /dev/null +++ b/toolkit/components/url-classifier/nsIUrlListManager.idl @@ -0,0 +1,67 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIPrincipal; + +/** + * Interface for a class that manages updates of the url classifier database. + */ + +// Interface for JS function callbacks +[scriptable, function, uuid(fa4caf12-d057-4e7e-81e9-ce066ceee90b)] +interface nsIUrlListManagerCallback : nsISupports { + void handleEvent(in ACString value); +}; + + +[scriptable, uuid(d60a08ee-5c83-4eb6-bdfb-79fd0716501e)] +interface nsIUrlListManager : nsISupports +{ + /** + * Get the gethash url for this table + */ + ACString getGethashUrl(in ACString tableName); + + /** + * Add a table to the list of tables we are managing. The name is a + * string of the format provider_name-semantic_type-table_type. For + * @param tableName A string of the format + * provider_name-semantic_type-table_type. For example, + * goog-white-enchash or goog-black-url. + * @param providerName The name of the entity providing the list. + * @param updateUrl The URL from which to fetch updates. + * @param gethashUrl The URL from which to fetch hash completions. + */ + boolean registerTable(in ACString tableName, + in ACString providerName, + in ACString updateUrl, + in ACString gethashUrl); + + /** + * Turn on update checking for a table. I.e., during the next server + * check, download updates for this table. + */ + void enableUpdate(in ACString tableName); + + /** + * Turn off update checking for a table. + */ + void disableUpdate(in ACString tableName); + + /** + * Toggle update checking, if necessary. + */ + void maybeToggleUpdateChecking(); + + /** + * Lookup a key. Should not raise exceptions. Calls the callback + * function with a comma-separated list of tables to which the key + * belongs. + */ + void safeLookup(in nsIPrincipal key, + in nsIUrlListManagerCallback cb); +}; diff --git a/toolkit/components/url-classifier/nsURLClassifier.manifest b/toolkit/components/url-classifier/nsURLClassifier.manifest new file mode 100644 index 000000000..f035dea80 --- /dev/null +++ b/toolkit/components/url-classifier/nsURLClassifier.manifest @@ -0,0 +1,6 @@ +component {26a4a019-2827-4a89-a85c-5931a678823a} nsUrlClassifierLib.js +contract @mozilla.org/url-classifier/jslib;1 {26a4a019-2827-4a89-a85c-5931a678823a} +component {ca168834-cc00-48f9-b83c-fd018e58cae3} nsUrlClassifierListManager.js +contract @mozilla.org/url-classifier/listmanager;1 {ca168834-cc00-48f9-b83c-fd018e58cae3} +component {9111de73-9322-4bfc-8b65-2b727f3e6ec8} nsUrlClassifierHashCompleter.js +contract @mozilla.org/url-classifier/hashcompleter;1 {9111de73-9322-4bfc-8b65-2b727f3e6ec8} diff --git a/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp new file mode 100644 index 000000000..2ad8b6b51 --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp @@ -0,0 +1,1866 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsAutoPtr.h" +#include "nsCOMPtr.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsCRT.h" +#include "nsICryptoHash.h" +#include "nsICryptoHMAC.h" +#include "nsIDirectoryService.h" +#include "nsIKeyModule.h" +#include "nsIObserverService.h" +#include "nsIPermissionManager.h" +#include "nsIPrefBranch.h" +#include "nsIPrefService.h" +#include "nsIProperties.h" +#include "nsToolkitCompsCID.h" +#include "nsIUrlClassifierUtils.h" +#include "nsIXULRuntime.h" +#include "nsUrlClassifierDBService.h" +#include "nsUrlClassifierUtils.h" +#include "nsUrlClassifierProxies.h" +#include "nsURILoader.h" +#include "nsString.h" +#include "nsReadableUtils.h" +#include "nsTArray.h" +#include "nsNetCID.h" +#include "nsThreadUtils.h" +#include "nsXPCOMStrings.h" +#include "nsProxyRelease.h" +#include "nsString.h" +#include "mozilla/Atomics.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/ErrorNames.h" +#include "mozilla/Mutex.h" +#include "mozilla/Preferences.h" +#include "mozilla/TimeStamp.h" +#include "mozilla/Telemetry.h" +#include "mozilla/Logging.h" +#include "prprf.h" +#include "prnetdb.h" +#include "Entries.h" +#include "HashStore.h" +#include "Classifier.h" +#include "ProtocolParser.h" +#include "mozilla/Attributes.h" +#include "nsIPrincipal.h" +#include "Classifier.h" +#include "ProtocolParser.h" +#include "nsContentUtils.h" + +namespace mozilla { +namespace safebrowsing { + +nsresult +TablesToResponse(const nsACString& tables) +{ + if (tables.IsEmpty()) { + return NS_OK; + } + + // We don't check mCheckMalware and friends because BuildTables never + // includes a table that is not enabled. + if (FindInReadable(NS_LITERAL_CSTRING("-malware-"), tables)) { + return NS_ERROR_MALWARE_URI; + } + if (FindInReadable(NS_LITERAL_CSTRING("-phish-"), tables)) { + return NS_ERROR_PHISHING_URI; + } + if (FindInReadable(NS_LITERAL_CSTRING("-unwanted-"), tables)) { + return NS_ERROR_UNWANTED_URI; + } + if (FindInReadable(NS_LITERAL_CSTRING("-track-"), tables)) { + return NS_ERROR_TRACKING_URI; + } + if (FindInReadable(NS_LITERAL_CSTRING("-block-"), tables)) { + return NS_ERROR_BLOCKED_URI; + } + return NS_OK; +} + +} // namespace safebrowsing +} // namespace mozilla + +using namespace mozilla; +using namespace mozilla::safebrowsing; + +// MOZ_LOG=UrlClassifierDbService:5 +LazyLogModule gUrlClassifierDbServiceLog("UrlClassifierDbService"); +#define LOG(args) MOZ_LOG(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug, args) +#define LOG_ENABLED() MOZ_LOG_TEST(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug) + +// Prefs for implementing nsIURIClassifier to block page loads +#define CHECK_MALWARE_PREF "browser.safebrowsing.malware.enabled" +#define CHECK_MALWARE_DEFAULT false + +#define CHECK_PHISHING_PREF "browser.safebrowsing.phishing.enabled" +#define CHECK_PHISHING_DEFAULT false + +#define CHECK_TRACKING_PREF "privacy.trackingprotection.enabled" +#define CHECK_TRACKING_DEFAULT false + +#define CHECK_TRACKING_PB_PREF "privacy.trackingprotection.pbmode.enabled" +#define CHECK_TRACKING_PB_DEFAULT false + +#define CHECK_BLOCKED_PREF "browser.safebrowsing.blockedURIs.enabled" +#define CHECK_BLOCKED_DEFAULT false + +#define GETHASH_NOISE_PREF "urlclassifier.gethashnoise" +#define GETHASH_NOISE_DEFAULT 4 + +// Comma-separated lists +#define MALWARE_TABLE_PREF "urlclassifier.malwareTable" +#define PHISH_TABLE_PREF "urlclassifier.phishTable" +#define TRACKING_TABLE_PREF "urlclassifier.trackingTable" +#define TRACKING_WHITELIST_TABLE_PREF "urlclassifier.trackingWhitelistTable" +#define BLOCKED_TABLE_PREF "urlclassifier.blockedTable" +#define DOWNLOAD_BLOCK_TABLE_PREF "urlclassifier.downloadBlockTable" +#define DOWNLOAD_ALLOW_TABLE_PREF "urlclassifier.downloadAllowTable" +#define DISALLOW_COMPLETION_TABLE_PREF "urlclassifier.disallow_completions" + +#define CONFIRM_AGE_PREF "urlclassifier.max-complete-age" +#define CONFIRM_AGE_DEFAULT_SEC (45 * 60) + +class nsUrlClassifierDBServiceWorker; + +// Singleton instance. +static nsUrlClassifierDBService* sUrlClassifierDBService; + +nsIThread* nsUrlClassifierDBService::gDbBackgroundThread = nullptr; + +// Once we've committed to shutting down, don't do work in the background +// thread. +static bool gShuttingDownThread = false; + +static mozilla::Atomic<int32_t> gFreshnessGuarantee(CONFIRM_AGE_DEFAULT_SEC); + +NS_IMPL_ISUPPORTS(nsUrlClassifierDBServiceWorker, + nsIUrlClassifierDBService) + +nsUrlClassifierDBServiceWorker::nsUrlClassifierDBServiceWorker() + : mInStream(false) + , mGethashNoise(0) + , mPendingLookupLock("nsUrlClassifierDBServerWorker.mPendingLookupLock") +{ +} + +nsUrlClassifierDBServiceWorker::~nsUrlClassifierDBServiceWorker() +{ + NS_ASSERTION(!mClassifier, + "Db connection not closed, leaking memory! Call CloseDb " + "to close the connection."); +} + +nsresult +nsUrlClassifierDBServiceWorker::Init(uint32_t aGethashNoise, + nsCOMPtr<nsIFile> aCacheDir) +{ + mGethashNoise = aGethashNoise; + mCacheDir = aCacheDir; + + ResetUpdate(); + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::QueueLookup(const nsACString& spec, + const nsACString& tables, + nsIUrlClassifierLookupCallback* callback) +{ + MutexAutoLock lock(mPendingLookupLock); + + PendingLookup* lookup = mPendingLookups.AppendElement(); + if (!lookup) return NS_ERROR_OUT_OF_MEMORY; + + lookup->mStartTime = TimeStamp::Now(); + lookup->mKey = spec; + lookup->mCallback = callback; + lookup->mTables = tables; + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::DoLocalLookup(const nsACString& spec, + const nsACString& tables, + LookupResultArray* results) +{ + if (gShuttingDownThread) { + return NS_ERROR_ABORT; + } + + MOZ_ASSERT(!NS_IsMainThread(), "DoLocalLookup must be on background thread"); + if (!results) { + return NS_ERROR_FAILURE; + } + // Bail if we haven't been initialized on the background thread. + if (!mClassifier) { + return NS_ERROR_NOT_AVAILABLE; + } + + // We ignore failures from Check because we'd rather return the + // results that were found than fail. + mClassifier->Check(spec, tables, gFreshnessGuarantee, *results); + + LOG(("Found %d results.", results->Length())); + return NS_OK; +} + +static nsCString +ProcessLookupResults(LookupResultArray* results) +{ + // Build a stringified list of result tables. + nsTArray<nsCString> tables; + for (uint32_t i = 0; i < results->Length(); i++) { + LookupResult& result = results->ElementAt(i); + MOZ_ASSERT(!result.mNoise, "Lookup results should not have noise added"); + LOG(("Found result from table %s", result.mTableName.get())); + if (tables.IndexOf(result.mTableName) == nsTArray<nsCString>::NoIndex) { + tables.AppendElement(result.mTableName); + } + } + nsAutoCString tableStr; + for (uint32_t i = 0; i < tables.Length(); i++) { + if (i != 0) + tableStr.Append(','); + tableStr.Append(tables[i]); + } + return tableStr; +} + +/** + * Lookup up a key in the database is a two step process: + * + * a) First we look for any Entries in the database that might apply to this + * url. For each URL there are one or two possible domain names to check: + * the two-part domain name (example.com) and the three-part name + * (www.example.com). We check the database for both of these. + * b) If we find any entries, we check the list of fragments for that entry + * against the possible subfragments of the URL as described in the + * "Simplified Regular Expression Lookup" section of the protocol doc. + */ +nsresult +nsUrlClassifierDBServiceWorker::DoLookup(const nsACString& spec, + const nsACString& tables, + nsIUrlClassifierLookupCallback* c) +{ + if (gShuttingDownThread) { + c->LookupComplete(nullptr); + return NS_ERROR_NOT_INITIALIZED; + } + + PRIntervalTime clockStart = 0; + if (LOG_ENABLED()) { + clockStart = PR_IntervalNow(); + } + + nsAutoPtr<LookupResultArray> results(new LookupResultArray()); + if (!results) { + c->LookupComplete(nullptr); + return NS_ERROR_OUT_OF_MEMORY; + } + + nsresult rv = DoLocalLookup(spec, tables, results); + if (NS_FAILED(rv)) { + c->LookupComplete(nullptr); + return rv; + } + + LOG(("Found %d results.", results->Length())); + + + if (LOG_ENABLED()) { + PRIntervalTime clockEnd = PR_IntervalNow(); + LOG(("query took %dms\n", + PR_IntervalToMilliseconds(clockEnd - clockStart))); + } + + nsAutoPtr<LookupResultArray> completes(new LookupResultArray()); + + for (uint32_t i = 0; i < results->Length(); i++) { + if (!mMissCache.Contains(results->ElementAt(i).hash.prefix)) { + completes->AppendElement(results->ElementAt(i)); + } + } + + for (uint32_t i = 0; i < completes->Length(); i++) { + if (!completes->ElementAt(i).Confirmed()) { + // We're going to be doing a gethash request, add some extra entries. + // Note that we cannot pass the first two by reference, because we + // add to completes, whicah can cause completes to reallocate and move. + AddNoise(completes->ElementAt(i).hash.prefix, + completes->ElementAt(i).mTableName, + mGethashNoise, *completes); + break; + } + } + + // At this point ownership of 'results' is handed to the callback. + c->LookupComplete(completes.forget()); + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::HandlePendingLookups() +{ + if (gShuttingDownThread) { + return NS_ERROR_ABORT; + } + + MutexAutoLock lock(mPendingLookupLock); + while (mPendingLookups.Length() > 0) { + PendingLookup lookup = mPendingLookups[0]; + mPendingLookups.RemoveElementAt(0); + { + MutexAutoUnlock unlock(mPendingLookupLock); + DoLookup(lookup.mKey, lookup.mTables, lookup.mCallback); + } + double lookupTime = (TimeStamp::Now() - lookup.mStartTime).ToMilliseconds(); + Telemetry::Accumulate(Telemetry::URLCLASSIFIER_LOOKUP_TIME, + static_cast<uint32_t>(lookupTime)); + } + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::AddNoise(const Prefix aPrefix, + const nsCString tableName, + uint32_t aCount, + LookupResultArray& results) +{ + if (gShuttingDownThread) { + return NS_ERROR_ABORT; + } + + if (aCount < 1) { + return NS_OK; + } + + PrefixArray noiseEntries; + nsresult rv = mClassifier->ReadNoiseEntries(aPrefix, tableName, + aCount, &noiseEntries); + NS_ENSURE_SUCCESS(rv, rv); + + for (uint32_t i = 0; i < noiseEntries.Length(); i++) { + LookupResult *result = results.AppendElement(); + if (!result) + return NS_ERROR_OUT_OF_MEMORY; + + result->hash.prefix = noiseEntries[i]; + result->mNoise = true; + + result->mTableName.Assign(tableName); + } + + return NS_OK; +} + +// Lookup a key in the db. +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::Lookup(nsIPrincipal* aPrincipal, + const nsACString& aTables, + nsIUrlClassifierCallback* c) +{ + if (gShuttingDownThread) { + return NS_ERROR_ABORT; + } + + return HandlePendingLookups(); +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::GetTables(nsIUrlClassifierCallback* c) +{ + if (gShuttingDownThread) { + return NS_ERROR_NOT_INITIALIZED; + } + + nsresult rv = OpenDb(); + if (NS_FAILED(rv)) { + NS_ERROR("Unable to open SafeBrowsing database"); + return NS_ERROR_FAILURE; + } + + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString response; + mClassifier->TableRequest(response); + LOG(("GetTables: %s", response.get())); + c->HandleEvent(response); + + return rv; +} + +void +nsUrlClassifierDBServiceWorker::ResetStream() +{ + LOG(("ResetStream")); + mInStream = false; + mProtocolParser = nullptr; +} + +void +nsUrlClassifierDBServiceWorker::ResetUpdate() +{ + LOG(("ResetUpdate")); + mUpdateWaitSec = 0; + mUpdateStatus = NS_OK; + mUpdateObserver = nullptr; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::SetHashCompleter(const nsACString &tableName, + nsIUrlClassifierHashCompleter *completer) +{ + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::BeginUpdate(nsIUrlClassifierUpdateObserver *observer, + const nsACString &tables) +{ + LOG(("nsUrlClassifierDBServiceWorker::BeginUpdate [%s]", PromiseFlatCString(tables).get())); + + if (gShuttingDownThread) { + return NS_ERROR_NOT_INITIALIZED; + } + + NS_ENSURE_STATE(!mUpdateObserver); + + nsresult rv = OpenDb(); + if (NS_FAILED(rv)) { + NS_ERROR("Unable to open SafeBrowsing database"); + return NS_ERROR_FAILURE; + } + + mUpdateStatus = NS_OK; + mUpdateObserver = observer; + Classifier::SplitTables(tables, mUpdateTables); + + return NS_OK; +} + +// Called from the stream updater. +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::BeginStream(const nsACString &table) +{ + LOG(("nsUrlClassifierDBServiceWorker::BeginStream")); + MOZ_ASSERT(!NS_IsMainThread(), "Streaming must be on the background thread"); + + if (gShuttingDownThread) { + return NS_ERROR_NOT_INITIALIZED; + } + + NS_ENSURE_STATE(mUpdateObserver); + NS_ENSURE_STATE(!mInStream); + + mInStream = true; + + NS_ASSERTION(!mProtocolParser, "Should not have a protocol parser."); + + // Check if we should use protobuf to parse the update. + bool useProtobuf = false; + for (size_t i = 0; i < mUpdateTables.Length(); i++) { + bool isCurProtobuf = + StringEndsWith(mUpdateTables[i], NS_LITERAL_CSTRING("-proto")); + + if (0 == i) { + // Use the first table name to decice if all the subsequent tables + // should be '-proto'. + useProtobuf = isCurProtobuf; + continue; + } + + if (useProtobuf != isCurProtobuf) { + NS_WARNING("Cannot mix 'proto' tables with other types " + "within the same provider."); + break; + } + } + + mProtocolParser = (useProtobuf ? static_cast<ProtocolParser*>(new ProtocolParserProtobuf()) + : static_cast<ProtocolParser*>(new ProtocolParserV2())); + if (!mProtocolParser) + return NS_ERROR_OUT_OF_MEMORY; + + mProtocolParser->Init(mCryptoHash); + + if (!table.IsEmpty()) { + mProtocolParser->SetCurrentTable(table); + } + + mProtocolParser->SetRequestedTables(mUpdateTables); + + return NS_OK; +} + +/** + * Updating the database: + * + * The Update() method takes a series of chunks separated with control data, + * as described in + * http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec + * + * It will iterate through the control data until it reaches a chunk. By + * the time it reaches a chunk, it should have received + * a) the table to which this chunk applies + * b) the type of chunk (add, delete, expire add, expire delete). + * c) the chunk ID + * d) the length of the chunk. + * + * For add and subtract chunks, it needs to read the chunk data (expires + * don't have any data). Chunk data is a list of URI fragments whose + * encoding depends on the type of table (which is indicated by the end + * of the table name): + * a) tables ending with -exp are a zlib-compressed list of URI fragments + * separated by newlines. + * b) tables ending with -sha128 have the form + * [domain][N][frag0]...[fragN] + * 16 1 16 16 + * If N is 0, the domain is reused as a fragment. + * c) any other tables are assumed to be a plaintext list of URI fragments + * separated by newlines. + * + * Update() can be fed partial data; It will accumulate data until there is + * enough to act on. Finish() should be called when there will be no more + * data. + */ +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::UpdateStream(const nsACString& chunk) +{ + if (gShuttingDownThread) { + return NS_ERROR_NOT_INITIALIZED; + } + + NS_ENSURE_STATE(mInStream); + + HandlePendingLookups(); + + // Feed the chunk to the parser. + return mProtocolParser->AppendStream(chunk); +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::FinishStream() +{ + if (gShuttingDownThread) { + LOG(("shutting down")); + return NS_ERROR_NOT_INITIALIZED; + } + + NS_ENSURE_STATE(mInStream); + NS_ENSURE_STATE(mUpdateObserver); + + mInStream = false; + + mProtocolParser->End(); + + if (NS_SUCCEEDED(mProtocolParser->Status())) { + if (mProtocolParser->UpdateWaitSec()) { + mUpdateWaitSec = mProtocolParser->UpdateWaitSec(); + } + // XXX: Only allow forwards from the initial update? + const nsTArray<ProtocolParser::ForwardedUpdate> &forwards = + mProtocolParser->Forwards(); + for (uint32_t i = 0; i < forwards.Length(); i++) { + const ProtocolParser::ForwardedUpdate &forward = forwards[i]; + mUpdateObserver->UpdateUrlRequested(forward.url, forward.table); + } + // Hold on to any TableUpdate objects that were created by the + // parser. + mTableUpdates.AppendElements(mProtocolParser->GetTableUpdates()); + mProtocolParser->ForgetTableUpdates(); + +#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES + // The assignment involves no string copy since the source string is sharable. + mRawTableUpdates = mProtocolParser->GetRawTableUpdates(); +#endif + } else { + LOG(("nsUrlClassifierDBService::FinishStream Failed to parse the stream " + "using mProtocolParser.")); + mUpdateStatus = mProtocolParser->Status(); + } + mUpdateObserver->StreamFinished(mProtocolParser->Status(), 0); + + if (NS_SUCCEEDED(mUpdateStatus)) { + if (mProtocolParser->ResetRequested()) { + mClassifier->ResetTables(Classifier::Clear_All, mUpdateTables); + } + } + + mProtocolParser = nullptr; + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::FinishUpdate() +{ + if (gShuttingDownThread) { + return NS_ERROR_NOT_INITIALIZED; + } + + NS_ENSURE_STATE(mUpdateObserver); + + if (NS_SUCCEEDED(mUpdateStatus)) { + mUpdateStatus = ApplyUpdate(); + } else { + LOG(("nsUrlClassifierDBServiceWorker::FinishUpdate() Not running " + "ApplyUpdate() since the update has already failed.")); + } + + mMissCache.Clear(); + + if (NS_SUCCEEDED(mUpdateStatus)) { + LOG(("Notifying success: %d", mUpdateWaitSec)); + mUpdateObserver->UpdateSuccess(mUpdateWaitSec); + } else if (NS_ERROR_NOT_IMPLEMENTED == mUpdateStatus) { + LOG(("Treating NS_ERROR_NOT_IMPLEMENTED a successful update " + "but still mark it spoiled.")); + mUpdateObserver->UpdateSuccess(0); + mClassifier->ResetTables(Classifier::Clear_Cache, mUpdateTables); + } else { + if (LOG_ENABLED()) { + nsAutoCString errorName; + mozilla::GetErrorName(mUpdateStatus, errorName); + LOG(("Notifying error: %s (%d)", errorName.get(), mUpdateStatus)); + } + + mUpdateObserver->UpdateError(mUpdateStatus); + /* + * mark the tables as spoiled(clear cache in LookupCache), we don't want to + * block hosts longer than normal because our update failed + */ + mClassifier->ResetTables(Classifier::Clear_Cache, mUpdateTables); + } + mUpdateObserver = nullptr; + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::ApplyUpdate() +{ + LOG(("nsUrlClassifierDBServiceWorker::ApplyUpdate()")); + nsresult rv = mClassifier->ApplyUpdates(&mTableUpdates); + +#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES + if (NS_FAILED(rv) && NS_ERROR_OUT_OF_MEMORY != rv) { + mClassifier->DumpRawTableUpdates(mRawTableUpdates); + } + // Invalidate the raw table updates. + mRawTableUpdates = EmptyCString(); +#endif + + return rv; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::ResetDatabase() +{ + nsresult rv = OpenDb(); + + if (NS_SUCCEEDED(rv)) { + mClassifier->Reset(); + } + + rv = CloseDb(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::ReloadDatabase() +{ + nsTArray<nsCString> tables; + nsTArray<int64_t> lastUpdateTimes; + nsresult rv = mClassifier->ActiveTables(tables); + NS_ENSURE_SUCCESS(rv, rv); + + // We need to make sure lastupdatetime is set after reload database + // Otherwise request will be skipped if it is not confirmed. + for (uint32_t table = 0; table < tables.Length(); table++) { + lastUpdateTimes.AppendElement(mClassifier->GetLastUpdateTime(tables[table])); + } + + // This will null out mClassifier + rv = CloseDb(); + NS_ENSURE_SUCCESS(rv, rv); + + // Create new mClassifier and load prefixset and completions from disk. + rv = OpenDb(); + NS_ENSURE_SUCCESS(rv, rv); + + for (uint32_t table = 0; table < tables.Length(); table++) { + int64_t time = lastUpdateTimes[table]; + if (time) { + mClassifier->SetLastUpdateTime(tables[table], lastUpdateTimes[table]); + } + } + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::CancelUpdate() +{ + LOG(("nsUrlClassifierDBServiceWorker::CancelUpdate")); + + if (mUpdateObserver) { + LOG(("UpdateObserver exists, cancelling")); + + mUpdateStatus = NS_BINDING_ABORTED; + + mUpdateObserver->UpdateError(mUpdateStatus); + + /* + * mark the tables as spoiled(clear cache in LookupCache), we don't want to + * block hosts longer than normal because our update failed + */ + mClassifier->ResetTables(Classifier::Clear_Cache, mUpdateTables); + + ResetStream(); + ResetUpdate(); + } else { + LOG(("No UpdateObserver, nothing to cancel")); + } + + return NS_OK; +} + +// Allows the main thread to delete the connection which may be in +// a background thread. +// XXX This could be turned into a single shutdown event so the logic +// is simpler in nsUrlClassifierDBService::Shutdown. +nsresult +nsUrlClassifierDBServiceWorker::CloseDb() +{ + if (mClassifier) { + mClassifier->Close(); + mClassifier = nullptr; + } + + mCryptoHash = nullptr; + LOG(("urlclassifier db closed\n")); + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::CacheCompletions(CacheResultArray *results) +{ + if (gShuttingDownThread) { + return NS_ERROR_ABORT; + } + + LOG(("nsUrlClassifierDBServiceWorker::CacheCompletions [%p]", this)); + if (!mClassifier) + return NS_OK; + + // Ownership is transferred in to us + nsAutoPtr<CacheResultArray> resultsPtr(results); + + if (mLastResults == *resultsPtr) { + LOG(("Skipping completions that have just been cached already.")); + return NS_OK; + } + + nsAutoPtr<ProtocolParserV2> pParse(new ProtocolParserV2()); + nsTArray<TableUpdate*> updates; + + // Only cache results for tables that we have, don't take + // in tables we might accidentally have hit during a completion. + // This happens due to goog vs googpub lists existing. + nsTArray<nsCString> tables; + nsresult rv = mClassifier->ActiveTables(tables); + NS_ENSURE_SUCCESS(rv, rv); + + for (uint32_t i = 0; i < resultsPtr->Length(); i++) { + bool activeTable = false; + for (uint32_t table = 0; table < tables.Length(); table++) { + if (tables[table].Equals(resultsPtr->ElementAt(i).table)) { + activeTable = true; + break; + } + } + if (activeTable) { + TableUpdateV2* tuV2 = TableUpdate::Cast<TableUpdateV2>( + pParse->GetTableUpdate(resultsPtr->ElementAt(i).table)); + + NS_ENSURE_TRUE(tuV2, NS_ERROR_FAILURE); + + LOG(("CacheCompletion Addchunk %d hash %X", resultsPtr->ElementAt(i).entry.addChunk, + resultsPtr->ElementAt(i).entry.ToUint32())); + rv = tuV2->NewAddComplete(resultsPtr->ElementAt(i).entry.addChunk, + resultsPtr->ElementAt(i).entry.complete); + if (NS_FAILED(rv)) { + // We can bail without leaking here because ForgetTableUpdates + // hasn't been called yet. + return rv; + } + rv = tuV2->NewAddChunk(resultsPtr->ElementAt(i).entry.addChunk); + if (NS_FAILED(rv)) { + return rv; + } + updates.AppendElement(tuV2); + pParse->ForgetTableUpdates(); + } else { + LOG(("Completion received, but table is not active, so not caching.")); + } + } + + mClassifier->ApplyFullHashes(&updates); + mLastResults = *resultsPtr; + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::CacheMisses(PrefixArray *results) +{ + LOG(("nsUrlClassifierDBServiceWorker::CacheMisses [%p] %d", + this, results->Length())); + + // Ownership is transferred in to us + nsAutoPtr<PrefixArray> resultsPtr(results); + + for (uint32_t i = 0; i < resultsPtr->Length(); i++) { + mMissCache.AppendElement(resultsPtr->ElementAt(i)); + } + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::OpenDb() +{ + if (gShuttingDownThread) { + return NS_ERROR_ABORT; + } + + MOZ_ASSERT(!NS_IsMainThread(), "Must initialize DB on background thread"); + // Connection already open, don't do anything. + if (mClassifier) { + return NS_OK; + } + + nsresult rv; + mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoPtr<Classifier> classifier(new Classifier()); + if (!classifier) { + return NS_ERROR_OUT_OF_MEMORY; + } + + rv = classifier->Open(*mCacheDir); + NS_ENSURE_SUCCESS(rv, rv); + + mClassifier = classifier; + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::SetLastUpdateTime(const nsACString &table, + uint64_t updateTime) +{ + MOZ_ASSERT(!NS_IsMainThread(), "Must be on the background thread"); + MOZ_ASSERT(mClassifier, "Classifier connection must be opened"); + + mClassifier->SetLastUpdateTime(table, updateTime); + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::ClearLastResults() +{ + MOZ_ASSERT(!NS_IsMainThread(), "Must be on the background thread"); + mLastResults.Clear(); + return NS_OK; +} + + +// ------------------------------------------------------------------------- +// nsUrlClassifierLookupCallback +// +// This class takes the results of a lookup found on the worker thread +// and handles any necessary partial hash expansions before calling +// the client callback. + +class nsUrlClassifierLookupCallback final : public nsIUrlClassifierLookupCallback + , public nsIUrlClassifierHashCompleterCallback +{ +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIURLCLASSIFIERLOOKUPCALLBACK + NS_DECL_NSIURLCLASSIFIERHASHCOMPLETERCALLBACK + + nsUrlClassifierLookupCallback(nsUrlClassifierDBService *dbservice, + nsIUrlClassifierCallback *c) + : mDBService(dbservice) + , mResults(nullptr) + , mPendingCompletions(0) + , mCallback(c) + {} + +private: + ~nsUrlClassifierLookupCallback(); + + nsresult HandleResults(); + + RefPtr<nsUrlClassifierDBService> mDBService; + nsAutoPtr<LookupResultArray> mResults; + + // Completed results to send back to the worker for caching. + nsAutoPtr<CacheResultArray> mCacheResults; + + uint32_t mPendingCompletions; + nsCOMPtr<nsIUrlClassifierCallback> mCallback; +}; + +NS_IMPL_ISUPPORTS(nsUrlClassifierLookupCallback, + nsIUrlClassifierLookupCallback, + nsIUrlClassifierHashCompleterCallback) + +nsUrlClassifierLookupCallback::~nsUrlClassifierLookupCallback() +{ + if (mCallback) { + NS_ReleaseOnMainThread(mCallback.forget()); + } +} + +NS_IMETHODIMP +nsUrlClassifierLookupCallback::LookupComplete(nsTArray<LookupResult>* results) +{ + NS_ASSERTION(mResults == nullptr, + "Should only get one set of results per nsUrlClassifierLookupCallback!"); + + if (!results) { + HandleResults(); + return NS_OK; + } + + mResults = results; + + // Check the results entries that need to be completed. + for (uint32_t i = 0; i < results->Length(); i++) { + LookupResult& result = results->ElementAt(i); + + // We will complete partial matches and matches that are stale. + if (!result.Confirmed()) { + nsCOMPtr<nsIUrlClassifierHashCompleter> completer; + nsCString gethashUrl; + nsresult rv; + nsCOMPtr<nsIUrlListManager> listManager = do_GetService( + "@mozilla.org/url-classifier/listmanager;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + rv = listManager->GetGethashUrl(result.mTableName, gethashUrl); + NS_ENSURE_SUCCESS(rv, rv); + LOG(("The match from %s needs to be completed at %s", + result.mTableName.get(), gethashUrl.get())); + // gethashUrls may be empty in 2 cases: test tables, and on startup where + // we may have found a prefix in an existing table before the listmanager + // has registered the table. In the second case we should not call + // complete. + if ((!gethashUrl.IsEmpty() || + StringBeginsWith(result.mTableName, NS_LITERAL_CSTRING("test-"))) && + mDBService->GetCompleter(result.mTableName, + getter_AddRefs(completer))) { + nsAutoCString partialHash; + partialHash.Assign(reinterpret_cast<char*>(&result.hash.prefix), + PREFIX_SIZE); + + nsresult rv = completer->Complete(partialHash, gethashUrl, this); + if (NS_SUCCEEDED(rv)) { + mPendingCompletions++; + } + } else { + // For tables with no hash completer, a complete hash match is + // good enough, we'll consider it fresh, even if it hasn't been updated + // in 45 minutes. + if (result.Complete()) { + result.mFresh = true; + LOG(("Skipping completion in a table without a valid completer (%s).", + result.mTableName.get())); + } else { + NS_WARNING("Partial match in a table without a valid completer, ignoring partial match."); + } + } + } + } + + LOG(("nsUrlClassifierLookupCallback::LookupComplete [%p] " + "%u pending completions", this, mPendingCompletions)); + if (mPendingCompletions == 0) { + // All results were complete, we're ready! + HandleResults(); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierLookupCallback::CompletionFinished(nsresult status) +{ + if (LOG_ENABLED()) { + nsAutoCString errorName; + mozilla::GetErrorName(status, errorName); + LOG(("nsUrlClassifierLookupCallback::CompletionFinished [%p, %s]", + this, errorName.get())); + } + + mPendingCompletions--; + if (mPendingCompletions == 0) { + HandleResults(); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierLookupCallback::Completion(const nsACString& completeHash, + const nsACString& tableName, + uint32_t chunkId) +{ + LOG(("nsUrlClassifierLookupCallback::Completion [%p, %s, %d]", + this, PromiseFlatCString(tableName).get(), chunkId)); + mozilla::safebrowsing::Completion hash; + hash.Assign(completeHash); + + // Send this completion to the store for caching. + if (!mCacheResults) { + mCacheResults = new CacheResultArray(); + if (!mCacheResults) + return NS_ERROR_OUT_OF_MEMORY; + } + + CacheResult result; + result.entry.addChunk = chunkId; + result.entry.complete = hash; + result.table = tableName; + + // OK if this fails, we just won't cache the item. + mCacheResults->AppendElement(result); + + // Check if this matched any of our results. + for (uint32_t i = 0; i < mResults->Length(); i++) { + LookupResult& result = mResults->ElementAt(i); + + // Now, see if it verifies a lookup + if (!result.mNoise + && result.CompleteHash() == hash + && result.mTableName.Equals(tableName)) { + result.mProtocolConfirmed = true; + } + } + + return NS_OK; +} + +nsresult +nsUrlClassifierLookupCallback::HandleResults() +{ + if (!mResults) { + // No results, this URI is clean. + LOG(("nsUrlClassifierLookupCallback::HandleResults [%p, no results]", this)); + return mCallback->HandleEvent(NS_LITERAL_CSTRING("")); + } + MOZ_ASSERT(mPendingCompletions == 0, "HandleResults() should never be " + "called while there are pending completions"); + + LOG(("nsUrlClassifierLookupCallback::HandleResults [%p, %u results]", + this, mResults->Length())); + + nsTArray<nsCString> tables; + // Build a stringified list of result tables. + for (uint32_t i = 0; i < mResults->Length(); i++) { + LookupResult& result = mResults->ElementAt(i); + + // Leave out results that weren't confirmed, as their existence on + // the list can't be verified. Also leave out randomly-generated + // noise. + if (result.mNoise) { + LOG(("Skipping result %X from table %s (noise)", + result.hash.prefix.ToUint32(), result.mTableName.get())); + continue; + } else if (!result.Confirmed()) { + LOG(("Skipping result %X from table %s (not confirmed)", + result.hash.prefix.ToUint32(), result.mTableName.get())); + continue; + } + + LOG(("Confirmed result %X from table %s", + result.hash.prefix.ToUint32(), result.mTableName.get())); + + if (tables.IndexOf(result.mTableName) == nsTArray<nsCString>::NoIndex) { + tables.AppendElement(result.mTableName); + } + } + + // Some parts of this gethash request generated no hits at all. + // Prefixes must have been removed from the database since our last update. + // Save the prefixes we checked to prevent repeated requests + // until the next update. + nsAutoPtr<PrefixArray> cacheMisses(new PrefixArray()); + if (cacheMisses) { + for (uint32_t i = 0; i < mResults->Length(); i++) { + LookupResult &result = mResults->ElementAt(i); + if (!result.Confirmed() && !result.mNoise) { + cacheMisses->AppendElement(result.PrefixHash()); + } + } + // Hands ownership of the miss array back to the worker thread. + mDBService->CacheMisses(cacheMisses.forget()); + } + + if (mCacheResults) { + // This hands ownership of the cache results array back to the worker + // thread. + mDBService->CacheCompletions(mCacheResults.forget()); + } + + nsAutoCString tableStr; + for (uint32_t i = 0; i < tables.Length(); i++) { + if (i != 0) + tableStr.Append(','); + tableStr.Append(tables[i]); + } + + return mCallback->HandleEvent(tableStr); +} + + +// ------------------------------------------------------------------------- +// Helper class for nsIURIClassifier implementation, translates table names +// to nsIURIClassifier enums. + +class nsUrlClassifierClassifyCallback final : public nsIUrlClassifierCallback +{ +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIURLCLASSIFIERCALLBACK + + explicit nsUrlClassifierClassifyCallback(nsIURIClassifierCallback *c) + : mCallback(c) + {} + +private: + ~nsUrlClassifierClassifyCallback() {} + + nsCOMPtr<nsIURIClassifierCallback> mCallback; +}; + +NS_IMPL_ISUPPORTS(nsUrlClassifierClassifyCallback, + nsIUrlClassifierCallback) + +NS_IMETHODIMP +nsUrlClassifierClassifyCallback::HandleEvent(const nsACString& tables) +{ + nsresult response = TablesToResponse(tables); + mCallback->OnClassifyComplete(response); + return NS_OK; +} + + +// ------------------------------------------------------------------------- +// Proxy class implementation + +NS_IMPL_ISUPPORTS(nsUrlClassifierDBService, + nsIUrlClassifierDBService, + nsIURIClassifier, + nsIObserver) + +/* static */ nsUrlClassifierDBService* +nsUrlClassifierDBService::GetInstance(nsresult *result) +{ + *result = NS_OK; + if (!sUrlClassifierDBService) { + sUrlClassifierDBService = new nsUrlClassifierDBService(); + if (!sUrlClassifierDBService) { + *result = NS_ERROR_OUT_OF_MEMORY; + return nullptr; + } + + NS_ADDREF(sUrlClassifierDBService); // addref the global + + *result = sUrlClassifierDBService->Init(); + if (NS_FAILED(*result)) { + NS_RELEASE(sUrlClassifierDBService); + return nullptr; + } + } else { + // Already exists, just add a ref + NS_ADDREF(sUrlClassifierDBService); // addref the return result + } + return sUrlClassifierDBService; +} + + +nsUrlClassifierDBService::nsUrlClassifierDBService() + : mCheckMalware(CHECK_MALWARE_DEFAULT) + , mCheckPhishing(CHECK_PHISHING_DEFAULT) + , mCheckTracking(CHECK_TRACKING_DEFAULT) + , mCheckBlockedURIs(CHECK_BLOCKED_DEFAULT) + , mInUpdate(false) +{ +} + +nsUrlClassifierDBService::~nsUrlClassifierDBService() +{ + sUrlClassifierDBService = nullptr; +} + +nsresult +nsUrlClassifierDBService::ReadTablesFromPrefs() +{ + nsCString allTables; + nsCString tables; + Preferences::GetCString(PHISH_TABLE_PREF, &allTables); + + Preferences::GetCString(MALWARE_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Preferences::GetCString(DOWNLOAD_BLOCK_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Preferences::GetCString(DOWNLOAD_ALLOW_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Preferences::GetCString(TRACKING_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Preferences::GetCString(TRACKING_WHITELIST_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Preferences::GetCString(BLOCKED_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Classifier::SplitTables(allTables, mGethashTables); + + Preferences::GetCString(DISALLOW_COMPLETION_TABLE_PREF, &tables); + Classifier::SplitTables(tables, mDisallowCompletionsTables); + + return NS_OK; +} + +nsresult +nsUrlClassifierDBService::Init() +{ + MOZ_ASSERT(NS_IsMainThread(), "Must initialize DB service on main thread"); + nsCOMPtr<nsIXULRuntime> appInfo = do_GetService("@mozilla.org/xre/app-info;1"); + if (appInfo) { + bool inSafeMode = false; + appInfo->GetInSafeMode(&inSafeMode); + if (inSafeMode) { + return NS_ERROR_NOT_AVAILABLE; + } + } + + // Retrieve all the preferences. + mCheckMalware = Preferences::GetBool(CHECK_MALWARE_PREF, + CHECK_MALWARE_DEFAULT); + mCheckPhishing = Preferences::GetBool(CHECK_PHISHING_PREF, + CHECK_PHISHING_DEFAULT); + mCheckTracking = + Preferences::GetBool(CHECK_TRACKING_PREF, CHECK_TRACKING_DEFAULT) || + Preferences::GetBool(CHECK_TRACKING_PB_PREF, CHECK_TRACKING_PB_DEFAULT); + mCheckBlockedURIs = Preferences::GetBool(CHECK_BLOCKED_PREF, + CHECK_BLOCKED_DEFAULT); + uint32_t gethashNoise = Preferences::GetUint(GETHASH_NOISE_PREF, + GETHASH_NOISE_DEFAULT); + gFreshnessGuarantee = Preferences::GetInt(CONFIRM_AGE_PREF, + CONFIRM_AGE_DEFAULT_SEC); + ReadTablesFromPrefs(); + + nsresult rv; + + { + // Force PSM loading on main thread + nsCOMPtr<nsICryptoHash> dummy = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + } + + { + // Force nsIUrlClassifierUtils loading on main thread. + nsCOMPtr<nsIUrlClassifierUtils> dummy = + do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Directory providers must also be accessed on the main thread. + nsCOMPtr<nsIFile> cacheDir; + rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_LOCAL_50_DIR, + getter_AddRefs(cacheDir)); + if (NS_FAILED(rv)) { + rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(cacheDir)); + if (NS_FAILED(rv)) { + return rv; + } + } + + // Start the background thread. + rv = NS_NewNamedThread("URL Classifier", &gDbBackgroundThread); + if (NS_FAILED(rv)) + return rv; + + mWorker = new nsUrlClassifierDBServiceWorker(); + if (!mWorker) + return NS_ERROR_OUT_OF_MEMORY; + + rv = mWorker->Init(gethashNoise, cacheDir); + if (NS_FAILED(rv)) { + mWorker = nullptr; + return rv; + } + + // Proxy for calling the worker on the background thread + mWorkerProxy = new UrlClassifierDBServiceWorkerProxy(mWorker); + rv = mWorkerProxy->OpenDb(); + if (NS_FAILED(rv)) { + return rv; + } + + // Add an observer for shutdown + nsCOMPtr<nsIObserverService> observerService = + mozilla::services::GetObserverService(); + if (!observerService) + return NS_ERROR_FAILURE; + + // The application is about to quit + observerService->AddObserver(this, "quit-application", false); + observerService->AddObserver(this, "profile-before-change", false); + + // XXX: Do we *really* need to be able to change all of these at runtime? + // Note: These observers should only be added when everything else above has + // succeeded. Failing to do so can cause long shutdown times in certain + // situations. See Bug 1247798 and Bug 1244803. + Preferences::AddStrongObserver(this, CHECK_MALWARE_PREF); + Preferences::AddStrongObserver(this, CHECK_PHISHING_PREF); + Preferences::AddStrongObserver(this, CHECK_TRACKING_PREF); + Preferences::AddStrongObserver(this, CHECK_TRACKING_PB_PREF); + Preferences::AddStrongObserver(this, CHECK_BLOCKED_PREF); + Preferences::AddStrongObserver(this, GETHASH_NOISE_PREF); + Preferences::AddStrongObserver(this, CONFIRM_AGE_PREF); + Preferences::AddStrongObserver(this, PHISH_TABLE_PREF); + Preferences::AddStrongObserver(this, MALWARE_TABLE_PREF); + Preferences::AddStrongObserver(this, TRACKING_TABLE_PREF); + Preferences::AddStrongObserver(this, TRACKING_WHITELIST_TABLE_PREF); + Preferences::AddStrongObserver(this, BLOCKED_TABLE_PREF); + Preferences::AddStrongObserver(this, DOWNLOAD_BLOCK_TABLE_PREF); + Preferences::AddStrongObserver(this, DOWNLOAD_ALLOW_TABLE_PREF); + Preferences::AddStrongObserver(this, DISALLOW_COMPLETION_TABLE_PREF); + + return NS_OK; +} + +void +nsUrlClassifierDBService::BuildTables(bool aTrackingProtectionEnabled, + nsCString &tables) +{ + nsAutoCString malware; + // LookupURI takes a comma-separated list already. + Preferences::GetCString(MALWARE_TABLE_PREF, &malware); + if (mCheckMalware && !malware.IsEmpty()) { + tables.Append(malware); + } + nsAutoCString phishing; + Preferences::GetCString(PHISH_TABLE_PREF, &phishing); + if (mCheckPhishing && !phishing.IsEmpty()) { + tables.Append(','); + tables.Append(phishing); + } + if (aTrackingProtectionEnabled) { + nsAutoCString tracking, trackingWhitelist; + Preferences::GetCString(TRACKING_TABLE_PREF, &tracking); + if (!tracking.IsEmpty()) { + tables.Append(','); + tables.Append(tracking); + } + Preferences::GetCString(TRACKING_WHITELIST_TABLE_PREF, &trackingWhitelist); + if (!trackingWhitelist.IsEmpty()) { + tables.Append(','); + tables.Append(trackingWhitelist); + } + } + nsAutoCString blocked; + Preferences::GetCString(BLOCKED_TABLE_PREF, &blocked); + if (mCheckBlockedURIs && !blocked.IsEmpty()) { + tables.Append(','); + tables.Append(blocked); + } + + if (StringBeginsWith(tables, NS_LITERAL_CSTRING(","))) { + tables.Cut(0, 1); + } +} + +// nsChannelClassifier is the only consumer of this interface. +NS_IMETHODIMP +nsUrlClassifierDBService::Classify(nsIPrincipal* aPrincipal, + bool aTrackingProtectionEnabled, + nsIURIClassifierCallback* c, + bool* result) +{ + NS_ENSURE_ARG(aPrincipal); + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + if (!(mCheckMalware || mCheckPhishing || aTrackingProtectionEnabled || + mCheckBlockedURIs)) { + *result = false; + return NS_OK; + } + + RefPtr<nsUrlClassifierClassifyCallback> callback = + new nsUrlClassifierClassifyCallback(c); + if (!callback) return NS_ERROR_OUT_OF_MEMORY; + + nsAutoCString tables; + BuildTables(aTrackingProtectionEnabled, tables); + + nsresult rv = LookupURI(aPrincipal, tables, callback, false, result); + if (rv == NS_ERROR_MALFORMED_URI) { + *result = false; + // The URI had no hostname, don't try to classify it. + return NS_OK; + } + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBService::ClassifyLocalWithTables(nsIURI *aURI, + const nsACString & aTables, + nsACString & aTableResults) +{ + if (gShuttingDownThread) { + return NS_ERROR_ABORT; + } + + PROFILER_LABEL_FUNC(js::ProfileEntry::Category::OTHER); + MOZ_ASSERT(NS_IsMainThread(), "ClassifyLocalWithTables must be on main thread"); + + nsCOMPtr<nsIURI> uri = NS_GetInnermostURI(aURI); + NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE); + + nsAutoCString key; + // Canonicalize the url + nsCOMPtr<nsIUrlClassifierUtils> utilsService = + do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID); + nsresult rv = utilsService->GetKeyForURI(uri, key); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoPtr<LookupResultArray> results(new LookupResultArray()); + if (!results) { + return NS_ERROR_OUT_OF_MEMORY; + } + + // In unittests, we may not have been initalized, so don't crash. + rv = mWorkerProxy->DoLocalLookup(key, aTables, results); + if (NS_SUCCEEDED(rv)) { + aTableResults = ProcessLookupResults(results); + } + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBService::Lookup(nsIPrincipal* aPrincipal, + const nsACString& tables, + nsIUrlClassifierCallback* c) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + bool dummy; + return LookupURI(aPrincipal, tables, c, true, &dummy); +} + +nsresult +nsUrlClassifierDBService::LookupURI(nsIPrincipal* aPrincipal, + const nsACString& tables, + nsIUrlClassifierCallback* c, + bool forceLookup, + bool *didLookup) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + NS_ENSURE_ARG(aPrincipal); + + if (nsContentUtils::IsSystemPrincipal(aPrincipal)) { + *didLookup = false; + return NS_OK; + } + + if (gShuttingDownThread) { + *didLookup = false; + return NS_ERROR_ABORT; + } + + nsCOMPtr<nsIURI> uri; + nsresult rv = aPrincipal->GetURI(getter_AddRefs(uri)); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE); + + uri = NS_GetInnermostURI(uri); + NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE); + + nsAutoCString key; + // Canonicalize the url + nsCOMPtr<nsIUrlClassifierUtils> utilsService = + do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID); + rv = utilsService->GetKeyForURI(uri, key); + if (NS_FAILED(rv)) + return rv; + + if (forceLookup) { + *didLookup = true; + } else { + bool clean = false; + + if (!clean) { + nsCOMPtr<nsIPermissionManager> permissionManager = + services::GetPermissionManager(); + + if (permissionManager) { + uint32_t perm; + rv = permissionManager->TestPermissionFromPrincipal(aPrincipal, + "safe-browsing", &perm); + NS_ENSURE_SUCCESS(rv, rv); + + clean |= (perm == nsIPermissionManager::ALLOW_ACTION); + } + } + + *didLookup = !clean; + if (clean) { + return NS_OK; + } + } + + // Create an nsUrlClassifierLookupCallback object. This object will + // take care of confirming partial hash matches if necessary before + // calling the client's callback. + nsCOMPtr<nsIUrlClassifierLookupCallback> callback = + new nsUrlClassifierLookupCallback(this, c); + if (!callback) + return NS_ERROR_OUT_OF_MEMORY; + + nsCOMPtr<nsIUrlClassifierLookupCallback> proxyCallback = + new UrlClassifierLookupCallbackProxy(callback); + + // Queue this lookup and call the lookup function to flush the queue if + // necessary. + rv = mWorker->QueueLookup(key, tables, proxyCallback); + NS_ENSURE_SUCCESS(rv, rv); + + // This seems to just call HandlePendingLookups. + nsAutoCString dummy; + return mWorkerProxy->Lookup(nullptr, dummy, nullptr); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::GetTables(nsIUrlClassifierCallback* c) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + // The proxy callback uses the current thread. + nsCOMPtr<nsIUrlClassifierCallback> proxyCallback = + new UrlClassifierCallbackProxy(c); + + return mWorkerProxy->GetTables(proxyCallback); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::SetHashCompleter(const nsACString &tableName, + nsIUrlClassifierHashCompleter *completer) +{ + if (completer) { + mCompleters.Put(tableName, completer); + } else { + mCompleters.Remove(tableName); + } + ClearLastResults(); + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBService::SetLastUpdateTime(const nsACString &tableName, + uint64_t lastUpdateTime) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->SetLastUpdateTime(tableName, lastUpdateTime); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::ClearLastResults() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->ClearLastResults(); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::BeginUpdate(nsIUrlClassifierUpdateObserver *observer, + const nsACString &updateTables) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + if (mInUpdate) { + LOG(("Already updating, not available")); + return NS_ERROR_NOT_AVAILABLE; + } + + mInUpdate = true; + + // The proxy observer uses the current thread + nsCOMPtr<nsIUrlClassifierUpdateObserver> proxyObserver = + new UrlClassifierUpdateObserverProxy(observer); + + return mWorkerProxy->BeginUpdate(proxyObserver, updateTables); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::BeginStream(const nsACString &table) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->BeginStream(table); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::UpdateStream(const nsACString& aUpdateChunk) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->UpdateStream(aUpdateChunk); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::FinishStream() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->FinishStream(); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::FinishUpdate() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + mInUpdate = false; + + return mWorkerProxy->FinishUpdate(); +} + + +NS_IMETHODIMP +nsUrlClassifierDBService::CancelUpdate() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + mInUpdate = false; + + return mWorkerProxy->CancelUpdate(); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::ResetDatabase() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->ResetDatabase(); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::ReloadDatabase() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->ReloadDatabase(); +} + +nsresult +nsUrlClassifierDBService::CacheCompletions(CacheResultArray *results) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->CacheCompletions(results); +} + +nsresult +nsUrlClassifierDBService::CacheMisses(PrefixArray *results) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->CacheMisses(results); +} + +bool +nsUrlClassifierDBService::GetCompleter(const nsACString &tableName, + nsIUrlClassifierHashCompleter **completer) +{ + // If we have specified a completer, go ahead and query it. This is only + // used by tests. + if (mCompleters.Get(tableName, completer)) { + return true; + } + + // If we don't know about this table at all, or are disallowing completions + // for it, skip completion checks. + if (!mGethashTables.Contains(tableName) || + mDisallowCompletionsTables.Contains(tableName)) { + return false; + } + + // Otherwise, call gethash to find the hash completions. + return NS_SUCCEEDED(CallGetService(NS_URLCLASSIFIERHASHCOMPLETER_CONTRACTID, + completer)); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::Observe(nsISupports *aSubject, const char *aTopic, + const char16_t *aData) +{ + if (!strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) { + nsresult rv; + nsCOMPtr<nsIPrefBranch> prefs(do_QueryInterface(aSubject, &rv)); + NS_ENSURE_SUCCESS(rv, rv); + Unused << prefs; + + if (NS_LITERAL_STRING(CHECK_MALWARE_PREF).Equals(aData)) { + mCheckMalware = Preferences::GetBool(CHECK_MALWARE_PREF, + CHECK_MALWARE_DEFAULT); + } else if (NS_LITERAL_STRING(CHECK_PHISHING_PREF).Equals(aData)) { + mCheckPhishing = Preferences::GetBool(CHECK_PHISHING_PREF, + CHECK_PHISHING_DEFAULT); + } else if (NS_LITERAL_STRING(CHECK_TRACKING_PREF).Equals(aData) || + NS_LITERAL_STRING(CHECK_TRACKING_PB_PREF).Equals(aData)) { + mCheckTracking = + Preferences::GetBool(CHECK_TRACKING_PREF, CHECK_TRACKING_DEFAULT) || + Preferences::GetBool(CHECK_TRACKING_PB_PREF, CHECK_TRACKING_PB_DEFAULT); + } else if (NS_LITERAL_STRING(CHECK_BLOCKED_PREF).Equals(aData)) { + mCheckBlockedURIs = Preferences::GetBool(CHECK_BLOCKED_PREF, + CHECK_BLOCKED_DEFAULT); + } else if ( + NS_LITERAL_STRING(PHISH_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(MALWARE_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(TRACKING_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(TRACKING_WHITELIST_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(BLOCKED_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(DOWNLOAD_BLOCK_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(DOWNLOAD_ALLOW_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(DISALLOW_COMPLETION_TABLE_PREF).Equals(aData)) { + // Just read everything again. + ReadTablesFromPrefs(); + } else if (NS_LITERAL_STRING(CONFIRM_AGE_PREF).Equals(aData)) { + gFreshnessGuarantee = Preferences::GetInt(CONFIRM_AGE_PREF, + CONFIRM_AGE_DEFAULT_SEC); + } + } else if (!strcmp(aTopic, "quit-application")) { + Shutdown(); + } else if (!strcmp(aTopic, "profile-before-change")) { + // Unit test does not receive "quit-application", + // need call shutdown in this case + Shutdown(); + LOG(("joining background thread")); + mWorkerProxy = nullptr; + + if (!gDbBackgroundThread) { + return NS_OK; + } + + nsIThread *backgroundThread = gDbBackgroundThread; + gDbBackgroundThread = nullptr; + backgroundThread->Shutdown(); + NS_RELEASE(backgroundThread); + } else { + return NS_ERROR_UNEXPECTED; + } + + return NS_OK; +} + +// Join the background thread if it exists. +nsresult +nsUrlClassifierDBService::Shutdown() +{ + LOG(("shutting down db service\n")); + + if (!gDbBackgroundThread || gShuttingDownThread) { + return NS_OK; + } + + gShuttingDownThread = true; + + Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_SHUTDOWN_TIME> timer; + + mCompleters.Clear(); + + nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID); + if (prefs) { + prefs->RemoveObserver(CHECK_MALWARE_PREF, this); + prefs->RemoveObserver(CHECK_PHISHING_PREF, this); + prefs->RemoveObserver(CHECK_TRACKING_PREF, this); + prefs->RemoveObserver(CHECK_TRACKING_PB_PREF, this); + prefs->RemoveObserver(CHECK_BLOCKED_PREF, this); + prefs->RemoveObserver(PHISH_TABLE_PREF, this); + prefs->RemoveObserver(MALWARE_TABLE_PREF, this); + prefs->RemoveObserver(TRACKING_TABLE_PREF, this); + prefs->RemoveObserver(TRACKING_WHITELIST_TABLE_PREF, this); + prefs->RemoveObserver(BLOCKED_TABLE_PREF, this); + prefs->RemoveObserver(DOWNLOAD_BLOCK_TABLE_PREF, this); + prefs->RemoveObserver(DOWNLOAD_ALLOW_TABLE_PREF, this); + prefs->RemoveObserver(DISALLOW_COMPLETION_TABLE_PREF, this); + prefs->RemoveObserver(CONFIRM_AGE_PREF, this); + } + + DebugOnly<nsresult> rv; + // First close the db connection. + if (mWorker) { + rv = mWorkerProxy->CancelUpdate(); + NS_ASSERTION(NS_SUCCEEDED(rv), "failed to post cancel update event"); + + rv = mWorkerProxy->CloseDb(); + NS_ASSERTION(NS_SUCCEEDED(rv), "failed to post close db event"); + } + return NS_OK; +} + +nsIThread* +nsUrlClassifierDBService::BackgroundThread() +{ + return gDbBackgroundThread; +} + +// static +bool +nsUrlClassifierDBService::ShutdownHasStarted() +{ + return gShuttingDownThread; +} diff --git a/toolkit/components/url-classifier/nsUrlClassifierDBService.h b/toolkit/components/url-classifier/nsUrlClassifierDBService.h new file mode 100644 index 000000000..55c10c1bf --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierDBService.h @@ -0,0 +1,270 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsUrlClassifierDBService_h_ +#define nsUrlClassifierDBService_h_ + +#include <nsISupportsUtils.h> + +#include "nsID.h" +#include "nsInterfaceHashtable.h" +#include "nsIObserver.h" +#include "nsUrlClassifierPrefixSet.h" +#include "nsIUrlClassifierHashCompleter.h" +#include "nsIUrlListManager.h" +#include "nsIUrlClassifierDBService.h" +#include "nsIURIClassifier.h" +#include "nsToolkitCompsCID.h" +#include "nsICryptoHMAC.h" +#include "mozilla/Attributes.h" +#include "mozilla/Mutex.h" +#include "mozilla/TimeStamp.h" + +#include "Entries.h" +#include "LookupCache.h" + +// GCC < 6.1 workaround, see bug 1329593 +#if defined(XP_WIN) && defined(__MINGW32__) +#define GCC_MANGLING_WORKAROUND __stdcall +#else +#define GCC_MANGLING_WORKAROUND +#endif + +// The hash length for a domain key. +#define DOMAIN_LENGTH 4 + +// The hash length of a partial hash entry. +#define PARTIAL_LENGTH 4 + +// The hash length of a complete hash entry. +#define COMPLETE_LENGTH 32 + +using namespace mozilla::safebrowsing; + +class nsUrlClassifierDBServiceWorker; +class nsIThread; +class nsIURI; +class UrlClassifierDBServiceWorkerProxy; +namespace mozilla { +namespace safebrowsing { +class Classifier; +class ProtocolParser; +class TableUpdate; + +nsresult +TablesToResponse(const nsACString& tables); + +} // namespace safebrowsing +} // namespace mozilla + +// This is a proxy class that just creates a background thread and delagates +// calls to the background thread. +class nsUrlClassifierDBService final : public nsIUrlClassifierDBService, + public nsIURIClassifier, + public nsIObserver +{ +public: + // This is thread safe. It throws an exception if the thread is busy. + nsUrlClassifierDBService(); + + nsresult Init(); + + static nsUrlClassifierDBService* GetInstance(nsresult *result); + + NS_DECLARE_STATIC_IID_ACCESSOR(NS_URLCLASSIFIERDBSERVICE_CID) + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIURLCLASSIFIERDBSERVICE + NS_DECL_NSIURICLASSIFIER + NS_DECL_NSIOBSERVER + + bool GetCompleter(const nsACString& tableName, + nsIUrlClassifierHashCompleter** completer); + nsresult CacheCompletions(mozilla::safebrowsing::CacheResultArray *results); + nsresult CacheMisses(mozilla::safebrowsing::PrefixArray *results); + + static nsIThread* BackgroundThread(); + + static bool ShutdownHasStarted(); + +private: + // No subclassing + ~nsUrlClassifierDBService(); + + // Disallow copy constructor + nsUrlClassifierDBService(nsUrlClassifierDBService&); + + nsresult LookupURI(nsIPrincipal* aPrincipal, + const nsACString& tables, + nsIUrlClassifierCallback* c, + bool forceCheck, bool *didCheck); + + // Close db connection and join the background thread if it exists. + nsresult Shutdown(); + + // Check if the key is on a known-clean host. + nsresult CheckClean(const nsACString &lookupKey, + bool *clean); + + // Read everything into mGethashTables and mDisallowCompletionTables + nsresult ReadTablesFromPrefs(); + + // Build a comma-separated list of tables to check + void BuildTables(bool trackingProtectionEnabled, nsCString& tables); + + RefPtr<nsUrlClassifierDBServiceWorker> mWorker; + RefPtr<UrlClassifierDBServiceWorkerProxy> mWorkerProxy; + + nsInterfaceHashtable<nsCStringHashKey, nsIUrlClassifierHashCompleter> mCompleters; + + // TRUE if the nsURIClassifier implementation should check for malware + // uris on document loads. + bool mCheckMalware; + + // TRUE if the nsURIClassifier implementation should check for phishing + // uris on document loads. + bool mCheckPhishing; + + // TRUE if the nsURIClassifier implementation should check for tracking + // uris on document loads. + bool mCheckTracking; + + // TRUE if the nsURIClassifier implementation should check for blocked + // uris on document loads. + bool mCheckBlockedURIs; + + // TRUE if a BeginUpdate() has been called without an accompanying + // CancelUpdate()/FinishUpdate(). This is used to prevent competing + // updates, not to determine whether an update is still being + // processed. + bool mInUpdate; + + // The list of tables that can use the default hash completer object. + nsTArray<nsCString> mGethashTables; + + // The list of tables that should never be hash completed. + nsTArray<nsCString> mDisallowCompletionsTables; + + // Thread that we do the updates on. + static nsIThread* gDbBackgroundThread; +}; + +class nsUrlClassifierDBServiceWorker final : public nsIUrlClassifierDBService +{ +public: + nsUrlClassifierDBServiceWorker(); + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIURLCLASSIFIERDBSERVICE + + nsresult Init(uint32_t aGethashNoise, nsCOMPtr<nsIFile> aCacheDir); + + // Queue a lookup for the worker to perform, called in the main thread. + // tables is a comma-separated list of tables to query + nsresult QueueLookup(const nsACString& lookupKey, + const nsACString& tables, + nsIUrlClassifierLookupCallback* callback); + + // Handle any queued-up lookups. We call this function during long-running + // update operations to prevent lookups from blocking for too long. + nsresult HandlePendingLookups(); + + // Perform a blocking classifier lookup for a given url. Can be called on + // either the main thread or the worker thread. + nsresult DoLocalLookup(const nsACString& spec, + const nsACString& tables, + LookupResultArray* results); + + // Open the DB connection + nsresult GCC_MANGLING_WORKAROUND OpenDb(); + + // Provide a way to forcibly close the db connection. + nsresult GCC_MANGLING_WORKAROUND CloseDb(); + + nsresult CacheCompletions(CacheResultArray * aEntries); + nsresult CacheMisses(PrefixArray * aEntries); + +private: + // No subclassing + ~nsUrlClassifierDBServiceWorker(); + + // Disallow copy constructor + nsUrlClassifierDBServiceWorker(nsUrlClassifierDBServiceWorker&); + + // Applies the current transaction and resets the update/working times. + nsresult ApplyUpdate(); + + // Reset the in-progress update stream + void ResetStream(); + + // Reset the in-progress update + void ResetUpdate(); + + // Perform a classifier lookup for a given url. + nsresult DoLookup(const nsACString& spec, + const nsACString& tables, + nsIUrlClassifierLookupCallback* c); + + nsresult AddNoise(const Prefix aPrefix, + const nsCString tableName, + uint32_t aCount, + LookupResultArray& results); + + // Can only be used on the background thread + nsCOMPtr<nsICryptoHash> mCryptoHash; + + nsAutoPtr<mozilla::safebrowsing::Classifier> mClassifier; + // The class that actually parses the update chunks. + nsAutoPtr<ProtocolParser> mProtocolParser; + + // Directory where to store the SB databases. + nsCOMPtr<nsIFile> mCacheDir; + + // XXX: maybe an array of autoptrs. Or maybe a class specifically + // storing a series of updates. + nsTArray<mozilla::safebrowsing::TableUpdate*> mTableUpdates; + + uint32_t mUpdateWaitSec; + + // Entries that cannot be completed. We expect them to die at + // the next update + PrefixArray mMissCache; + + // Stores the last results that triggered a table update. + CacheResultArray mLastResults; + + nsresult mUpdateStatus; + nsTArray<nsCString> mUpdateTables; + + nsCOMPtr<nsIUrlClassifierUpdateObserver> mUpdateObserver; + bool mInStream; + + // The number of noise entries to add to the set of lookup results. + uint32_t mGethashNoise; + + // Pending lookups are stored in a queue for processing. The queue + // is protected by mPendingLookupLock. + mozilla::Mutex mPendingLookupLock; + + class PendingLookup { + public: + mozilla::TimeStamp mStartTime; + nsCString mKey; + nsCString mTables; + nsCOMPtr<nsIUrlClassifierLookupCallback> mCallback; + }; + + // list of pending lookups + nsTArray<PendingLookup> mPendingLookups; + +#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES + // The raw update response for debugging. + nsCString mRawTableUpdates; +#endif +}; + +NS_DEFINE_STATIC_IID_ACCESSOR(nsUrlClassifierDBService, NS_URLCLASSIFIERDBSERVICE_CID) + +#endif // nsUrlClassifierDBService_h_ diff --git a/toolkit/components/url-classifier/nsUrlClassifierHashCompleter.js b/toolkit/components/url-classifier/nsUrlClassifierHashCompleter.js new file mode 100644 index 000000000..ba4bf225a --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierHashCompleter.js @@ -0,0 +1,589 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cr = Components.results; +const Cu = Components.utils; + +// COMPLETE_LENGTH and PARTIAL_LENGTH copied from nsUrlClassifierDBService.h, +// they correspond to the length, in bytes, of a hash prefix and the total +// hash. +const COMPLETE_LENGTH = 32; +const PARTIAL_LENGTH = 4; + +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/NetUtil.jsm"); + + +// Log only if browser.safebrowsing.debug is true +function log(...stuff) { + let logging = null; + try { + logging = Services.prefs.getBoolPref("browser.safebrowsing.debug"); + } catch(e) { + return; + } + if (!logging) { + return; + } + + var d = new Date(); + let msg = "hashcompleter: " + d.toTimeString() + ": " + stuff.join(" "); + dump(Services.urlFormatter.trimSensitiveURLs(msg) + "\n"); +} + +// Map the HTTP response code to a Telemetry bucket +// https://developers.google.com/safe-browsing/developers_guide_v2?hl=en +function httpStatusToBucket(httpStatus) { + var statusBucket; + switch (httpStatus) { + case 100: + case 101: + // Unexpected 1xx return code + statusBucket = 0; + break; + case 200: + // OK - Data is available in the HTTP response body. + statusBucket = 1; + break; + case 201: + case 202: + case 203: + case 205: + case 206: + // Unexpected 2xx return code + statusBucket = 2; + break; + case 204: + // No Content - There are no full-length hashes with the requested prefix. + statusBucket = 3; + break; + case 300: + case 301: + case 302: + case 303: + case 304: + case 305: + case 307: + case 308: + // Unexpected 3xx return code + statusBucket = 4; + break; + case 400: + // Bad Request - The HTTP request was not correctly formed. + // The client did not provide all required CGI parameters. + statusBucket = 5; + break; + case 401: + case 402: + case 405: + case 406: + case 407: + case 409: + case 410: + case 411: + case 412: + case 414: + case 415: + case 416: + case 417: + case 421: + case 426: + case 428: + case 429: + case 431: + case 451: + // Unexpected 4xx return code + statusBucket = 6; + break; + case 403: + // Forbidden - The client id is invalid. + statusBucket = 7; + break; + case 404: + // Not Found + statusBucket = 8; + break; + case 408: + // Request Timeout + statusBucket = 9; + break; + case 413: + // Request Entity Too Large - Bug 1150334 + statusBucket = 10; + break; + case 500: + case 501: + case 510: + // Unexpected 5xx return code + statusBucket = 11; + break; + case 502: + case 504: + case 511: + // Local network errors, we'll ignore these. + statusBucket = 12; + break; + case 503: + // Service Unavailable - The server cannot handle the request. + // Clients MUST follow the backoff behavior specified in the + // Request Frequency section. + statusBucket = 13; + break; + case 505: + // HTTP Version Not Supported - The server CANNOT handle the requested + // protocol major version. + statusBucket = 14; + break; + default: + statusBucket = 15; + }; + return statusBucket; +} + +function HashCompleter() { + // The current HashCompleterRequest in flight. Once it is started, it is set + // to null. It may be used by multiple calls to |complete| in succession to + // avoid creating multiple requests to the same gethash URL. + this._currentRequest = null; + // A map of gethashUrls to HashCompleterRequests that haven't yet begun. + this._pendingRequests = {}; + + // A map of gethash URLs to RequestBackoff objects. + this._backoffs = {}; + + // Whether we have been informed of a shutdown by the shutdown event. + this._shuttingDown = false; + + Services.obs.addObserver(this, "quit-application", false); + +} + +HashCompleter.prototype = { + classID: Components.ID("{9111de73-9322-4bfc-8b65-2b727f3e6ec8}"), + QueryInterface: XPCOMUtils.generateQI([Ci.nsIUrlClassifierHashCompleter, + Ci.nsIRunnable, + Ci.nsIObserver, + Ci.nsISupportsWeakReference, + Ci.nsITimerCallback, + Ci.nsISupports]), + + // This is mainly how the HashCompleter interacts with other components. + // Even though it only takes one partial hash and callback, subsequent + // calls are made into the same HTTP request by using a thread dispatch. + complete: function HC_complete(aPartialHash, aGethashUrl, aCallback) { + if (!aGethashUrl) { + throw Cr.NS_ERROR_NOT_INITIALIZED; + } + + if (!this._currentRequest) { + this._currentRequest = new HashCompleterRequest(this, aGethashUrl); + } + if (this._currentRequest.gethashUrl == aGethashUrl) { + this._currentRequest.add(aPartialHash, aCallback); + } else { + if (!this._pendingRequests[aGethashUrl]) { + this._pendingRequests[aGethashUrl] = + new HashCompleterRequest(this, aGethashUrl); + } + this._pendingRequests[aGethashUrl].add(aPartialHash, aCallback); + } + + if (!this._backoffs[aGethashUrl]) { + // Initialize request backoffs separately, since requests are deleted + // after they are dispatched. + var jslib = Cc["@mozilla.org/url-classifier/jslib;1"] + .getService().wrappedJSObject; + + // Using the V4 backoff algorithm for both V2 and V4. See bug 1273398. + this._backoffs[aGethashUrl] = new jslib.RequestBackoffV4( + 10 /* keep track of max requests */, + 0 /* don't throttle on successful requests per time period */); + } + // Start off this request. Without dispatching to a thread, every call to + // complete makes an individual HTTP request. + Services.tm.currentThread.dispatch(this, Ci.nsIThread.DISPATCH_NORMAL); + }, + + // This is called after several calls to |complete|, or after the + // currentRequest has finished. It starts off the HTTP request by making a + // |begin| call to the HashCompleterRequest. + run: function() { + // Clear everything on shutdown + if (this._shuttingDown) { + this._currentRequest = null; + this._pendingRequests = null; + for (var url in this._backoffs) { + this._backoffs[url] = null; + } + throw Cr.NS_ERROR_NOT_INITIALIZED; + } + + // If we don't have an in-flight request, make one + let pendingUrls = Object.keys(this._pendingRequests); + if (!this._currentRequest && (pendingUrls.length > 0)) { + let nextUrl = pendingUrls[0]; + this._currentRequest = this._pendingRequests[nextUrl]; + delete this._pendingRequests[nextUrl]; + } + + if (this._currentRequest) { + try { + this._currentRequest.begin(); + } finally { + // If |begin| fails, we should get rid of our request. + this._currentRequest = null; + } + } + }, + + // Pass the server response status to the RequestBackoff for the given + // gethashUrl and fetch the next pending request, if there is one. + finishRequest: function(url, aStatus) { + this._backoffs[url].noteServerResponse(aStatus); + Services.tm.currentThread.dispatch(this, Ci.nsIThread.DISPATCH_NORMAL); + }, + + // Returns true if we can make a request from the given url, false otherwise. + canMakeRequest: function(aGethashUrl) { + return this._backoffs[aGethashUrl].canMakeRequest(); + }, + + // Notifies the RequestBackoff of a new request so we can throttle based on + // max requests/time period. This must be called before a channel is opened, + // and finishRequest must be called once the response is received. + noteRequest: function(aGethashUrl) { + return this._backoffs[aGethashUrl].noteRequest(); + }, + + observe: function HC_observe(aSubject, aTopic, aData) { + if (aTopic == "quit-application") { + this._shuttingDown = true; + Services.obs.removeObserver(this, "quit-application"); + } + }, +}; + +function HashCompleterRequest(aCompleter, aGethashUrl) { + // HashCompleter object that created this HashCompleterRequest. + this._completer = aCompleter; + // The internal set of hashes and callbacks that this request corresponds to. + this._requests = []; + // nsIChannel that the hash completion query is transmitted over. + this._channel = null; + // Response body of hash completion. Created in onDataAvailable. + this._response = ""; + // Whether we have been informed of a shutdown by the quit-application event. + this._shuttingDown = false; + this.gethashUrl = aGethashUrl; +} +HashCompleterRequest.prototype = { + QueryInterface: XPCOMUtils.generateQI([Ci.nsIRequestObserver, + Ci.nsIStreamListener, + Ci.nsIObserver, + Ci.nsISupports]), + + // This is called by the HashCompleter to add a hash and callback to the + // HashCompleterRequest. It must be called before calling |begin|. + add: function HCR_add(aPartialHash, aCallback) { + this._requests.push({ + partialHash: aPartialHash, + callback: aCallback, + responses: [] + }); + }, + + // This initiates the HTTP request. It can fail due to backoff timings and + // will notify all callbacks as necessary. We notify the backoff object on + // begin. + begin: function HCR_begin() { + if (!this._completer.canMakeRequest(this.gethashUrl)) { + log("Can't make request to " + this.gethashUrl + "\n"); + this.notifyFailure(Cr.NS_ERROR_ABORT); + return; + } + + Services.obs.addObserver(this, "quit-application", false); + + try { + this.openChannel(); + // Notify the RequestBackoff if opening the channel succeeded. At this + // point, finishRequest must be called. + this._completer.noteRequest(this.gethashUrl); + } + catch (err) { + this.notifyFailure(err); + throw err; + } + }, + + notify: function HCR_notify() { + // If we haven't gotten onStopRequest, just cancel. This will call us + // with onStopRequest since we implement nsIStreamListener on the + // channel. + if (this._channel && this._channel.isPending()) { + log("cancelling request to " + this.gethashUrl + "\n"); + Services.telemetry.getHistogramById("URLCLASSIFIER_COMPLETE_TIMEOUT").add(1); + this._channel.cancel(Cr.NS_BINDING_ABORTED); + } + }, + + // Creates an nsIChannel for the request and fills the body. + openChannel: function HCR_openChannel() { + let loadFlags = Ci.nsIChannel.INHIBIT_CACHING | + Ci.nsIChannel.LOAD_BYPASS_CACHE; + + let channel = NetUtil.newChannel({ + uri: this.gethashUrl, + loadUsingSystemPrincipal: true + }); + channel.loadFlags = loadFlags; + + // Disable keepalive. + let httpChannel = channel.QueryInterface(Ci.nsIHttpChannel); + httpChannel.setRequestHeader("Connection", "close", false); + + this._channel = channel; + + let body = this.buildRequest(); + this.addRequestBody(body); + + // Set a timer that cancels the channel after timeout_ms in case we + // don't get a gethash response. + this.timer_ = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + // Ask the timer to use nsITimerCallback (.notify()) when ready + let timeout = Services.prefs.getIntPref( + "urlclassifier.gethash.timeout_ms"); + this.timer_.initWithCallback(this, timeout, this.timer_.TYPE_ONE_SHOT); + channel.asyncOpen2(this); + }, + + // Returns a string for the request body based on the contents of + // this._requests. + buildRequest: function HCR_buildRequest() { + // Sometimes duplicate entries are sent to HashCompleter but we do not need + // to propagate these to the server. (bug 633644) + let prefixes = []; + + for (let i = 0; i < this._requests.length; i++) { + let request = this._requests[i]; + if (prefixes.indexOf(request.partialHash) == -1) { + prefixes.push(request.partialHash); + } + } + + // Randomize the order to obscure the original request from noise + // unbiased Fisher-Yates shuffle + let i = prefixes.length; + while (i--) { + let j = Math.floor(Math.random() * (i + 1)); + let temp = prefixes[i]; + prefixes[i] = prefixes[j]; + prefixes[j] = temp; + } + + let body; + body = PARTIAL_LENGTH + ":" + (PARTIAL_LENGTH * prefixes.length) + + "\n" + prefixes.join(""); + + log('Requesting completions for ' + prefixes.length + ' ' + PARTIAL_LENGTH + '-byte prefixes: ' + body); + return body; + }, + + // Sets the request body of this._channel. + addRequestBody: function HCR_addRequestBody(aBody) { + let inputStream = Cc["@mozilla.org/io/string-input-stream;1"]. + createInstance(Ci.nsIStringInputStream); + + inputStream.setData(aBody, aBody.length); + + let uploadChannel = this._channel.QueryInterface(Ci.nsIUploadChannel); + uploadChannel.setUploadStream(inputStream, "text/plain", -1); + + let httpChannel = this._channel.QueryInterface(Ci.nsIHttpChannel); + httpChannel.requestMethod = "POST"; + }, + + // Parses the response body and eventually adds items to the |responses| array + // for elements of |this._requests|. + handleResponse: function HCR_handleResponse() { + if (this._response == "") { + return; + } + + log('Response: ' + this._response); + let start = 0; + + let length = this._response.length; + while (start != length) { + start = this.handleTable(start); + } + }, + + // This parses a table entry in the response body and calls |handleItem| + // for complete hash in the table entry. + handleTable: function HCR_handleTable(aStart) { + let body = this._response.substring(aStart); + + // deal with new line indexes as there could be + // new line characters in the data parts. + let newlineIndex = body.indexOf("\n"); + if (newlineIndex == -1) { + throw errorWithStack(); + } + let header = body.substring(0, newlineIndex); + let entries = header.split(":"); + if (entries.length != 3) { + throw errorWithStack(); + } + + let list = entries[0]; + let addChunk = parseInt(entries[1]); + let dataLength = parseInt(entries[2]); + + log('Response includes add chunks for ' + list + ': ' + addChunk); + if (dataLength % COMPLETE_LENGTH != 0 || + dataLength == 0 || + dataLength > body.length - (newlineIndex + 1)) { + throw errorWithStack(); + } + + let data = body.substr(newlineIndex + 1, dataLength); + for (let i = 0; i < (dataLength / COMPLETE_LENGTH); i++) { + this.handleItem(data.substr(i * COMPLETE_LENGTH, COMPLETE_LENGTH), list, + addChunk); + } + + return aStart + newlineIndex + 1 + dataLength; + }, + + // This adds a complete hash to any entry in |this._requests| that matches + // the hash. + handleItem: function HCR_handleItem(aData, aTableName, aChunkId) { + for (let i = 0; i < this._requests.length; i++) { + let request = this._requests[i]; + if (aData.substring(0,4) == request.partialHash) { + request.responses.push({ + completeHash: aData, + tableName: aTableName, + chunkId: aChunkId, + }); + } + } + }, + + // notifySuccess and notifyFailure are used to alert the callbacks with + // results. notifySuccess makes |completion| and |completionFinished| calls + // while notifyFailure only makes a |completionFinished| call with the error + // code. + notifySuccess: function HCR_notifySuccess() { + for (let i = 0; i < this._requests.length; i++) { + let request = this._requests[i]; + for (let j = 0; j < request.responses.length; j++) { + let response = request.responses[j]; + request.callback.completion(response.completeHash, response.tableName, + response.chunkId); + } + + request.callback.completionFinished(Cr.NS_OK); + } + }, + + notifyFailure: function HCR_notifyFailure(aStatus) { + log("notifying failure\n"); + for (let i = 0; i < this._requests.length; i++) { + let request = this._requests[i]; + request.callback.completionFinished(aStatus); + } + }, + + onDataAvailable: function HCR_onDataAvailable(aRequest, aContext, + aInputStream, aOffset, aCount) { + let sis = Cc["@mozilla.org/scriptableinputstream;1"]. + createInstance(Ci.nsIScriptableInputStream); + sis.init(aInputStream); + this._response += sis.readBytes(aCount); + }, + + onStartRequest: function HCR_onStartRequest(aRequest, aContext) { + // At this point no data is available for us and we have no reason to + // terminate the connection, so we do nothing until |onStopRequest|. + }, + + onStopRequest: function HCR_onStopRequest(aRequest, aContext, aStatusCode) { + Services.obs.removeObserver(this, "quit-application"); + + if (this._shuttingDown) { + throw Cr.NS_ERROR_ABORT; + } + + // Default HTTP status to service unavailable, in case we can't retrieve + // the true status from the channel. + let httpStatus = 503; + if (Components.isSuccessCode(aStatusCode)) { + let channel = aRequest.QueryInterface(Ci.nsIHttpChannel); + let success = channel.requestSucceeded; + httpStatus = channel.responseStatus; + if (!success) { + aStatusCode = Cr.NS_ERROR_ABORT; + } + } + let success = Components.isSuccessCode(aStatusCode); + log('Received a ' + httpStatus + ' status code from the gethash server (success=' + success + ').'); + + let histogram = + Services.telemetry.getHistogramById("URLCLASSIFIER_COMPLETE_REMOTE_STATUS"); + histogram.add(httpStatusToBucket(httpStatus)); + Services.telemetry.getHistogramById("URLCLASSIFIER_COMPLETE_TIMEOUT").add(0); + + // Notify the RequestBackoff once a response is received. + this._completer.finishRequest(this.gethashUrl, httpStatus); + + if (success) { + try { + this.handleResponse(); + } + catch (err) { + log(err.stack); + aStatusCode = err.value; + success = false; + } + } + + if (success) { + this.notifySuccess(); + } else { + this.notifyFailure(aStatusCode); + } + }, + + observe: function HCR_observe(aSubject, aTopic, aData) { + if (aTopic == "quit-application") { + this._shuttingDown = true; + if (this._channel) { + this._channel.cancel(Cr.NS_ERROR_ABORT); + } + + Services.obs.removeObserver(this, "quit-application"); + } + }, +}; + +// Converts a URL safe base64 string to a normal base64 string. Will not change +// normal base64 strings. This is modelled after the same function in +// nsUrlClassifierUtils.h. +function unUrlsafeBase64(aStr) { + return !aStr ? "" : aStr.replace(/-/g, "+") + .replace(/_/g, "/"); +} + +function errorWithStack() { + let err = new Error(); + err.value = Cr.NS_ERROR_FAILURE; + return err; +} + +this.NSGetFactory = XPCOMUtils.generateNSGetFactory([HashCompleter]); diff --git a/toolkit/components/url-classifier/nsUrlClassifierLib.js b/toolkit/components/url-classifier/nsUrlClassifierLib.js new file mode 100644 index 000000000..bb0d2b421 --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierLib.js @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// We wastefully reload the same JS files across components. This puts all +// the common JS files used by safebrowsing and url-classifier into a +// single component. + +const Cc = Components.classes; +const Ci = Components.interfaces; +const G_GDEBUG = false; + +Components.utils.import("resource://gre/modules/XPCOMUtils.jsm"); + +#include ./content/moz/lang.js +#include ./content/moz/preferences.js +#include ./content/moz/debug.js +#include ./content/moz/alarm.js +#include ./content/moz/cryptohasher.js +#include ./content/moz/observer.js +#include ./content/moz/protocol4.js + +#include ./content/request-backoff.js +#include ./content/xml-fetcher.js + +// Wrap a general-purpose |RequestBackoff| to a v4-specific one +// since both listmanager and hashcompleter would use it. +// Note that |maxRequests| and |requestPeriod| is still configurable +// to throttle pending requests. +function RequestBackoffV4(maxRequests, requestPeriod) { + let rand = Math.random(); + let retryInterval = Math.floor(15 * 60 * 1000 * (rand + 1)); // 15 ~ 30 min. + let backoffInterval = Math.floor(30 * 60 * 1000 * (rand + 1)); // 30 ~ 60 min. + + return new RequestBackoff(2 /* max errors */, + retryInterval /* retry interval, 15~30 min */, + maxRequests /* num requests */, + requestPeriod /* request time, 60 min */, + backoffInterval /* backoff interval, 60 min */, + 24 * 60 * 60 * 1000 /* max backoff, 24hr */); +} + +// Expose this whole component. +var lib = this; + +function UrlClassifierLib() { + this.wrappedJSObject = lib; +} +UrlClassifierLib.prototype.classID = Components.ID("{26a4a019-2827-4a89-a85c-5931a678823a}"); +UrlClassifierLib.prototype.QueryInterface = XPCOMUtils.generateQI([]); + +this.NSGetFactory = XPCOMUtils.generateNSGetFactory([UrlClassifierLib]); diff --git a/toolkit/components/url-classifier/nsUrlClassifierListManager.js b/toolkit/components/url-classifier/nsUrlClassifierListManager.js new file mode 100644 index 000000000..7b3c181af --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierListManager.js @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +const Cc = Components.classes; +const Ci = Components.interfaces; + +Components.utils.import("resource://gre/modules/XPCOMUtils.jsm"); + +#include ./content/listmanager.js + +var modScope = this; +function Init() { + // Pull the library in. + var jslib = Cc["@mozilla.org/url-classifier/jslib;1"] + .getService().wrappedJSObject; + Function.prototype.inherits = function(parentCtor) { + var tempCtor = function(){}; + tempCtor.prototype = parentCtor.prototype; + this.superClass_ = parentCtor.prototype; + this.prototype = new tempCtor(); + }, + modScope.G_Preferences = jslib.G_Preferences; + modScope.G_PreferenceObserver = jslib.G_PreferenceObserver; + modScope.G_ObserverServiceObserver = jslib.G_ObserverServiceObserver; + modScope.G_Debug = jslib.G_Debug; + modScope.G_Assert = jslib.G_Assert; + modScope.G_debugService = jslib.G_debugService; + modScope.G_Alarm = jslib.G_Alarm; + modScope.BindToObject = jslib.BindToObject; + modScope.PROT_XMLFetcher = jslib.PROT_XMLFetcher; + modScope.RequestBackoffV4 = jslib.RequestBackoffV4; + + // We only need to call Init once. + modScope.Init = function() {}; +} + +function RegistrationData() +{ +} +RegistrationData.prototype = { + classID: Components.ID("{ca168834-cc00-48f9-b83c-fd018e58cae3}"), + _xpcom_factory: { + createInstance: function(outer, iid) { + if (outer != null) + throw Components.results.NS_ERROR_NO_AGGREGATION; + Init(); + return (new PROT_ListManager()).QueryInterface(iid); + } + }, +}; + +this.NSGetFactory = XPCOMUtils.generateNSGetFactory([RegistrationData]); diff --git a/toolkit/components/url-classifier/nsUrlClassifierPrefixSet.cpp b/toolkit/components/url-classifier/nsUrlClassifierPrefixSet.cpp new file mode 100644 index 000000000..874565470 --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierPrefixSet.cpp @@ -0,0 +1,526 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsUrlClassifierPrefixSet.h" +#include "nsIUrlClassifierPrefixSet.h" +#include "nsCOMPtr.h" +#include "nsDebug.h" +#include "nsPrintfCString.h" +#include "nsTArray.h" +#include "nsString.h" +#include "nsIFile.h" +#include "nsToolkitCompsCID.h" +#include "nsTArray.h" +#include "nsThreadUtils.h" +#include "nsNetUtil.h" +#include "nsISeekableStream.h" +#include "nsIBufferedStreams.h" +#include "nsIFileStreams.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/Telemetry.h" +#include "mozilla/FileUtils.h" +#include "mozilla/Logging.h" +#include "mozilla/Unused.h" +#include <algorithm> + +using namespace mozilla; + +// MOZ_LOG=UrlClassifierPrefixSet:5 +static LazyLogModule gUrlClassifierPrefixSetLog("UrlClassifierPrefixSet"); +#define LOG(args) MOZ_LOG(gUrlClassifierPrefixSetLog, mozilla::LogLevel::Debug, args) +#define LOG_ENABLED() MOZ_LOG_TEST(gUrlClassifierPrefixSetLog, mozilla::LogLevel::Debug) + +NS_IMPL_ISUPPORTS( + nsUrlClassifierPrefixSet, nsIUrlClassifierPrefixSet, nsIMemoryReporter) + +// Definition required due to std::max<>() +const uint32_t nsUrlClassifierPrefixSet::MAX_BUFFER_SIZE; + +nsUrlClassifierPrefixSet::nsUrlClassifierPrefixSet() + : mLock("nsUrlClassifierPrefixSet.mLock") + , mTotalPrefixes(0) + , mMemoryReportPath() +{ +} + +NS_IMETHODIMP +nsUrlClassifierPrefixSet::Init(const nsACString& aName) +{ + mMemoryReportPath = + nsPrintfCString( + "explicit/storage/prefix-set/%s", + (!aName.IsEmpty() ? PromiseFlatCString(aName).get() : "?!") + ); + + RegisterWeakMemoryReporter(this); + + return NS_OK; +} + +nsUrlClassifierPrefixSet::~nsUrlClassifierPrefixSet() +{ + UnregisterWeakMemoryReporter(this); +} + +NS_IMETHODIMP +nsUrlClassifierPrefixSet::SetPrefixes(const uint32_t* aArray, uint32_t aLength) +{ + MutexAutoLock lock(mLock); + + nsresult rv = NS_OK; + + if (aLength <= 0) { + if (mIndexPrefixes.Length() > 0) { + LOG(("Clearing PrefixSet")); + mIndexDeltas.Clear(); + mIndexPrefixes.Clear(); + mTotalPrefixes = 0; + } + } else { + rv = MakePrefixSet(aArray, aLength); + } + + return rv; +} + +nsresult +nsUrlClassifierPrefixSet::MakePrefixSet(const uint32_t* aPrefixes, uint32_t aLength) +{ + mLock.AssertCurrentThreadOwns(); + + if (aLength == 0) { + return NS_OK; + } + +#ifdef DEBUG + for (uint32_t i = 1; i < aLength; i++) { + MOZ_ASSERT(aPrefixes[i] >= aPrefixes[i-1]); + } +#endif + + mIndexPrefixes.Clear(); + mIndexDeltas.Clear(); + mTotalPrefixes = aLength; + + mIndexPrefixes.AppendElement(aPrefixes[0]); + mIndexDeltas.AppendElement(); + + uint32_t numOfDeltas = 0; + uint32_t totalDeltas = 0; + uint32_t previousItem = aPrefixes[0]; + for (uint32_t i = 1; i < aLength; i++) { + if ((numOfDeltas >= DELTAS_LIMIT) || + (aPrefixes[i] - previousItem >= MAX_INDEX_DIFF)) { + // Compact the previous element. + // Note there is always at least one element when we get here, + // because we created the first element before the loop. + mIndexDeltas.LastElement().Compact(); + mIndexDeltas.AppendElement(); + mIndexPrefixes.AppendElement(aPrefixes[i]); + numOfDeltas = 0; + } else { + uint16_t delta = aPrefixes[i] - previousItem; + mIndexDeltas.LastElement().AppendElement(delta); + numOfDeltas++; + totalDeltas++; + } + previousItem = aPrefixes[i]; + } + + mIndexDeltas.LastElement().Compact(); + mIndexDeltas.Compact(); + mIndexPrefixes.Compact(); + + LOG(("Total number of indices: %d", aLength)); + LOG(("Total number of deltas: %d", totalDeltas)); + LOG(("Total number of delta chunks: %d", mIndexDeltas.Length())); + + return NS_OK; +} + +nsresult +nsUrlClassifierPrefixSet::GetPrefixesNative(FallibleTArray<uint32_t>& outArray) +{ + MutexAutoLock lock(mLock); + + if (!outArray.SetLength(mTotalPrefixes, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + uint32_t prefixIdxLength = mIndexPrefixes.Length(); + uint32_t prefixCnt = 0; + + for (uint32_t i = 0; i < prefixIdxLength; i++) { + uint32_t prefix = mIndexPrefixes[i]; + + outArray[prefixCnt++] = prefix; + for (uint32_t j = 0; j < mIndexDeltas[i].Length(); j++) { + prefix += mIndexDeltas[i][j]; + outArray[prefixCnt++] = prefix; + } + } + + NS_ASSERTION(mTotalPrefixes == prefixCnt, "Lengths are inconsistent"); + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierPrefixSet::GetPrefixes(uint32_t* aCount, + uint32_t** aPrefixes) +{ + // No need to get mLock here because this function does not directly touch + // the class's data members. (GetPrefixesNative() will get mLock, however.) + + NS_ENSURE_ARG_POINTER(aCount); + *aCount = 0; + NS_ENSURE_ARG_POINTER(aPrefixes); + *aPrefixes = nullptr; + + FallibleTArray<uint32_t> prefixes; + nsresult rv = GetPrefixesNative(prefixes); + if (NS_FAILED(rv)) { + return rv; + } + + uint64_t itemCount = prefixes.Length(); + uint32_t* prefixArray = static_cast<uint32_t*>(moz_xmalloc(itemCount * sizeof(uint32_t))); + NS_ENSURE_TRUE(prefixArray, NS_ERROR_OUT_OF_MEMORY); + + memcpy(prefixArray, prefixes.Elements(), sizeof(uint32_t) * itemCount); + + *aCount = itemCount; + *aPrefixes = prefixArray; + + return NS_OK; +} + +uint32_t nsUrlClassifierPrefixSet::BinSearch(uint32_t start, + uint32_t end, + uint32_t target) +{ + mLock.AssertCurrentThreadOwns(); + + while (start != end && end >= start) { + uint32_t i = start + ((end - start) >> 1); + uint32_t value = mIndexPrefixes[i]; + if (value < target) { + start = i + 1; + } else if (value > target) { + end = i - 1; + } else { + return i; + } + } + return end; +} + +NS_IMETHODIMP +nsUrlClassifierPrefixSet::Contains(uint32_t aPrefix, bool* aFound) +{ + MutexAutoLock lock(mLock); + + *aFound = false; + + if (mIndexPrefixes.Length() == 0) { + return NS_OK; + } + + uint32_t target = aPrefix; + + // We want to do a "Price is Right" binary search, that is, we want to find + // the index of the value either equal to the target or the closest value + // that is less than the target. + // + if (target < mIndexPrefixes[0]) { + return NS_OK; + } + + // |binsearch| does not necessarily return the correct index (when the + // target is not found) but rather it returns an index at least one away + // from the correct index. + // Because of this, we need to check if the target lies before the beginning + // of the indices. + + uint32_t i = BinSearch(0, mIndexPrefixes.Length() - 1, target); + if (mIndexPrefixes[i] > target && i > 0) { + i--; + } + + // Now search through the deltas for the target. + uint32_t diff = target - mIndexPrefixes[i]; + uint32_t deltaSize = mIndexDeltas[i].Length(); + uint32_t deltaIndex = 0; + + while (diff > 0 && deltaIndex < deltaSize) { + diff -= mIndexDeltas[i][deltaIndex]; + deltaIndex++; + } + + if (diff == 0) { + *aFound = true; + } + + return NS_OK; +} + +MOZ_DEFINE_MALLOC_SIZE_OF(UrlClassifierMallocSizeOf) + +NS_IMETHODIMP +nsUrlClassifierPrefixSet::CollectReports(nsIHandleReportCallback* aHandleReport, + nsISupports* aData, bool aAnonymize) +{ + MOZ_ASSERT(NS_IsMainThread()); + + // No need to get mLock here because this function does not directly touch + // the class's data members. (SizeOfIncludingThis() will get mLock, however.) + + aHandleReport->Callback( + EmptyCString(), mMemoryReportPath, KIND_HEAP, UNITS_BYTES, + SizeOfIncludingThis(UrlClassifierMallocSizeOf), + NS_LITERAL_CSTRING("Memory used by the prefix set for a URL classifier."), + aData); + + return NS_OK; +} + +size_t +nsUrlClassifierPrefixSet::SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf) +{ + MutexAutoLock lock(mLock); + + size_t n = 0; + n += aMallocSizeOf(this); + n += mIndexDeltas.ShallowSizeOfExcludingThis(aMallocSizeOf); + for (uint32_t i = 0; i < mIndexDeltas.Length(); i++) { + n += mIndexDeltas[i].ShallowSizeOfExcludingThis(aMallocSizeOf); + } + n += mIndexPrefixes.ShallowSizeOfExcludingThis(aMallocSizeOf); + return n; +} + +NS_IMETHODIMP +nsUrlClassifierPrefixSet::IsEmpty(bool * aEmpty) +{ + MutexAutoLock lock(mLock); + + *aEmpty = (mIndexPrefixes.Length() == 0); + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierPrefixSet::LoadFromFile(nsIFile* aFile) +{ + MutexAutoLock lock(mLock); + + Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_FILELOAD_TIME> timer; + + nsCOMPtr<nsIInputStream> localInFile; + nsresult rv = NS_NewLocalFileInputStream(getter_AddRefs(localInFile), aFile, + PR_RDONLY | nsIFile::OS_READAHEAD); + NS_ENSURE_SUCCESS(rv, rv); + + // Calculate how big the file is, make sure our read buffer isn't bigger + // than the file itself which is just wasting memory. + int64_t fileSize; + rv = aFile->GetFileSize(&fileSize); + NS_ENSURE_SUCCESS(rv, rv); + + if (fileSize < 0 || fileSize > UINT32_MAX) { + return NS_ERROR_FAILURE; + } + + uint32_t bufferSize = std::min<uint32_t>(static_cast<uint32_t>(fileSize), + MAX_BUFFER_SIZE); + + // Convert to buffered stream + nsCOMPtr<nsIInputStream> in = NS_BufferInputStream(localInFile, bufferSize); + + rv = LoadPrefixes(in); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierPrefixSet::StoreToFile(nsIFile* aFile) +{ + MutexAutoLock lock(mLock); + + nsCOMPtr<nsIOutputStream> localOutFile; + nsresult rv = NS_NewLocalFileOutputStream(getter_AddRefs(localOutFile), aFile, + PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE); + NS_ENSURE_SUCCESS(rv, rv); + + uint32_t fileSize; + + // Preallocate the file storage + { + nsCOMPtr<nsIFileOutputStream> fos(do_QueryInterface(localOutFile)); + Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_FALLOCATE_TIME> timer; + + fileSize = CalculatePreallocateSize(); + + // Ignore failure, the preallocation is a hint and we write out the entire + // file later on + Unused << fos->Preallocate(fileSize); + } + + // Convert to buffered stream + nsCOMPtr<nsIOutputStream> out = + NS_BufferOutputStream(localOutFile, std::min(fileSize, MAX_BUFFER_SIZE)); + + rv = WritePrefixes(out); + NS_ENSURE_SUCCESS(rv, rv); + + LOG(("Saving PrefixSet successful\n")); + + return NS_OK; +} + +nsresult +nsUrlClassifierPrefixSet::LoadPrefixes(nsIInputStream* in) +{ + uint32_t magic; + uint32_t read; + + nsresult rv = in->Read(reinterpret_cast<char*>(&magic), sizeof(uint32_t), &read); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(read == sizeof(uint32_t), NS_ERROR_FAILURE); + + if (magic == PREFIXSET_VERSION_MAGIC) { + uint32_t indexSize; + uint32_t deltaSize; + + rv = in->Read(reinterpret_cast<char*>(&indexSize), sizeof(uint32_t), &read); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(read == sizeof(uint32_t), NS_ERROR_FAILURE); + + rv = in->Read(reinterpret_cast<char*>(&deltaSize), sizeof(uint32_t), &read); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(read == sizeof(uint32_t), NS_ERROR_FAILURE); + + if (indexSize == 0) { + LOG(("stored PrefixSet is empty!")); + return NS_OK; + } + + if (deltaSize > (indexSize * DELTAS_LIMIT)) { + return NS_ERROR_FILE_CORRUPTED; + } + + nsTArray<uint32_t> indexStarts; + indexStarts.SetLength(indexSize); + mIndexPrefixes.SetLength(indexSize); + mIndexDeltas.SetLength(indexSize); + + mTotalPrefixes = indexSize; + + uint32_t toRead = indexSize*sizeof(uint32_t); + rv = in->Read(reinterpret_cast<char*>(mIndexPrefixes.Elements()), toRead, &read); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(read == toRead, NS_ERROR_FAILURE); + + rv = in->Read(reinterpret_cast<char*>(indexStarts.Elements()), toRead, &read); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(read == toRead, NS_ERROR_FAILURE); + + if (indexSize != 0 && indexStarts[0] != 0) { + return NS_ERROR_FILE_CORRUPTED; + } + for (uint32_t i = 0; i < indexSize; i++) { + uint32_t numInDelta = i == indexSize - 1 ? deltaSize - indexStarts[i] + : indexStarts[i + 1] - indexStarts[i]; + if (numInDelta > DELTAS_LIMIT) { + return NS_ERROR_FILE_CORRUPTED; + } + if (numInDelta > 0) { + mIndexDeltas[i].SetLength(numInDelta); + mTotalPrefixes += numInDelta; + toRead = numInDelta * sizeof(uint16_t); + rv = in->Read(reinterpret_cast<char*>(mIndexDeltas[i].Elements()), toRead, &read); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(read == toRead, NS_ERROR_FAILURE); + } + } + } else { + LOG(("Version magic mismatch, not loading")); + return NS_ERROR_FILE_CORRUPTED; + } + + MOZ_ASSERT(mIndexPrefixes.Length() == mIndexDeltas.Length()); + LOG(("Loading PrefixSet successful")); + + return NS_OK; +} + +uint32_t +nsUrlClassifierPrefixSet::CalculatePreallocateSize() +{ + uint32_t fileSize = 4 * sizeof(uint32_t); + uint32_t deltas = mTotalPrefixes - mIndexPrefixes.Length(); + fileSize += 2 * mIndexPrefixes.Length() * sizeof(uint32_t); + fileSize += deltas * sizeof(uint16_t); + return fileSize; +} + +nsresult +nsUrlClassifierPrefixSet::WritePrefixes(nsIOutputStream* out) +{ + uint32_t written; + uint32_t writelen = sizeof(uint32_t); + uint32_t magic = PREFIXSET_VERSION_MAGIC; + nsresult rv = out->Write(reinterpret_cast<char*>(&magic), writelen, &written); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE); + + uint32_t indexSize = mIndexPrefixes.Length(); + uint32_t indexDeltaSize = mIndexDeltas.Length(); + uint32_t totalDeltas = 0; + + // Store the shape of mIndexDeltas by noting at which "count" of total + // indexes a new subarray starts. This is slightly cumbersome but keeps + // file format compatibility. + // If we ever update the format, we can gain space by storing the delta + // subarray sizes, which fit in bytes. + nsTArray<uint32_t> indexStarts; + indexStarts.AppendElement(0); + + for (uint32_t i = 0; i < indexDeltaSize; i++) { + uint32_t deltaLength = mIndexDeltas[i].Length(); + totalDeltas += deltaLength; + indexStarts.AppendElement(totalDeltas); + } + + rv = out->Write(reinterpret_cast<char*>(&indexSize), writelen, &written); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE); + + rv = out->Write(reinterpret_cast<char*>(&totalDeltas), writelen, &written); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE); + + writelen = indexSize * sizeof(uint32_t); + rv = out->Write(reinterpret_cast<char*>(mIndexPrefixes.Elements()), writelen, &written); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE); + + rv = out->Write(reinterpret_cast<char*>(indexStarts.Elements()), writelen, &written); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE); + + if (totalDeltas > 0) { + for (uint32_t i = 0; i < indexDeltaSize; i++) { + writelen = mIndexDeltas[i].Length() * sizeof(uint16_t); + rv = out->Write(reinterpret_cast<char*>(mIndexDeltas[i].Elements()), writelen, &written); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(written == writelen, NS_ERROR_FAILURE); + } + } + + LOG(("Saving PrefixSet successful\n")); + + return NS_OK; +} diff --git a/toolkit/components/url-classifier/nsUrlClassifierPrefixSet.h b/toolkit/components/url-classifier/nsUrlClassifierPrefixSet.h new file mode 100644 index 000000000..8627b75d0 --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierPrefixSet.h @@ -0,0 +1,89 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsUrlClassifierPrefixSet_h_ +#define nsUrlClassifierPrefixSet_h_ + +#include "nsISupportsUtils.h" +#include "nsID.h" +#include "nsIFile.h" +#include "nsIMemoryReporter.h" +#include "nsIMutableArray.h" +#include "nsIFileStreams.h" +#include "nsIUrlClassifierPrefixSet.h" +#include "nsTArray.h" +#include "nsToolkitCompsCID.h" +#include "mozilla/FileUtils.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/Mutex.h" + +namespace mozilla { +namespace safebrowsing { + +class VariableLengthPrefixSet; + +} // namespace safebrowsing +} // namespace mozilla + +class nsUrlClassifierPrefixSet final + : public nsIUrlClassifierPrefixSet + , public nsIMemoryReporter +{ +public: + nsUrlClassifierPrefixSet(); + + NS_IMETHOD Init(const nsACString& aName) override; + NS_IMETHOD SetPrefixes(const uint32_t* aArray, uint32_t aLength) override; + NS_IMETHOD GetPrefixes(uint32_t* aCount, uint32_t** aPrefixes) override; + NS_IMETHOD Contains(uint32_t aPrefix, bool* aFound) override; + NS_IMETHOD IsEmpty(bool* aEmpty) override; + NS_IMETHOD LoadFromFile(nsIFile* aFile) override; + NS_IMETHOD StoreToFile(nsIFile* aFile) override; + + nsresult GetPrefixesNative(FallibleTArray<uint32_t>& outArray); + + size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf); + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIMEMORYREPORTER + + friend class mozilla::safebrowsing::VariableLengthPrefixSet; + +private: + virtual ~nsUrlClassifierPrefixSet(); + + static const uint32_t MAX_BUFFER_SIZE = 64 * 1024; + static const uint32_t DELTAS_LIMIT = 120; + static const uint32_t MAX_INDEX_DIFF = (1 << 16); + static const uint32_t PREFIXSET_VERSION_MAGIC = 1; + + nsresult MakePrefixSet(const uint32_t* aArray, uint32_t aLength); + uint32_t BinSearch(uint32_t start, uint32_t end, uint32_t target); + + uint32_t CalculatePreallocateSize(); + nsresult WritePrefixes(nsIOutputStream* out); + nsresult LoadPrefixes(nsIInputStream* in); + + // Lock to prevent races between the url-classifier thread (which does most + // of the operations) and the main thread (which does memory reporting). + // It should be held for all operations between Init() and destruction that + // touch this class's data members. + mozilla::Mutex mLock; + // list of fully stored prefixes, that also form the + // start of a run of deltas in mIndexDeltas. + nsTArray<uint32_t> mIndexPrefixes; + // array containing arrays of deltas from indices. + // Index to the place that matches the closest lower + // prefix from mIndexPrefix. Then every "delta" corresponds + // to a prefix in the PrefixSet. + nsTArray<nsTArray<uint16_t> > mIndexDeltas; + // how many prefixes we have. + uint32_t mTotalPrefixes; + + nsCString mMemoryReportPath; +}; + +#endif diff --git a/toolkit/components/url-classifier/nsUrlClassifierProxies.cpp b/toolkit/components/url-classifier/nsUrlClassifierProxies.cpp new file mode 100644 index 000000000..90cb967ea --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierProxies.cpp @@ -0,0 +1,356 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsUrlClassifierProxies.h" +#include "nsUrlClassifierDBService.h" + +#include "mozilla/SyncRunnable.h" + +using namespace mozilla::safebrowsing; +using mozilla::NewRunnableMethod; + +static nsresult +DispatchToWorkerThread(nsIRunnable* r) +{ + nsIThread* t = nsUrlClassifierDBService::BackgroundThread(); + if (!t) + return NS_ERROR_FAILURE; + + return t->Dispatch(r, NS_DISPATCH_NORMAL); +} + +NS_IMPL_ISUPPORTS(UrlClassifierDBServiceWorkerProxy, nsIUrlClassifierDBService) + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::Lookup(nsIPrincipal* aPrincipal, + const nsACString& aTables, + nsIUrlClassifierCallback* aCB) +{ + nsCOMPtr<nsIRunnable> r = new LookupRunnable(mTarget, aPrincipal, aTables, + aCB); + return DispatchToWorkerThread(r); +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::LookupRunnable::Run() +{ + (void) mTarget->Lookup(mPrincipal, mLookupTables, mCB); + return NS_OK; +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::GetTables(nsIUrlClassifierCallback* aCB) +{ + nsCOMPtr<nsIRunnable> r = new GetTablesRunnable(mTarget, aCB); + return DispatchToWorkerThread(r); +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::GetTablesRunnable::Run() +{ + mTarget->GetTables(mCB); + return NS_OK; +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::SetHashCompleter + (const nsACString&, nsIUrlClassifierHashCompleter*) +{ + NS_NOTREACHED("This method should not be called!"); + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::BeginUpdate + (nsIUrlClassifierUpdateObserver* aUpdater, + const nsACString& aTables) +{ + nsCOMPtr<nsIRunnable> r = new BeginUpdateRunnable(mTarget, aUpdater, + aTables); + return DispatchToWorkerThread(r); +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::BeginUpdateRunnable::Run() +{ + mTarget->BeginUpdate(mUpdater, mTables); + return NS_OK; +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::BeginStream(const nsACString& aTable) +{ + nsCOMPtr<nsIRunnable> r = + new BeginStreamRunnable(mTarget, aTable); + return DispatchToWorkerThread(r); +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::BeginStreamRunnable::Run() +{ + mTarget->BeginStream(mTable); + return NS_OK; +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::UpdateStream(const nsACString& aUpdateChunk) +{ + nsCOMPtr<nsIRunnable> r = + new UpdateStreamRunnable(mTarget, aUpdateChunk); + return DispatchToWorkerThread(r); +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::UpdateStreamRunnable::Run() +{ + mTarget->UpdateStream(mUpdateChunk); + return NS_OK; +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::FinishStream() +{ + nsCOMPtr<nsIRunnable> r = + NewRunnableMethod(mTarget, + &nsUrlClassifierDBServiceWorker::FinishStream); + return DispatchToWorkerThread(r); +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::DoLocalLookupRunnable::Run() +{ + mTarget->DoLocalLookup(mSpec, mTables, mResults); + return NS_OK; +} + +nsresult +UrlClassifierDBServiceWorkerProxy::DoLocalLookup(const nsACString& spec, + const nsACString& tables, + LookupResultArray* results) + +{ + // Run synchronously on background thread. NS_DISPATCH_SYNC does *not* do + // what we want -- it continues processing events on the main thread loop + // before the Dispatch returns. + nsCOMPtr<nsIRunnable> r = new DoLocalLookupRunnable(mTarget, spec, tables, results); + nsIThread* t = nsUrlClassifierDBService::BackgroundThread(); + if (!t) + return NS_ERROR_FAILURE; + + mozilla::SyncRunnable::DispatchToThread(t, r); + return NS_OK; +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::FinishUpdate() +{ + nsCOMPtr<nsIRunnable> r = + NewRunnableMethod(mTarget, + &nsUrlClassifierDBServiceWorker::FinishUpdate); + return DispatchToWorkerThread(r); +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::CancelUpdate() +{ + nsCOMPtr<nsIRunnable> r = + NewRunnableMethod(mTarget, + &nsUrlClassifierDBServiceWorker::CancelUpdate); + return DispatchToWorkerThread(r); +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::ResetDatabase() +{ + nsCOMPtr<nsIRunnable> r = + NewRunnableMethod(mTarget, + &nsUrlClassifierDBServiceWorker::ResetDatabase); + return DispatchToWorkerThread(r); +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::ReloadDatabase() +{ + nsCOMPtr<nsIRunnable> r = + NewRunnableMethod(mTarget, + &nsUrlClassifierDBServiceWorker::ReloadDatabase); + return DispatchToWorkerThread(r); +} + +nsresult +UrlClassifierDBServiceWorkerProxy::OpenDb() +{ + nsCOMPtr<nsIRunnable> r = + NewRunnableMethod(mTarget, + &nsUrlClassifierDBServiceWorker::OpenDb); + return DispatchToWorkerThread(r); +} + +nsresult +UrlClassifierDBServiceWorkerProxy::CloseDb() +{ + nsCOMPtr<nsIRunnable> r = + NewRunnableMethod(mTarget, + &nsUrlClassifierDBServiceWorker::CloseDb); + return DispatchToWorkerThread(r); +} + +nsresult +UrlClassifierDBServiceWorkerProxy::CacheCompletions(CacheResultArray * aEntries) +{ + nsCOMPtr<nsIRunnable> r = new CacheCompletionsRunnable(mTarget, aEntries); + return DispatchToWorkerThread(r); +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::CacheCompletionsRunnable::Run() +{ + mTarget->CacheCompletions(mEntries); + return NS_OK; +} + +nsresult +UrlClassifierDBServiceWorkerProxy::CacheMisses(PrefixArray * aEntries) +{ + nsCOMPtr<nsIRunnable> r = new CacheMissesRunnable(mTarget, aEntries); + return DispatchToWorkerThread(r); +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::CacheMissesRunnable::Run() +{ + mTarget->CacheMisses(mEntries); + return NS_OK; +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::SetLastUpdateTime(const nsACString& table, + uint64_t lastUpdateTime) +{ + nsCOMPtr<nsIRunnable> r = + new SetLastUpdateTimeRunnable(mTarget, table, lastUpdateTime); + return DispatchToWorkerThread(r); +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::SetLastUpdateTimeRunnable::Run() +{ + mTarget->SetLastUpdateTime(mTable, mUpdateTime); + return NS_OK; +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::ClearLastResults() +{ + nsCOMPtr<nsIRunnable> r = new ClearLastResultsRunnable(mTarget); + return DispatchToWorkerThread(r); +} + +NS_IMETHODIMP +UrlClassifierDBServiceWorkerProxy::ClearLastResultsRunnable::Run() +{ + return mTarget->ClearLastResults(); +} + +NS_IMPL_ISUPPORTS(UrlClassifierLookupCallbackProxy, + nsIUrlClassifierLookupCallback) + +NS_IMETHODIMP +UrlClassifierLookupCallbackProxy::LookupComplete + (LookupResultArray * aResults) +{ + nsCOMPtr<nsIRunnable> r = new LookupCompleteRunnable(mTarget, aResults); + return NS_DispatchToMainThread(r); +} + +NS_IMETHODIMP +UrlClassifierLookupCallbackProxy::LookupCompleteRunnable::Run() +{ + mTarget->LookupComplete(mResults); + return NS_OK; +} + +NS_IMPL_ISUPPORTS(UrlClassifierCallbackProxy, + nsIUrlClassifierCallback) + +NS_IMETHODIMP +UrlClassifierCallbackProxy::HandleEvent(const nsACString& aValue) +{ + nsCOMPtr<nsIRunnable> r = new HandleEventRunnable(mTarget, aValue); + return NS_DispatchToMainThread(r); +} + +NS_IMETHODIMP +UrlClassifierCallbackProxy::HandleEventRunnable::Run() +{ + mTarget->HandleEvent(mValue); + return NS_OK; +} + +NS_IMPL_ISUPPORTS(UrlClassifierUpdateObserverProxy, + nsIUrlClassifierUpdateObserver) + +NS_IMETHODIMP +UrlClassifierUpdateObserverProxy::UpdateUrlRequested + (const nsACString& aURL, + const nsACString& aTable) +{ + nsCOMPtr<nsIRunnable> r = + new UpdateUrlRequestedRunnable(mTarget, aURL, aTable); + return NS_DispatchToMainThread(r); +} + +NS_IMETHODIMP +UrlClassifierUpdateObserverProxy::UpdateUrlRequestedRunnable::Run() +{ + mTarget->UpdateUrlRequested(mURL, mTable); + return NS_OK; +} + +NS_IMETHODIMP +UrlClassifierUpdateObserverProxy::StreamFinished(nsresult aStatus, + uint32_t aDelay) +{ + nsCOMPtr<nsIRunnable> r = + new StreamFinishedRunnable(mTarget, aStatus, aDelay); + return NS_DispatchToMainThread(r); +} + +NS_IMETHODIMP +UrlClassifierUpdateObserverProxy::StreamFinishedRunnable::Run() +{ + mTarget->StreamFinished(mStatus, mDelay); + return NS_OK; +} + +NS_IMETHODIMP +UrlClassifierUpdateObserverProxy::UpdateError(nsresult aError) +{ + nsCOMPtr<nsIRunnable> r = + new UpdateErrorRunnable(mTarget, aError); + return NS_DispatchToMainThread(r); +} + +NS_IMETHODIMP +UrlClassifierUpdateObserverProxy::UpdateErrorRunnable::Run() +{ + mTarget->UpdateError(mError); + return NS_OK; +} + +NS_IMETHODIMP +UrlClassifierUpdateObserverProxy::UpdateSuccess(uint32_t aRequestedTimeout) +{ + nsCOMPtr<nsIRunnable> r = + new UpdateSuccessRunnable(mTarget, aRequestedTimeout); + return NS_DispatchToMainThread(r); +} + +NS_IMETHODIMP +UrlClassifierUpdateObserverProxy::UpdateSuccessRunnable::Run() +{ + mTarget->UpdateSuccess(mRequestedTimeout); + return NS_OK; +} diff --git a/toolkit/components/url-classifier/nsUrlClassifierProxies.h b/toolkit/components/url-classifier/nsUrlClassifierProxies.h new file mode 100644 index 000000000..3a6c39434 --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierProxies.h @@ -0,0 +1,373 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsUrlClassifierProxies_h +#define nsUrlClassifierProxies_h + +#include "nsIUrlClassifierDBService.h" +#include "nsUrlClassifierDBService.h" +#include "nsProxyRelease.h" +#include "nsThreadUtils.h" +#include "mozilla/Attributes.h" +#include "nsIPrincipal.h" +#include "LookupCache.h" + + +/** + * Thread proxy from the main thread to the worker thread. + */ +class UrlClassifierDBServiceWorkerProxy final : public nsIUrlClassifierDBService +{ +public: + explicit UrlClassifierDBServiceWorkerProxy(nsUrlClassifierDBServiceWorker* aTarget) + : mTarget(aTarget) + { } + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIURLCLASSIFIERDBSERVICE + + class LookupRunnable : public mozilla::Runnable + { + public: + LookupRunnable(nsUrlClassifierDBServiceWorker* aTarget, + nsIPrincipal* aPrincipal, + const nsACString& aTables, + nsIUrlClassifierCallback* aCB) + : mTarget(aTarget) + , mPrincipal(aPrincipal) + , mLookupTables(aTables) + , mCB(aCB) + { } + + NS_DECL_NSIRUNNABLE + + private: + RefPtr<nsUrlClassifierDBServiceWorker> mTarget; + nsCOMPtr<nsIPrincipal> mPrincipal; + nsCString mLookupTables; + nsCOMPtr<nsIUrlClassifierCallback> mCB; + }; + + class GetTablesRunnable : public mozilla::Runnable + { + public: + GetTablesRunnable(nsUrlClassifierDBServiceWorker* aTarget, + nsIUrlClassifierCallback* aCB) + : mTarget(aTarget) + , mCB(aCB) + { } + + NS_DECL_NSIRUNNABLE + + private: + RefPtr<nsUrlClassifierDBServiceWorker> mTarget; + nsCOMPtr<nsIUrlClassifierCallback> mCB; + }; + + class BeginUpdateRunnable : public mozilla::Runnable + { + public: + BeginUpdateRunnable(nsUrlClassifierDBServiceWorker* aTarget, + nsIUrlClassifierUpdateObserver* aUpdater, + const nsACString& aTables) + : mTarget(aTarget) + , mUpdater(aUpdater) + , mTables(aTables) + { } + + NS_DECL_NSIRUNNABLE + + private: + RefPtr<nsUrlClassifierDBServiceWorker> mTarget; + nsCOMPtr<nsIUrlClassifierUpdateObserver> mUpdater; + nsCString mTables; + }; + + class BeginStreamRunnable : public mozilla::Runnable + { + public: + BeginStreamRunnable(nsUrlClassifierDBServiceWorker* aTarget, + const nsACString& aTable) + : mTarget(aTarget) + , mTable(aTable) + { } + + NS_DECL_NSIRUNNABLE + + private: + RefPtr<nsUrlClassifierDBServiceWorker> mTarget; + nsCString mTable; + }; + + class UpdateStreamRunnable : public mozilla::Runnable + { + public: + UpdateStreamRunnable(nsUrlClassifierDBServiceWorker* aTarget, + const nsACString& aUpdateChunk) + : mTarget(aTarget) + , mUpdateChunk(aUpdateChunk) + { } + + NS_DECL_NSIRUNNABLE + + private: + RefPtr<nsUrlClassifierDBServiceWorker> mTarget; + nsCString mUpdateChunk; + }; + + class CacheCompletionsRunnable : public mozilla::Runnable + { + public: + CacheCompletionsRunnable(nsUrlClassifierDBServiceWorker* aTarget, + mozilla::safebrowsing::CacheResultArray *aEntries) + : mTarget(aTarget) + , mEntries(aEntries) + { } + + NS_DECL_NSIRUNNABLE + + private: + RefPtr<nsUrlClassifierDBServiceWorker> mTarget; + mozilla::safebrowsing::CacheResultArray *mEntries; + }; + + class CacheMissesRunnable : public mozilla::Runnable + { + public: + CacheMissesRunnable(nsUrlClassifierDBServiceWorker* aTarget, + mozilla::safebrowsing::PrefixArray *aEntries) + : mTarget(aTarget) + , mEntries(aEntries) + { } + + NS_DECL_NSIRUNNABLE + + private: + RefPtr<nsUrlClassifierDBServiceWorker> mTarget; + mozilla::safebrowsing::PrefixArray *mEntries; + }; + + class DoLocalLookupRunnable : public mozilla::Runnable + { + public: + DoLocalLookupRunnable(nsUrlClassifierDBServiceWorker* aTarget, + const nsACString& spec, + const nsACString& tables, + mozilla::safebrowsing::LookupResultArray* results) + : mTarget(aTarget) + , mSpec(spec) + , mTables(tables) + , mResults(results) + { } + + NS_DECL_NSIRUNNABLE + private: + RefPtr<nsUrlClassifierDBServiceWorker> mTarget; + + nsCString mSpec; + nsCString mTables; + mozilla::safebrowsing::LookupResultArray* mResults; + }; + + class SetLastUpdateTimeRunnable : public mozilla::Runnable + { + public: + SetLastUpdateTimeRunnable(nsUrlClassifierDBServiceWorker* aTarget, + const nsACString& table, + uint64_t updateTime) + : mTarget(aTarget), + mTable(table), + mUpdateTime(updateTime) + { } + + NS_DECL_NSIRUNNABLE + private: + RefPtr<nsUrlClassifierDBServiceWorker> mTarget; + nsCString mTable; + uint64_t mUpdateTime; + }; + + class ClearLastResultsRunnable : public mozilla::Runnable + { + public: + explicit ClearLastResultsRunnable(nsUrlClassifierDBServiceWorker* aTarget) + : mTarget(aTarget) + { } + + NS_DECL_NSIRUNNABLE + private: + RefPtr<nsUrlClassifierDBServiceWorker> mTarget; + }; + +public: + nsresult DoLocalLookup(const nsACString& spec, + const nsACString& tables, + mozilla::safebrowsing::LookupResultArray* results); + + nsresult OpenDb(); + nsresult CloseDb(); + + nsresult CacheCompletions(mozilla::safebrowsing::CacheResultArray * aEntries); + nsresult CacheMisses(mozilla::safebrowsing::PrefixArray * aEntries); + +private: + ~UrlClassifierDBServiceWorkerProxy() {} + + RefPtr<nsUrlClassifierDBServiceWorker> mTarget; +}; + +// The remaining classes here are all proxies to the main thread + +class UrlClassifierLookupCallbackProxy final : + public nsIUrlClassifierLookupCallback +{ +public: + explicit UrlClassifierLookupCallbackProxy(nsIUrlClassifierLookupCallback* aTarget) + : mTarget(new nsMainThreadPtrHolder<nsIUrlClassifierLookupCallback>(aTarget)) + { } + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIURLCLASSIFIERLOOKUPCALLBACK + + class LookupCompleteRunnable : public mozilla::Runnable + { + public: + LookupCompleteRunnable(const nsMainThreadPtrHandle<nsIUrlClassifierLookupCallback>& aTarget, + mozilla::safebrowsing::LookupResultArray *aResults) + : mTarget(aTarget) + , mResults(aResults) + { } + + NS_DECL_NSIRUNNABLE + + private: + nsMainThreadPtrHandle<nsIUrlClassifierLookupCallback> mTarget; + mozilla::safebrowsing::LookupResultArray * mResults; + }; + +private: + ~UrlClassifierLookupCallbackProxy() {} + + nsMainThreadPtrHandle<nsIUrlClassifierLookupCallback> mTarget; +}; + +class UrlClassifierCallbackProxy final : public nsIUrlClassifierCallback +{ +public: + explicit UrlClassifierCallbackProxy(nsIUrlClassifierCallback* aTarget) + : mTarget(new nsMainThreadPtrHolder<nsIUrlClassifierCallback>(aTarget)) + { } + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIURLCLASSIFIERCALLBACK + + class HandleEventRunnable : public mozilla::Runnable + { + public: + HandleEventRunnable(const nsMainThreadPtrHandle<nsIUrlClassifierCallback>& aTarget, + const nsACString& aValue) + : mTarget(aTarget) + , mValue(aValue) + { } + + NS_DECL_NSIRUNNABLE + + private: + nsMainThreadPtrHandle<nsIUrlClassifierCallback> mTarget; + nsCString mValue; + }; + +private: + ~UrlClassifierCallbackProxy() {} + + nsMainThreadPtrHandle<nsIUrlClassifierCallback> mTarget; +}; + +class UrlClassifierUpdateObserverProxy final : + public nsIUrlClassifierUpdateObserver +{ +public: + explicit UrlClassifierUpdateObserverProxy(nsIUrlClassifierUpdateObserver* aTarget) + : mTarget(new nsMainThreadPtrHolder<nsIUrlClassifierUpdateObserver>(aTarget)) + { } + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIURLCLASSIFIERUPDATEOBSERVER + + class UpdateUrlRequestedRunnable : public mozilla::Runnable + { + public: + UpdateUrlRequestedRunnable(const nsMainThreadPtrHandle<nsIUrlClassifierUpdateObserver>& aTarget, + const nsACString& aURL, + const nsACString& aTable) + : mTarget(aTarget) + , mURL(aURL) + , mTable(aTable) + { } + + NS_DECL_NSIRUNNABLE + + private: + nsMainThreadPtrHandle<nsIUrlClassifierUpdateObserver> mTarget; + nsCString mURL, mTable; + }; + + class StreamFinishedRunnable : public mozilla::Runnable + { + public: + StreamFinishedRunnable(const nsMainThreadPtrHandle<nsIUrlClassifierUpdateObserver>& aTarget, + nsresult aStatus, uint32_t aDelay) + : mTarget(aTarget) + , mStatus(aStatus) + , mDelay(aDelay) + { } + + NS_DECL_NSIRUNNABLE + + private: + nsMainThreadPtrHandle<nsIUrlClassifierUpdateObserver> mTarget; + nsresult mStatus; + uint32_t mDelay; + }; + + class UpdateErrorRunnable : public mozilla::Runnable + { + public: + UpdateErrorRunnable(const nsMainThreadPtrHandle<nsIUrlClassifierUpdateObserver>& aTarget, + nsresult aError) + : mTarget(aTarget) + , mError(aError) + { } + + NS_DECL_NSIRUNNABLE + + private: + nsMainThreadPtrHandle<nsIUrlClassifierUpdateObserver> mTarget; + nsresult mError; + }; + + class UpdateSuccessRunnable : public mozilla::Runnable + { + public: + UpdateSuccessRunnable(const nsMainThreadPtrHandle<nsIUrlClassifierUpdateObserver>& aTarget, + uint32_t aRequestedTimeout) + : mTarget(aTarget) + , mRequestedTimeout(aRequestedTimeout) + { } + + NS_DECL_NSIRUNNABLE + + private: + nsMainThreadPtrHandle<nsIUrlClassifierUpdateObserver> mTarget; + uint32_t mRequestedTimeout; + }; + +private: + ~UrlClassifierUpdateObserverProxy() {} + + nsMainThreadPtrHandle<nsIUrlClassifierUpdateObserver> mTarget; +}; + +#endif // nsUrlClassifierProxies_h diff --git a/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.cpp b/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.cpp new file mode 100644 index 000000000..554bff342 --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.cpp @@ -0,0 +1,812 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCRT.h" +#include "nsIHttpChannel.h" +#include "nsIObserverService.h" +#include "nsIStringStream.h" +#include "nsIUploadChannel.h" +#include "nsIURI.h" +#include "nsIUrlClassifierDBService.h" +#include "nsNetUtil.h" +#include "nsStreamUtils.h" +#include "nsStringStream.h" +#include "nsToolkitCompsCID.h" +#include "nsUrlClassifierStreamUpdater.h" +#include "mozilla/BasePrincipal.h" +#include "mozilla/ErrorNames.h" +#include "mozilla/Logging.h" +#include "nsIInterfaceRequestor.h" +#include "mozilla/LoadContext.h" +#include "mozilla/Telemetry.h" +#include "nsContentUtils.h" +#include "nsIURLFormatter.h" + +using mozilla::DocShellOriginAttributes; + +static const char* gQuitApplicationMessage = "quit-application"; + +// Limit the list file size to 32mb +const uint32_t MAX_FILE_SIZE = (32 * 1024 * 1024); + +#undef LOG + +// MOZ_LOG=UrlClassifierStreamUpdater:5 +static mozilla::LazyLogModule gUrlClassifierStreamUpdaterLog("UrlClassifierStreamUpdater"); +#define LOG(args) TrimAndLog args + +// Calls nsIURLFormatter::TrimSensitiveURLs to remove sensitive +// info from the logging message. +static void TrimAndLog(const char* aFmt, ...) +{ + nsString raw; + + va_list ap; + va_start(ap, aFmt); + raw.AppendPrintf(aFmt, ap); + va_end(ap); + + nsCOMPtr<nsIURLFormatter> urlFormatter = + do_GetService("@mozilla.org/toolkit/URLFormatterService;1"); + + nsString trimmed; + nsresult rv = urlFormatter->TrimSensitiveURLs(raw, trimmed); + if (NS_FAILED(rv)) { + trimmed = EmptyString(); + } + + MOZ_LOG(gUrlClassifierStreamUpdaterLog, + mozilla::LogLevel::Debug, + (NS_ConvertUTF16toUTF8(trimmed).get())); +} + +// This class does absolutely nothing, except pass requests onto the DBService. + +/////////////////////////////////////////////////////////////////////////////// +// nsIUrlClassiferStreamUpdater implementation +// Handles creating/running the stream listener + +nsUrlClassifierStreamUpdater::nsUrlClassifierStreamUpdater() + : mIsUpdating(false), mInitialized(false), mDownloadError(false), + mBeganStream(false), mChannel(nullptr) +{ + LOG(("nsUrlClassifierStreamUpdater init [this=%p]", this)); +} + +NS_IMPL_ISUPPORTS(nsUrlClassifierStreamUpdater, + nsIUrlClassifierStreamUpdater, + nsIUrlClassifierUpdateObserver, + nsIRequestObserver, + nsIStreamListener, + nsIObserver, + nsIInterfaceRequestor, + nsITimerCallback) + +/** + * Clear out the update. + */ +void +nsUrlClassifierStreamUpdater::DownloadDone() +{ + LOG(("nsUrlClassifierStreamUpdater::DownloadDone [this=%p]", this)); + mIsUpdating = false; + + mPendingUpdates.Clear(); + mDownloadError = false; + mSuccessCallback = nullptr; + mUpdateErrorCallback = nullptr; + mDownloadErrorCallback = nullptr; +} + +/////////////////////////////////////////////////////////////////////////////// +// nsIUrlClassifierStreamUpdater implementation + +nsresult +nsUrlClassifierStreamUpdater::FetchUpdate(nsIURI *aUpdateUrl, + const nsACString & aRequestPayload, + bool aIsPostRequest, + const nsACString & aStreamTable) +{ + +#ifdef DEBUG + LOG(("Fetching update %s from %s", + aRequestPayload.Data(), aUpdateUrl->GetSpecOrDefault().get())); +#endif + + nsresult rv; + uint32_t loadFlags = nsIChannel::INHIBIT_CACHING | + nsIChannel::LOAD_BYPASS_CACHE; + rv = NS_NewChannel(getter_AddRefs(mChannel), + aUpdateUrl, + nsContentUtils::GetSystemPrincipal(), + nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_DATA_IS_NULL, + nsIContentPolicy::TYPE_OTHER, + nullptr, // aLoadGroup + this, // aInterfaceRequestor + loadFlags); + + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsILoadInfo> loadInfo = mChannel->GetLoadInfo(); + loadInfo->SetOriginAttributes(mozilla::NeckoOriginAttributes(NECKO_SAFEBROWSING_APP_ID, false)); + + mBeganStream = false; + + if (!aIsPostRequest) { + // We use POST method to send our request in v2. In v4, the request + // needs to be embedded to the URL and use GET method to send. + // However, from the Chromium source code, a extended HTTP header has + // to be sent along with the request to make the request succeed. + // The following description is from Chromium source code: + // + // "The following header informs the envelope server (which sits in + // front of Google's stubby server) that the received GET request should be + // interpreted as a POST." + // + nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(mChannel, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = httpChannel->SetRequestHeader(NS_LITERAL_CSTRING("X-HTTP-Method-Override"), + NS_LITERAL_CSTRING("POST"), + false); + NS_ENSURE_SUCCESS(rv, rv); + } else if (!aRequestPayload.IsEmpty()) { + rv = AddRequestBody(aRequestPayload); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Set the appropriate content type for file/data URIs, for unit testing + // purposes. + // This is only used for testing and should be deleted. + bool match; + if ((NS_SUCCEEDED(aUpdateUrl->SchemeIs("file", &match)) && match) || + (NS_SUCCEEDED(aUpdateUrl->SchemeIs("data", &match)) && match)) { + mChannel->SetContentType(NS_LITERAL_CSTRING("application/vnd.google.safebrowsing-update")); + } else { + // We assume everything else is an HTTP request. + + // Disable keepalive. + nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(mChannel, &rv); + NS_ENSURE_SUCCESS(rv, rv); + rv = httpChannel->SetRequestHeader(NS_LITERAL_CSTRING("Connection"), NS_LITERAL_CSTRING("close"), false); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Create a custom LoadContext for SafeBrowsing, so we can use callbacks on + // the channel to query the appId which allows separation of safebrowsing + // cookies in a separate jar. + DocShellOriginAttributes attrs; + attrs.mAppId = NECKO_SAFEBROWSING_APP_ID; + nsCOMPtr<nsIInterfaceRequestor> sbContext = new mozilla::LoadContext(attrs); + rv = mChannel->SetNotificationCallbacks(sbContext); + NS_ENSURE_SUCCESS(rv, rv); + + // Make the request. + rv = mChannel->AsyncOpen2(this); + NS_ENSURE_SUCCESS(rv, rv); + + mStreamTable = aStreamTable; + + return NS_OK; +} + +nsresult +nsUrlClassifierStreamUpdater::FetchUpdate(const nsACString & aUpdateUrl, + const nsACString & aRequestPayload, + bool aIsPostRequest, + const nsACString & aStreamTable) +{ + LOG(("(pre) Fetching update from %s\n", PromiseFlatCString(aUpdateUrl).get())); + + nsCString updateUrl(aUpdateUrl); + if (!aIsPostRequest) { + updateUrl.AppendPrintf("&$req=%s", nsCString(aRequestPayload).get()); + } + + nsCOMPtr<nsIURI> uri; + nsresult rv = NS_NewURI(getter_AddRefs(uri), updateUrl); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString urlSpec; + uri->GetAsciiSpec(urlSpec); + + LOG(("(post) Fetching update from %s\n", urlSpec.get())); + + return FetchUpdate(uri, aRequestPayload, aIsPostRequest, aStreamTable); +} + +NS_IMETHODIMP +nsUrlClassifierStreamUpdater::DownloadUpdates( + const nsACString &aRequestTables, + const nsACString &aRequestPayload, + bool aIsPostRequest, + const nsACString &aUpdateUrl, + nsIUrlClassifierCallback *aSuccessCallback, + nsIUrlClassifierCallback *aUpdateErrorCallback, + nsIUrlClassifierCallback *aDownloadErrorCallback, + bool *_retval) +{ + NS_ENSURE_ARG(aSuccessCallback); + NS_ENSURE_ARG(aUpdateErrorCallback); + NS_ENSURE_ARG(aDownloadErrorCallback); + + if (mIsUpdating) { + LOG(("Already updating, queueing update %s from %s", aRequestPayload.Data(), + aUpdateUrl.Data())); + *_retval = false; + PendingRequest *request = mPendingRequests.AppendElement(); + request->mTables = aRequestTables; + request->mRequestPayload = aRequestPayload; + request->mIsPostRequest = aIsPostRequest; + request->mUrl = aUpdateUrl; + request->mSuccessCallback = aSuccessCallback; + request->mUpdateErrorCallback = aUpdateErrorCallback; + request->mDownloadErrorCallback = aDownloadErrorCallback; + return NS_OK; + } + + if (aUpdateUrl.IsEmpty()) { + NS_ERROR("updateUrl not set"); + return NS_ERROR_NOT_INITIALIZED; + } + + nsresult rv; + + if (!mInitialized) { + // Add an observer for shutdown so we can cancel any pending list + // downloads. quit-application is the same event that the download + // manager listens for and uses to cancel pending downloads. + nsCOMPtr<nsIObserverService> observerService = + mozilla::services::GetObserverService(); + if (!observerService) + return NS_ERROR_FAILURE; + + observerService->AddObserver(this, gQuitApplicationMessage, false); + + mDBService = do_GetService(NS_URLCLASSIFIERDBSERVICE_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + mInitialized = true; + } + + rv = mDBService->BeginUpdate(this, aRequestTables); + if (rv == NS_ERROR_NOT_AVAILABLE) { + LOG(("Service busy, already updating, queuing update %s from %s", + aRequestPayload.Data(), aUpdateUrl.Data())); + *_retval = false; + PendingRequest *request = mPendingRequests.AppendElement(); + request->mTables = aRequestTables; + request->mRequestPayload = aRequestPayload; + request->mIsPostRequest = aIsPostRequest; + request->mUrl = aUpdateUrl; + request->mSuccessCallback = aSuccessCallback; + request->mUpdateErrorCallback = aUpdateErrorCallback; + request->mDownloadErrorCallback = aDownloadErrorCallback; + return NS_OK; + } + + if (NS_FAILED(rv)) { + return rv; + } + + mSuccessCallback = aSuccessCallback; + mUpdateErrorCallback = aUpdateErrorCallback; + mDownloadErrorCallback = aDownloadErrorCallback; + + mIsUpdating = true; + *_retval = true; + + LOG(("FetchUpdate: %s", aUpdateUrl.Data())); + + return FetchUpdate(aUpdateUrl, aRequestPayload, aIsPostRequest, EmptyCString()); +} + +/////////////////////////////////////////////////////////////////////////////// +// nsIUrlClassifierUpdateObserver implementation + +NS_IMETHODIMP +nsUrlClassifierStreamUpdater::UpdateUrlRequested(const nsACString &aUrl, + const nsACString &aTable) +{ + LOG(("Queuing requested update from %s\n", PromiseFlatCString(aUrl).get())); + + PendingUpdate *update = mPendingUpdates.AppendElement(); + if (!update) + return NS_ERROR_OUT_OF_MEMORY; + + // Allow data: and file: urls for unit testing purposes, otherwise assume http + if (StringBeginsWith(aUrl, NS_LITERAL_CSTRING("data:")) || + StringBeginsWith(aUrl, NS_LITERAL_CSTRING("file:"))) { + update->mUrl = aUrl; + } else { + // For unittesting update urls to localhost should use http, not https + // (otherwise the connection will fail silently, since there will be no + // cert available). + if (!StringBeginsWith(aUrl, NS_LITERAL_CSTRING("localhost"))) { + update->mUrl = NS_LITERAL_CSTRING("https://") + aUrl; + } else { + update->mUrl = NS_LITERAL_CSTRING("http://") + aUrl; + } + } + update->mTable = aTable; + + return NS_OK; +} + +nsresult +nsUrlClassifierStreamUpdater::FetchNext() +{ + if (mPendingUpdates.Length() == 0) { + return NS_OK; + } + + PendingUpdate &update = mPendingUpdates[0]; + LOG(("Fetching update url: %s\n", update.mUrl.get())); + nsresult rv = FetchUpdate(update.mUrl, + EmptyCString(), + true, // This method is for v2 and v2 is always a POST. + update.mTable); + if (NS_FAILED(rv)) { + LOG(("Error fetching update url: %s\n", update.mUrl.get())); + // We can commit the urls that we've applied so far. This is + // probably a transient server problem, so trigger backoff. + mDownloadErrorCallback->HandleEvent(EmptyCString()); + mDownloadError = true; + mDBService->FinishUpdate(); + return rv; + } + + mPendingUpdates.RemoveElementAt(0); + + return NS_OK; +} + +nsresult +nsUrlClassifierStreamUpdater::FetchNextRequest() +{ + if (mPendingRequests.Length() == 0) { + LOG(("No more requests, returning")); + return NS_OK; + } + + PendingRequest &request = mPendingRequests[0]; + LOG(("Stream updater: fetching next request: %s, %s", + request.mTables.get(), request.mUrl.get())); + bool dummy; + DownloadUpdates( + request.mTables, + request.mRequestPayload, + request.mIsPostRequest, + request.mUrl, + request.mSuccessCallback, + request.mUpdateErrorCallback, + request.mDownloadErrorCallback, + &dummy); + request.mSuccessCallback = nullptr; + request.mUpdateErrorCallback = nullptr; + request.mDownloadErrorCallback = nullptr; + mPendingRequests.RemoveElementAt(0); + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierStreamUpdater::StreamFinished(nsresult status, + uint32_t requestedDelay) +{ + // We are a service and may not be reset with Init between calls, so reset + // mBeganStream manually. + mBeganStream = false; + LOG(("nsUrlClassifierStreamUpdater::StreamFinished [%x, %d]", status, requestedDelay)); + if (NS_FAILED(status) || mPendingUpdates.Length() == 0) { + // We're done. + LOG(("nsUrlClassifierStreamUpdater::Done [this=%p]", this)); + mDBService->FinishUpdate(); + return NS_OK; + } + + // This timer is for fetching indirect updates ("forwards") from any "u:" lines + // that we encountered while processing the server response. It is NOT for + // scheduling the next time we pull the list from the server. That's a different + // timer in listmanager.js (see bug 1110891). + nsresult rv; + mTimer = do_CreateInstance("@mozilla.org/timer;1", &rv); + if (NS_SUCCEEDED(rv)) { + rv = mTimer->InitWithCallback(this, requestedDelay, + nsITimer::TYPE_ONE_SHOT); + } + + if (NS_FAILED(rv)) { + NS_WARNING("Unable to initialize timer, fetching next safebrowsing item immediately"); + return FetchNext(); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierStreamUpdater::UpdateSuccess(uint32_t requestedTimeout) +{ + LOG(("nsUrlClassifierStreamUpdater::UpdateSuccess [this=%p]", this)); + if (mPendingUpdates.Length() != 0) { + NS_WARNING("Didn't fetch all safebrowsing update redirects"); + } + + // DownloadDone() clears mSuccessCallback, so we save it off here. + nsCOMPtr<nsIUrlClassifierCallback> successCallback = mDownloadError ? nullptr : mSuccessCallback.get(); + DownloadDone(); + + nsAutoCString strTimeout; + strTimeout.AppendInt(requestedTimeout); + if (successCallback) { + LOG(("nsUrlClassifierStreamUpdater::UpdateSuccess callback [this=%p]", + this)); + successCallback->HandleEvent(strTimeout); + } else { + LOG(("nsUrlClassifierStreamUpdater::UpdateSuccess skipping callback [this=%p]", + this)); + } + // Now fetch the next request + LOG(("stream updater: calling into fetch next request")); + FetchNextRequest(); + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierStreamUpdater::UpdateError(nsresult result) +{ + LOG(("nsUrlClassifierStreamUpdater::UpdateError [this=%p]", this)); + + // DownloadDone() clears mUpdateErrorCallback, so we save it off here. + nsCOMPtr<nsIUrlClassifierCallback> errorCallback = mDownloadError ? nullptr : mUpdateErrorCallback.get(); + + DownloadDone(); + + nsAutoCString strResult; + strResult.AppendInt(static_cast<uint32_t>(result)); + if (errorCallback) { + errorCallback->HandleEvent(strResult); + } + + return NS_OK; +} + +nsresult +nsUrlClassifierStreamUpdater::AddRequestBody(const nsACString &aRequestBody) +{ + nsresult rv; + nsCOMPtr<nsIStringInputStream> strStream = + do_CreateInstance(NS_STRINGINPUTSTREAM_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = strStream->SetData(aRequestBody.BeginReading(), + aRequestBody.Length()); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIUploadChannel> uploadChannel = do_QueryInterface(mChannel, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = uploadChannel->SetUploadStream(strStream, + NS_LITERAL_CSTRING("text/plain"), + -1); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(mChannel, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = httpChannel->SetRequestMethod(NS_LITERAL_CSTRING("POST")); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +// Map the HTTP response code to a Telemetry bucket +static uint32_t HTTPStatusToBucket(uint32_t status) +{ + uint32_t statusBucket; + switch (status) { + case 100: + case 101: + // Unexpected 1xx return code + statusBucket = 0; + break; + case 200: + // OK - Data is available in the HTTP response body. + statusBucket = 1; + break; + case 201: + case 202: + case 203: + case 205: + case 206: + // Unexpected 2xx return code + statusBucket = 2; + break; + case 204: + // No Content + statusBucket = 3; + break; + case 300: + case 301: + case 302: + case 303: + case 304: + case 305: + case 307: + case 308: + // Unexpected 3xx return code + statusBucket = 4; + break; + case 400: + // Bad Request - The HTTP request was not correctly formed. + // The client did not provide all required CGI parameters. + statusBucket = 5; + break; + case 401: + case 402: + case 405: + case 406: + case 407: + case 409: + case 410: + case 411: + case 412: + case 414: + case 415: + case 416: + case 417: + case 421: + case 426: + case 428: + case 429: + case 431: + case 451: + // Unexpected 4xx return code + statusBucket = 6; + break; + case 403: + // Forbidden - The client id is invalid. + statusBucket = 7; + break; + case 404: + // Not Found + statusBucket = 8; + break; + case 408: + // Request Timeout + statusBucket = 9; + break; + case 413: + // Request Entity Too Large - Bug 1150334 + statusBucket = 10; + break; + case 500: + case 501: + case 510: + // Unexpected 5xx return code + statusBucket = 11; + break; + case 502: + case 504: + case 511: + // Local network errors, we'll ignore these. + statusBucket = 12; + break; + case 503: + // Service Unavailable - The server cannot handle the request. + // Clients MUST follow the backoff behavior specified in the + // Request Frequency section. + statusBucket = 13; + break; + case 505: + // HTTP Version Not Supported - The server CANNOT handle the requested + // protocol major version. + statusBucket = 14; + break; + default: + statusBucket = 15; + }; + return statusBucket; +} + +/////////////////////////////////////////////////////////////////////////////// +// nsIStreamListenerObserver implementation + +NS_IMETHODIMP +nsUrlClassifierStreamUpdater::OnStartRequest(nsIRequest *request, + nsISupports* context) +{ + nsresult rv; + bool downloadError = false; + nsAutoCString strStatus; + nsresult status = NS_OK; + + // Only update if we got http success header + nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(request); + if (httpChannel) { + rv = httpChannel->GetStatus(&status); + NS_ENSURE_SUCCESS(rv, rv); + + if (MOZ_LOG_TEST(gUrlClassifierStreamUpdaterLog, mozilla::LogLevel::Debug)) { + nsAutoCString errorName, spec; + mozilla::GetErrorName(status, errorName); + nsCOMPtr<nsIURI> uri; + rv = httpChannel->GetURI(getter_AddRefs(uri)); + if (NS_SUCCEEDED(rv) && uri) { + uri->GetAsciiSpec(spec); + } + LOG(("nsUrlClassifierStreamUpdater::OnStartRequest " + "(status=%s, uri=%s, this=%p)", errorName.get(), + spec.get(), this)); + } + + if (NS_FAILED(status)) { + // Assume we're overloading the server and trigger backoff. + downloadError = true; + mozilla::Telemetry::Accumulate(mozilla::Telemetry::URLCLASSIFIER_UPDATE_REMOTE_STATUS, + 15 /* unknown response code */); + + } else { + bool succeeded = false; + rv = httpChannel->GetRequestSucceeded(&succeeded); + NS_ENSURE_SUCCESS(rv, rv); + + uint32_t requestStatus; + rv = httpChannel->GetResponseStatus(&requestStatus); + NS_ENSURE_SUCCESS(rv, rv); + mozilla::Telemetry::Accumulate(mozilla::Telemetry::URLCLASSIFIER_UPDATE_REMOTE_STATUS, + HTTPStatusToBucket(requestStatus)); + LOG(("nsUrlClassifierStreamUpdater::OnStartRequest %s (%d)", succeeded ? + "succeeded" : "failed", requestStatus)); + if (!succeeded) { + // 404 or other error, pass error status back + strStatus.AppendInt(requestStatus); + downloadError = true; + } + } + } + + if (downloadError) { + LOG(("nsUrlClassifierStreamUpdater::Download error [this=%p]", this)); + + // It's possible for mDownloadErrorCallback to be null on shutdown. + if (mDownloadErrorCallback) { + mDownloadErrorCallback->HandleEvent(strStatus); + } + + mDownloadError = true; + status = NS_ERROR_ABORT; + } else if (NS_SUCCEEDED(status)) { + MOZ_ASSERT(mDownloadErrorCallback); + mBeganStream = true; + LOG(("nsUrlClassifierStreamUpdater::Beginning stream [this=%p]", this)); + rv = mDBService->BeginStream(mStreamTable); + NS_ENSURE_SUCCESS(rv, rv); + } + + mStreamTable.Truncate(); + + return status; +} + +NS_IMETHODIMP +nsUrlClassifierStreamUpdater::OnDataAvailable(nsIRequest *request, + nsISupports* context, + nsIInputStream *aIStream, + uint64_t aSourceOffset, + uint32_t aLength) +{ + if (!mDBService) + return NS_ERROR_NOT_INITIALIZED; + + LOG(("OnDataAvailable (%d bytes)", aLength)); + + if (aSourceOffset > MAX_FILE_SIZE) { + LOG(("OnDataAvailable::Abort because exceeded the maximum file size(%lld)", aSourceOffset)); + return NS_ERROR_FILE_TOO_BIG; + } + + nsresult rv; + + // Copy the data into a nsCString + nsCString chunk; + rv = NS_ConsumeStream(aIStream, aLength, chunk); + NS_ENSURE_SUCCESS(rv, rv); + + //LOG(("Chunk (%d): %s\n\n", chunk.Length(), chunk.get())); + rv = mDBService->UpdateStream(chunk); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierStreamUpdater::OnStopRequest(nsIRequest *request, nsISupports* context, + nsresult aStatus) +{ + if (!mDBService) + return NS_ERROR_NOT_INITIALIZED; + + LOG(("OnStopRequest (status %x, beganStream %s, this=%p)", aStatus, + mBeganStream ? "true" : "false", this)); + + nsresult rv; + + if (NS_SUCCEEDED(aStatus)) { + // Success, finish this stream and move on to the next. + rv = mDBService->FinishStream(); + } else if (mBeganStream) { + LOG(("OnStopRequest::Canceling update [this=%p]", this)); + // We began this stream and couldn't finish it. We have to cancel the + // update, it's not in a consistent state. + rv = mDBService->CancelUpdate(); + } else { + LOG(("OnStopRequest::Finishing update [this=%p]", this)); + // The fetch failed, but we didn't start the stream (probably a + // server or connection error). We can commit what we've applied + // so far, and request again later. + rv = mDBService->FinishUpdate(); + } + + mChannel = nullptr; + + // If the fetch failed, return the network status rather than NS_OK, the + // result of finishing a possibly-empty update + if (NS_SUCCEEDED(aStatus)) { + return rv; + } + return aStatus; +} + +/////////////////////////////////////////////////////////////////////////////// +// nsIObserver implementation + +NS_IMETHODIMP +nsUrlClassifierStreamUpdater::Observe(nsISupports *aSubject, const char *aTopic, + const char16_t *aData) +{ + if (nsCRT::strcmp(aTopic, gQuitApplicationMessage) == 0) { + if (mIsUpdating && mChannel) { + LOG(("Cancel download")); + nsresult rv; + rv = mChannel->Cancel(NS_ERROR_ABORT); + NS_ENSURE_SUCCESS(rv, rv); + mIsUpdating = false; + mChannel = nullptr; + } + if (mTimer) { + mTimer->Cancel(); + mTimer = nullptr; + } + } + return NS_OK; +} + +/////////////////////////////////////////////////////////////////////////////// +// nsIInterfaceRequestor implementation + +NS_IMETHODIMP +nsUrlClassifierStreamUpdater::GetInterface(const nsIID & eventSinkIID, void* *_retval) +{ + return QueryInterface(eventSinkIID, _retval); +} + + +/////////////////////////////////////////////////////////////////////////////// +// nsITimerCallback implementation +NS_IMETHODIMP +nsUrlClassifierStreamUpdater::Notify(nsITimer *timer) +{ + LOG(("nsUrlClassifierStreamUpdater::Notify [%p]", this)); + + mTimer = nullptr; + + // Start the update process up again. + FetchNext(); + + return NS_OK; +} + diff --git a/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.h b/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.h new file mode 100644 index 000000000..b24df61d2 --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierStreamUpdater.h @@ -0,0 +1,103 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsUrlClassifierStreamUpdater_h_ +#define nsUrlClassifierStreamUpdater_h_ + +#include <nsISupportsUtils.h> + +#include "nsCOMPtr.h" +#include "nsIObserver.h" +#include "nsIUrlClassifierStreamUpdater.h" +#include "nsIStreamListener.h" +#include "nsIChannel.h" +#include "nsTArray.h" +#include "nsITimer.h" +#include "mozilla/Attributes.h" + +// Forward declare pointers +class nsIURI; + +class nsUrlClassifierStreamUpdater final : public nsIUrlClassifierStreamUpdater, + public nsIUrlClassifierUpdateObserver, + public nsIStreamListener, + public nsIObserver, + public nsIInterfaceRequestor, + public nsITimerCallback +{ +public: + nsUrlClassifierStreamUpdater(); + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIURLCLASSIFIERSTREAMUPDATER + NS_DECL_NSIURLCLASSIFIERUPDATEOBSERVER + NS_DECL_NSIINTERFACEREQUESTOR + NS_DECL_NSIREQUESTOBSERVER + NS_DECL_NSISTREAMLISTENER + NS_DECL_NSIOBSERVER + NS_DECL_NSITIMERCALLBACK + +private: + // No subclassing + ~nsUrlClassifierStreamUpdater() {} + + // When the dbservice sends an UpdateComplete or UpdateFailure, we call this + // to reset the stream updater. + void DownloadDone(); + + // Disallow copy constructor + nsUrlClassifierStreamUpdater(nsUrlClassifierStreamUpdater&); + + nsresult AddRequestBody(const nsACString &aRequestBody); + + // Fetches an update for a single table. + nsresult FetchUpdate(nsIURI *aURI, + const nsACString &aRequest, + bool aIsPostRequest, + const nsACString &aTable); + // Dumb wrapper so we don't have to create URIs. + nsresult FetchUpdate(const nsACString &aURI, + const nsACString &aRequest, + bool aIsPostRequest, + const nsACString &aTable); + + // Fetches the next table, from mPendingUpdates. + nsresult FetchNext(); + // Fetches the next request, from mPendingRequests + nsresult FetchNextRequest(); + + + bool mIsUpdating; + bool mInitialized; + bool mDownloadError; + bool mBeganStream; + nsCString mStreamTable; + nsCOMPtr<nsIChannel> mChannel; + nsCOMPtr<nsIUrlClassifierDBService> mDBService; + nsCOMPtr<nsITimer> mTimer; + + struct PendingRequest { + nsCString mTables; + nsCString mRequestPayload; + bool mIsPostRequest; + nsCString mUrl; + nsCOMPtr<nsIUrlClassifierCallback> mSuccessCallback; + nsCOMPtr<nsIUrlClassifierCallback> mUpdateErrorCallback; + nsCOMPtr<nsIUrlClassifierCallback> mDownloadErrorCallback; + }; + nsTArray<PendingRequest> mPendingRequests; + + struct PendingUpdate { + nsCString mUrl; + nsCString mTable; + }; + nsTArray<PendingUpdate> mPendingUpdates; + + nsCOMPtr<nsIUrlClassifierCallback> mSuccessCallback; + nsCOMPtr<nsIUrlClassifierCallback> mUpdateErrorCallback; + nsCOMPtr<nsIUrlClassifierCallback> mDownloadErrorCallback; +}; + +#endif // nsUrlClassifierStreamUpdater_h_ diff --git a/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp b/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp new file mode 100644 index 000000000..e4cf68c98 --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp @@ -0,0 +1,665 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsEscape.h" +#include "nsString.h" +#include "nsIURI.h" +#include "nsUrlClassifierUtils.h" +#include "nsTArray.h" +#include "nsReadableUtils.h" +#include "plbase64.h" +#include "nsPrintfCString.h" +#include "safebrowsing.pb.h" +#include "mozilla/Sprintf.h" +#include "mozilla/Mutex.h" + +#define DEFAULT_PROTOCOL_VERSION "2.2" + +static char int_to_hex_digit(int32_t i) +{ + NS_ASSERTION((i >= 0) && (i <= 15), "int too big in int_to_hex_digit"); + return static_cast<char>(((i < 10) ? (i + '0') : ((i - 10) + 'A'))); +} + +static bool +IsDecimal(const nsACString & num) +{ + for (uint32_t i = 0; i < num.Length(); i++) { + if (!isdigit(num[i])) { + return false; + } + } + + return true; +} + +static bool +IsHex(const nsACString & num) +{ + if (num.Length() < 3) { + return false; + } + + if (num[0] != '0' || !(num[1] == 'x' || num[1] == 'X')) { + return false; + } + + for (uint32_t i = 2; i < num.Length(); i++) { + if (!isxdigit(num[i])) { + return false; + } + } + + return true; +} + +static bool +IsOctal(const nsACString & num) +{ + if (num.Length() < 2) { + return false; + } + + if (num[0] != '0') { + return false; + } + + for (uint32_t i = 1; i < num.Length(); i++) { + if (!isdigit(num[i]) || num[i] == '8' || num[i] == '9') { + return false; + } + } + + return true; +} + +///////////////////////////////////////////////////////////////// +// SafeBrowsing V4 related utits. + +namespace mozilla { +namespace safebrowsing { + +static PlatformType +GetPlatformType() +{ +#if defined(ANDROID) + return ANDROID_PLATFORM; +#elif defined(XP_MACOSX) + return OSX_PLATFORM; +#elif defined(XP_LINUX) + return LINUX_PLATFORM; +#elif defined(XP_WIN) + return WINDOWS_PLATFORM; +#else + return PLATFORM_TYPE_UNSPECIFIED; +#endif +} + +typedef FetchThreatListUpdatesRequest_ListUpdateRequest ListUpdateRequest; +typedef FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints Constraints; + +static void +InitListUpdateRequest(ThreatType aThreatType, + const char* aStateBase64, + ListUpdateRequest* aListUpdateRequest) +{ + aListUpdateRequest->set_threat_type(aThreatType); + aListUpdateRequest->set_platform_type(GetPlatformType()); + aListUpdateRequest->set_threat_entry_type(URL); + + Constraints* contraints = new Constraints(); + contraints->add_supported_compressions(RICE); + aListUpdateRequest->set_allocated_constraints(contraints); + + // Only set non-empty state. + if (aStateBase64[0] != '\0') { + nsCString stateBinary; + nsresult rv = Base64Decode(nsCString(aStateBase64), stateBinary); + if (NS_SUCCEEDED(rv)) { + aListUpdateRequest->set_state(stateBinary.get(), stateBinary.Length()); + } + } +} + +static ClientInfo* +CreateClientInfo() +{ + ClientInfo* c = new ClientInfo(); + + nsCOMPtr<nsIPrefBranch> prefBranch = + do_GetService(NS_PREFSERVICE_CONTRACTID); + + nsXPIDLCString clientId; + nsresult rv = prefBranch->GetCharPref("browser.safebrowsing.id", + getter_Copies(clientId)); + + if (NS_FAILED(rv)) { + clientId = "Firefox"; // Use "Firefox" as fallback. + } + + c->set_client_id(clientId.get()); + + return c; +} + +} // end of namespace safebrowsing. +} // end of namespace mozilla. + +nsUrlClassifierUtils::nsUrlClassifierUtils() + : mEscapeCharmap(nullptr) + , mProviderDictLock("nsUrlClassifierUtils.mProviderDictLock") +{ +} + +nsresult +nsUrlClassifierUtils::Init() +{ + // Everything but alpha numerics, - and . + mEscapeCharmap = new Charmap(0xffffffff, 0xfc009fff, 0xf8000001, 0xf8000001, + 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff); + if (!mEscapeCharmap) + return NS_ERROR_OUT_OF_MEMORY; + + // nsIUrlClassifierUtils is a thread-safe service so it's + // allowed to use on non-main threads. However, building + // the provider dictionary must be on the main thread. + // We forcefully load nsUrlClassifierUtils in + // nsUrlClassifierDBService::Init() to ensure we must + // now be on the main thread. + nsresult rv = ReadProvidersFromPrefs(mProviderDict); + NS_ENSURE_SUCCESS(rv, rv); + + // Add an observer for shutdown + nsCOMPtr<nsIObserverService> observerService = + mozilla::services::GetObserverService(); + if (!observerService) + return NS_ERROR_FAILURE; + + observerService->AddObserver(this, "xpcom-shutdown-threads", false); + Preferences::AddStrongObserver(this, "browser.safebrowsing"); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(nsUrlClassifierUtils, + nsIUrlClassifierUtils, + nsIObserver) + +///////////////////////////////////////////////////////////////////////////// +// nsIUrlClassifierUtils + +NS_IMETHODIMP +nsUrlClassifierUtils::GetKeyForURI(nsIURI * uri, nsACString & _retval) +{ + nsCOMPtr<nsIURI> innerURI = NS_GetInnermostURI(uri); + if (!innerURI) + innerURI = uri; + + nsAutoCString host; + innerURI->GetAsciiHost(host); + + if (host.IsEmpty()) { + return NS_ERROR_MALFORMED_URI; + } + + nsresult rv = CanonicalizeHostname(host, _retval); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString path; + rv = innerURI->GetPath(path); + NS_ENSURE_SUCCESS(rv, rv); + + // strip out anchors + int32_t ref = path.FindChar('#'); + if (ref != kNotFound) + path.SetLength(ref); + + nsAutoCString temp; + rv = CanonicalizePath(path, temp); + NS_ENSURE_SUCCESS(rv, rv); + + _retval.Append(temp); + + return NS_OK; +} + +// We use "goog-*-proto" as the list name for v4, where "proto" indicates +// it's updated (as well as hash completion) via protobuf. +// +// In the mozilla official build, we are allowed to use the +// private phishing list (goog-phish-proto). See Bug 1288840. +static const struct { + const char* mListName; + uint32_t mThreatType; +} THREAT_TYPE_CONV_TABLE[] = { + { "goog-malware-proto", MALWARE_THREAT}, // 1 + { "googpub-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2 + { "goog-unwanted-proto", UNWANTED_SOFTWARE}, // 3 + { "goog-phish-proto", SOCIAL_ENGINEERING}, // 5 + + // For testing purpose. + { "test-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2 + { "test-unwanted-proto", UNWANTED_SOFTWARE}, // 3 +}; + +NS_IMETHODIMP +nsUrlClassifierUtils::ConvertThreatTypeToListNames(uint32_t aThreatType, + nsACString& aListNames) +{ + for (uint32_t i = 0; i < ArrayLength(THREAT_TYPE_CONV_TABLE); i++) { + if (aThreatType == THREAT_TYPE_CONV_TABLE[i].mThreatType) { + if (!aListNames.IsEmpty()) { + aListNames.AppendLiteral(","); + } + aListNames += THREAT_TYPE_CONV_TABLE[i].mListName; + } + } + + return aListNames.IsEmpty() ? NS_ERROR_FAILURE : NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierUtils::ConvertListNameToThreatType(const nsACString& aListName, + uint32_t* aThreatType) +{ + for (uint32_t i = 0; i < ArrayLength(THREAT_TYPE_CONV_TABLE); i++) { + if (aListName.EqualsASCII(THREAT_TYPE_CONV_TABLE[i].mListName)) { + *aThreatType = THREAT_TYPE_CONV_TABLE[i].mThreatType; + return NS_OK; + } + } + + return NS_ERROR_FAILURE; +} + +NS_IMETHODIMP +nsUrlClassifierUtils::GetProvider(const nsACString& aTableName, + nsACString& aProvider) +{ + MutexAutoLock lock(mProviderDictLock); + nsCString* provider = nullptr; + if (mProviderDict.Get(aTableName, &provider)) { + aProvider = provider ? *provider : EmptyCString(); + } else { + aProvider = EmptyCString(); + } + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierUtils::GetProtocolVersion(const nsACString& aProvider, + nsACString& aVersion) +{ + nsCOMPtr<nsIPrefBranch> prefBranch = do_GetService(NS_PREFSERVICE_CONTRACTID); + if (prefBranch) { + nsPrintfCString prefName("browser.safebrowsing.provider.%s.pver", + nsCString(aProvider).get()); + nsXPIDLCString version; + nsresult rv = prefBranch->GetCharPref(prefName.get(), getter_Copies(version)); + + aVersion = NS_SUCCEEDED(rv) ? version : DEFAULT_PROTOCOL_VERSION; + } else { + aVersion = DEFAULT_PROTOCOL_VERSION; + } + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierUtils::MakeUpdateRequestV4(const char** aListNames, + const char** aStatesBase64, + uint32_t aCount, + nsACString &aRequest) +{ + using namespace mozilla::safebrowsing; + + FetchThreatListUpdatesRequest r; + r.set_allocated_client(CreateClientInfo()); + + for (uint32_t i = 0; i < aCount; i++) { + nsCString listName(aListNames[i]); + uint32_t threatType; + nsresult rv = ConvertListNameToThreatType(listName, &threatType); + if (NS_FAILED(rv)) { + continue; // Unknown list name. + } + auto lur = r.mutable_list_update_requests()->Add(); + InitListUpdateRequest(static_cast<ThreatType>(threatType), aStatesBase64[i], lur); + } + + // Then serialize. + std::string s; + r.SerializeToString(&s); + + nsCString out; + nsresult rv = Base64URLEncode(s.size(), + (const uint8_t*)s.c_str(), + Base64URLEncodePaddingPolicy::Include, + out); + NS_ENSURE_SUCCESS(rv, rv); + + aRequest = out; + + return NS_OK; +} + +////////////////////////////////////////////////////////// +// nsIObserver + +NS_IMETHODIMP +nsUrlClassifierUtils::Observe(nsISupports *aSubject, const char *aTopic, + const char16_t *aData) +{ + if (0 == strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) { + MutexAutoLock lock(mProviderDictLock); + return ReadProvidersFromPrefs(mProviderDict); + } + + if (0 == strcmp(aTopic, "xpcom-shutdown-threads")) { + nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID); + NS_ENSURE_TRUE(prefs, NS_ERROR_FAILURE); + return prefs->RemoveObserver("browser.safebrowsing", this); + } + + return NS_ERROR_UNEXPECTED; +} + +///////////////////////////////////////////////////////////////////////////// +// non-interface methods + +nsresult +nsUrlClassifierUtils::ReadProvidersFromPrefs(ProviderDictType& aDict) +{ + MOZ_ASSERT(NS_IsMainThread(), "ReadProvidersFromPrefs must be on main thread"); + + nsCOMPtr<nsIPrefService> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID); + NS_ENSURE_TRUE(prefs, NS_ERROR_FAILURE); + nsCOMPtr<nsIPrefBranch> prefBranch; + nsresult rv = prefs->GetBranch("browser.safebrowsing.provider.", + getter_AddRefs(prefBranch)); + NS_ENSURE_SUCCESS(rv, rv); + + // We've got a pref branch for "browser.safebrowsing.provider.". + // Enumerate all children prefs and parse providers. + uint32_t childCount; + char** childArray; + rv = prefBranch->GetChildList("", &childCount, &childArray); + NS_ENSURE_SUCCESS(rv, rv); + + // Collect providers from childArray. + nsTHashtable<nsCStringHashKey> providers; + for (uint32_t i = 0; i < childCount; i++) { + nsCString child(childArray[i]); + auto dotPos = child.FindChar('.'); + if (dotPos < 0) { + continue; + } + + nsDependentCSubstring provider = Substring(child, 0, dotPos); + + providers.PutEntry(provider); + } + NS_FREE_XPCOM_ALLOCATED_POINTER_ARRAY(childCount, childArray); + + // Now we have all providers. Check which one owns |aTableName|. + // e.g. The owning lists of provider "google" is defined in + // "browser.safebrowsing.provider.google.lists". + for (auto itr = providers.Iter(); !itr.Done(); itr.Next()) { + auto entry = itr.Get(); + nsCString provider(entry->GetKey()); + nsPrintfCString owninListsPref("%s.lists", provider.get()); + + nsXPIDLCString owningLists; + nsresult rv = prefBranch->GetCharPref(owninListsPref.get(), + getter_Copies(owningLists)); + if (NS_FAILED(rv)) { + continue; + } + + // We've got the owning lists (represented as string) of |provider|. + // Build the dictionary for the owning list and the current provider. + nsTArray<nsCString> tables; + Classifier::SplitTables(owningLists, tables); + for (auto tableName : tables) { + aDict.Put(tableName, new nsCString(provider)); + } + } + + return NS_OK; +} + +nsresult +nsUrlClassifierUtils::CanonicalizeHostname(const nsACString & hostname, + nsACString & _retval) +{ + nsAutoCString unescaped; + if (!NS_UnescapeURL(PromiseFlatCString(hostname).get(), + PromiseFlatCString(hostname).Length(), + 0, unescaped)) { + unescaped.Assign(hostname); + } + + nsAutoCString cleaned; + CleanupHostname(unescaped, cleaned); + + nsAutoCString temp; + ParseIPAddress(cleaned, temp); + if (!temp.IsEmpty()) { + cleaned.Assign(temp); + } + + ToLowerCase(cleaned); + SpecialEncode(cleaned, false, _retval); + + return NS_OK; +} + + +nsresult +nsUrlClassifierUtils::CanonicalizePath(const nsACString & path, + nsACString & _retval) +{ + _retval.Truncate(); + + nsAutoCString decodedPath(path); + nsAutoCString temp; + while (NS_UnescapeURL(decodedPath.get(), decodedPath.Length(), 0, temp)) { + decodedPath.Assign(temp); + temp.Truncate(); + } + + SpecialEncode(decodedPath, true, _retval); + // XXX: lowercase the path? + + return NS_OK; +} + +void +nsUrlClassifierUtils::CleanupHostname(const nsACString & hostname, + nsACString & _retval) +{ + _retval.Truncate(); + + const char* curChar = hostname.BeginReading(); + const char* end = hostname.EndReading(); + char lastChar = '\0'; + while (curChar != end) { + unsigned char c = static_cast<unsigned char>(*curChar); + if (c == '.' && (lastChar == '\0' || lastChar == '.')) { + // skip + } else { + _retval.Append(*curChar); + } + lastChar = c; + ++curChar; + } + + // cut off trailing dots + while (_retval.Length() > 0 && _retval[_retval.Length() - 1] == '.') { + _retval.SetLength(_retval.Length() - 1); + } +} + +void +nsUrlClassifierUtils::ParseIPAddress(const nsACString & host, + nsACString & _retval) +{ + _retval.Truncate(); + nsACString::const_iterator iter, end; + host.BeginReading(iter); + host.EndReading(end); + + if (host.Length() <= 15) { + // The Windows resolver allows a 4-part dotted decimal IP address to + // have a space followed by any old rubbish, so long as the total length + // of the string doesn't get above 15 characters. So, "10.192.95.89 xy" + // is resolved to 10.192.95.89. + // If the string length is greater than 15 characters, e.g. + // "10.192.95.89 xy.wildcard.example.com", it will be resolved through + // DNS. + + if (FindCharInReadable(' ', iter, end)) { + end = iter; + } + } + + for (host.BeginReading(iter); iter != end; iter++) { + if (!(isxdigit(*iter) || *iter == 'x' || *iter == 'X' || *iter == '.')) { + // not an IP + return; + } + } + + host.BeginReading(iter); + nsTArray<nsCString> parts; + ParseString(PromiseFlatCString(Substring(iter, end)), '.', parts); + if (parts.Length() > 4) { + return; + } + + // If any potentially-octal numbers (start with 0 but not hex) have + // non-octal digits, no part of the ip can be in octal + // XXX: this came from the old javascript implementation, is it really + // supposed to be like this? + bool allowOctal = true; + uint32_t i; + + for (i = 0; i < parts.Length(); i++) { + const nsCString& part = parts[i]; + if (part[0] == '0') { + for (uint32_t j = 1; j < part.Length(); j++) { + if (part[j] == 'x') { + break; + } + if (part[j] == '8' || part[j] == '9') { + allowOctal = false; + break; + } + } + } + } + + for (i = 0; i < parts.Length(); i++) { + nsAutoCString canonical; + + if (i == parts.Length() - 1) { + CanonicalNum(parts[i], 5 - parts.Length(), allowOctal, canonical); + } else { + CanonicalNum(parts[i], 1, allowOctal, canonical); + } + + if (canonical.IsEmpty()) { + _retval.Truncate(); + return; + } + + if (_retval.IsEmpty()) { + _retval.Assign(canonical); + } else { + _retval.Append('.'); + _retval.Append(canonical); + } + } + return; +} + +void +nsUrlClassifierUtils::CanonicalNum(const nsACString& num, + uint32_t bytes, + bool allowOctal, + nsACString& _retval) +{ + _retval.Truncate(); + + if (num.Length() < 1) { + return; + } + + uint32_t val; + if (allowOctal && IsOctal(num)) { + if (PR_sscanf(PromiseFlatCString(num).get(), "%o", &val) != 1) { + return; + } + } else if (IsDecimal(num)) { + if (PR_sscanf(PromiseFlatCString(num).get(), "%u", &val) != 1) { + return; + } + } else if (IsHex(num)) { + if (PR_sscanf(PromiseFlatCString(num).get(), num[1] == 'X' ? "0X%x" : "0x%x", + &val) != 1) { + return; + } + } else { + return; + } + + while (bytes--) { + char buf[20]; + SprintfLiteral(buf, "%u", val & 0xff); + if (_retval.IsEmpty()) { + _retval.Assign(buf); + } else { + _retval = nsDependentCString(buf) + NS_LITERAL_CSTRING(".") + _retval; + } + val >>= 8; + } +} + +// This function will encode all "special" characters in typical url +// encoding, that is %hh where h is a valid hex digit. It will also fold +// any duplicated slashes. +bool +nsUrlClassifierUtils::SpecialEncode(const nsACString & url, + bool foldSlashes, + nsACString & _retval) +{ + bool changed = false; + const char* curChar = url.BeginReading(); + const char* end = url.EndReading(); + + unsigned char lastChar = '\0'; + while (curChar != end) { + unsigned char c = static_cast<unsigned char>(*curChar); + if (ShouldURLEscape(c)) { + _retval.Append('%'); + _retval.Append(int_to_hex_digit(c / 16)); + _retval.Append(int_to_hex_digit(c % 16)); + + changed = true; + } else if (foldSlashes && (c == '/' && lastChar == '/')) { + // skip + } else { + _retval.Append(*curChar); + } + lastChar = c; + curChar++; + } + return changed; +} + +bool +nsUrlClassifierUtils::ShouldURLEscape(const unsigned char c) const +{ + return c <= 32 || c == '%' || c >=127; +} diff --git a/toolkit/components/url-classifier/nsUrlClassifierUtils.h b/toolkit/components/url-classifier/nsUrlClassifierUtils.h new file mode 100644 index 000000000..cd14cf2a7 --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierUtils.h @@ -0,0 +1,99 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsUrlClassifierUtils_h_ +#define nsUrlClassifierUtils_h_ + +#include "nsAutoPtr.h" +#include "nsIUrlClassifierUtils.h" +#include "nsClassHashtable.h" +#include "nsIObserver.h" + +class nsUrlClassifierUtils final : public nsIUrlClassifierUtils, + public nsIObserver +{ +public: + typedef nsClassHashtable<nsCStringHashKey, nsCString> ProviderDictType; + +private: + /** + * A fast, bit-vector map for ascii characters. + * + * Internally stores 256 bits in an array of 8 ints. + * Does quick bit-flicking to lookup needed characters. + */ + class Charmap + { + public: + Charmap(uint32_t b0, uint32_t b1, uint32_t b2, uint32_t b3, + uint32_t b4, uint32_t b5, uint32_t b6, uint32_t b7) + { + mMap[0] = b0; mMap[1] = b1; mMap[2] = b2; mMap[3] = b3; + mMap[4] = b4; mMap[5] = b5; mMap[6] = b6; mMap[7] = b7; + } + + /** + * Do a quick lookup to see if the letter is in the map. + */ + bool Contains(unsigned char c) const + { + return mMap[c >> 5] & (1 << (c & 31)); + } + + private: + // Store the 256 bits in an 8 byte array. + uint32_t mMap[8]; + }; + + +public: + nsUrlClassifierUtils(); + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIURLCLASSIFIERUTILS + NS_DECL_NSIOBSERVER + + nsresult Init(); + + nsresult CanonicalizeHostname(const nsACString & hostname, + nsACString & _retval); + nsresult CanonicalizePath(const nsACString & url, nsACString & _retval); + + // This function will encode all "special" characters in typical url encoding, + // that is %hh where h is a valid hex digit. The characters which are encoded + // by this function are any ascii characters under 32(control characters and + // space), 37(%), and anything 127 or above (special characters). Url is the + // string to encode, ret is the encoded string. Function returns true if + // ret != url. + bool SpecialEncode(const nsACString & url, + bool foldSlashes, + nsACString & _retval); + + void ParseIPAddress(const nsACString & host, nsACString & _retval); + void CanonicalNum(const nsACString & num, + uint32_t bytes, + bool allowOctal, + nsACString & _retval); + +private: + ~nsUrlClassifierUtils() {} + + // Disallow copy constructor + nsUrlClassifierUtils(const nsUrlClassifierUtils&); + + // Function to tell if we should encode a character. + bool ShouldURLEscape(const unsigned char c) const; + + void CleanupHostname(const nsACString & host, nsACString & _retval); + + nsresult ReadProvidersFromPrefs(ProviderDictType& aDict); + + nsAutoPtr<Charmap> mEscapeCharmap; + + // The provider lookup table and its mutex. + ProviderDictType mProviderDict; + mozilla::Mutex mProviderDictLock; +}; + +#endif // nsUrlClassifierUtils_h_ diff --git a/toolkit/components/url-classifier/protobuf/safebrowsing.pb.cc b/toolkit/components/url-classifier/protobuf/safebrowsing.pb.cc new file mode 100644 index 000000000..d3e49251b --- /dev/null +++ b/toolkit/components/url-classifier/protobuf/safebrowsing.pb.cc @@ -0,0 +1,7166 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: safebrowsing.proto + +#define INTERNAL_SUPPRESS_PROTOBUF_FIELD_DEPRECATION +#include "safebrowsing.pb.h" + +#include <algorithm> + +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/stubs/once.h> +#include <google/protobuf/io/coded_stream.h> +#include <google/protobuf/wire_format_lite_inl.h> +#include <google/protobuf/io/zero_copy_stream_impl_lite.h> +// @@protoc_insertion_point(includes) + +namespace mozilla { +namespace safebrowsing { + +void protobuf_ShutdownFile_safebrowsing_2eproto() { + delete ThreatInfo::default_instance_; + delete ThreatMatch::default_instance_; + delete FindThreatMatchesRequest::default_instance_; + delete FindThreatMatchesResponse::default_instance_; + delete FetchThreatListUpdatesRequest::default_instance_; + delete FetchThreatListUpdatesRequest_ListUpdateRequest::default_instance_; + delete FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::default_instance_; + delete FetchThreatListUpdatesResponse::default_instance_; + delete FetchThreatListUpdatesResponse_ListUpdateResponse::default_instance_; + delete FindFullHashesRequest::default_instance_; + delete FindFullHashesResponse::default_instance_; + delete ThreatHit::default_instance_; + delete ThreatHit_ThreatSource::default_instance_; + delete ClientInfo::default_instance_; + delete Checksum::default_instance_; + delete ThreatEntry::default_instance_; + delete ThreatEntrySet::default_instance_; + delete RawIndices::default_instance_; + delete RawHashes::default_instance_; + delete RiceDeltaEncoding::default_instance_; + delete ThreatEntryMetadata::default_instance_; + delete ThreatEntryMetadata_MetadataEntry::default_instance_; + delete ThreatListDescriptor::default_instance_; + delete ListThreatListsResponse::default_instance_; + delete Duration::default_instance_; +} + +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER +void protobuf_AddDesc_safebrowsing_2eproto_impl() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + +#else +void protobuf_AddDesc_safebrowsing_2eproto() { + static bool already_here = false; + if (already_here) return; + already_here = true; + GOOGLE_PROTOBUF_VERIFY_VERSION; + +#endif + ThreatInfo::default_instance_ = new ThreatInfo(); + ThreatMatch::default_instance_ = new ThreatMatch(); + FindThreatMatchesRequest::default_instance_ = new FindThreatMatchesRequest(); + FindThreatMatchesResponse::default_instance_ = new FindThreatMatchesResponse(); + FetchThreatListUpdatesRequest::default_instance_ = new FetchThreatListUpdatesRequest(); + FetchThreatListUpdatesRequest_ListUpdateRequest::default_instance_ = new FetchThreatListUpdatesRequest_ListUpdateRequest(); + FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::default_instance_ = new FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints(); + FetchThreatListUpdatesResponse::default_instance_ = new FetchThreatListUpdatesResponse(); + FetchThreatListUpdatesResponse_ListUpdateResponse::default_instance_ = new FetchThreatListUpdatesResponse_ListUpdateResponse(); + FindFullHashesRequest::default_instance_ = new FindFullHashesRequest(); + FindFullHashesResponse::default_instance_ = new FindFullHashesResponse(); + ThreatHit::default_instance_ = new ThreatHit(); + ThreatHit_ThreatSource::default_instance_ = new ThreatHit_ThreatSource(); + ClientInfo::default_instance_ = new ClientInfo(); + Checksum::default_instance_ = new Checksum(); + ThreatEntry::default_instance_ = new ThreatEntry(); + ThreatEntrySet::default_instance_ = new ThreatEntrySet(); + RawIndices::default_instance_ = new RawIndices(); + RawHashes::default_instance_ = new RawHashes(); + RiceDeltaEncoding::default_instance_ = new RiceDeltaEncoding(); + ThreatEntryMetadata::default_instance_ = new ThreatEntryMetadata(); + ThreatEntryMetadata_MetadataEntry::default_instance_ = new ThreatEntryMetadata_MetadataEntry(); + ThreatListDescriptor::default_instance_ = new ThreatListDescriptor(); + ListThreatListsResponse::default_instance_ = new ListThreatListsResponse(); + Duration::default_instance_ = new Duration(); + ThreatInfo::default_instance_->InitAsDefaultInstance(); + ThreatMatch::default_instance_->InitAsDefaultInstance(); + FindThreatMatchesRequest::default_instance_->InitAsDefaultInstance(); + FindThreatMatchesResponse::default_instance_->InitAsDefaultInstance(); + FetchThreatListUpdatesRequest::default_instance_->InitAsDefaultInstance(); + FetchThreatListUpdatesRequest_ListUpdateRequest::default_instance_->InitAsDefaultInstance(); + FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::default_instance_->InitAsDefaultInstance(); + FetchThreatListUpdatesResponse::default_instance_->InitAsDefaultInstance(); + FetchThreatListUpdatesResponse_ListUpdateResponse::default_instance_->InitAsDefaultInstance(); + FindFullHashesRequest::default_instance_->InitAsDefaultInstance(); + FindFullHashesResponse::default_instance_->InitAsDefaultInstance(); + ThreatHit::default_instance_->InitAsDefaultInstance(); + ThreatHit_ThreatSource::default_instance_->InitAsDefaultInstance(); + ClientInfo::default_instance_->InitAsDefaultInstance(); + Checksum::default_instance_->InitAsDefaultInstance(); + ThreatEntry::default_instance_->InitAsDefaultInstance(); + ThreatEntrySet::default_instance_->InitAsDefaultInstance(); + RawIndices::default_instance_->InitAsDefaultInstance(); + RawHashes::default_instance_->InitAsDefaultInstance(); + RiceDeltaEncoding::default_instance_->InitAsDefaultInstance(); + ThreatEntryMetadata::default_instance_->InitAsDefaultInstance(); + ThreatEntryMetadata_MetadataEntry::default_instance_->InitAsDefaultInstance(); + ThreatListDescriptor::default_instance_->InitAsDefaultInstance(); + ListThreatListsResponse::default_instance_->InitAsDefaultInstance(); + Duration::default_instance_->InitAsDefaultInstance(); + ::google::protobuf::internal::OnShutdown(&protobuf_ShutdownFile_safebrowsing_2eproto); +} + +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER +GOOGLE_PROTOBUF_DECLARE_ONCE(protobuf_AddDesc_safebrowsing_2eproto_once_); +void protobuf_AddDesc_safebrowsing_2eproto() { + ::google::protobuf::GoogleOnceInit(&protobuf_AddDesc_safebrowsing_2eproto_once_, + &protobuf_AddDesc_safebrowsing_2eproto_impl); +} +#else +// Force AddDescriptors() to be called at static initialization time. +struct StaticDescriptorInitializer_safebrowsing_2eproto { + StaticDescriptorInitializer_safebrowsing_2eproto() { + protobuf_AddDesc_safebrowsing_2eproto(); + } +} static_descriptor_initializer_safebrowsing_2eproto_; +#endif +bool ThreatType_IsValid(int value) { + switch(value) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + return true; + default: + return false; + } +} + +bool PlatformType_IsValid(int value) { + switch(value) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + return true; + default: + return false; + } +} + +bool CompressionType_IsValid(int value) { + switch(value) { + case 0: + case 1: + case 2: + return true; + default: + return false; + } +} + +bool ThreatEntryType_IsValid(int value) { + switch(value) { + case 0: + case 1: + case 2: + case 3: + return true; + default: + return false; + } +} + + +// =================================================================== + +#ifndef _MSC_VER +const int ThreatInfo::kThreatTypesFieldNumber; +const int ThreatInfo::kPlatformTypesFieldNumber; +const int ThreatInfo::kThreatEntryTypesFieldNumber; +const int ThreatInfo::kThreatEntriesFieldNumber; +#endif // !_MSC_VER + +ThreatInfo::ThreatInfo() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.ThreatInfo) +} + +void ThreatInfo::InitAsDefaultInstance() { +} + +ThreatInfo::ThreatInfo(const ThreatInfo& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.ThreatInfo) +} + +void ThreatInfo::SharedCtor() { + _cached_size_ = 0; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +ThreatInfo::~ThreatInfo() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.ThreatInfo) + SharedDtor(); +} + +void ThreatInfo::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void ThreatInfo::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ThreatInfo& ThreatInfo::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +ThreatInfo* ThreatInfo::default_instance_ = NULL; + +ThreatInfo* ThreatInfo::New() const { + return new ThreatInfo; +} + +void ThreatInfo::Clear() { + threat_types_.Clear(); + platform_types_.Clear(); + threat_entry_types_.Clear(); + threat_entries_.Clear(); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool ThreatInfo::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.ThreatInfo) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // repeated .mozilla.safebrowsing.ThreatType threat_types = 1; + case 1: { + if (tag == 8) { + parse_threat_types: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::ThreatType_IsValid(value)) { + add_threat_types(static_cast< ::mozilla::safebrowsing::ThreatType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else if (tag == 10) { + DO_((::google::protobuf::internal::WireFormatLite::ReadPackedEnumNoInline( + input, + &::mozilla::safebrowsing::ThreatType_IsValid, + this->mutable_threat_types()))); + } else { + goto handle_unusual; + } + if (input->ExpectTag(8)) goto parse_threat_types; + if (input->ExpectTag(16)) goto parse_platform_types; + break; + } + + // repeated .mozilla.safebrowsing.PlatformType platform_types = 2; + case 2: { + if (tag == 16) { + parse_platform_types: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::PlatformType_IsValid(value)) { + add_platform_types(static_cast< ::mozilla::safebrowsing::PlatformType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else if (tag == 18) { + DO_((::google::protobuf::internal::WireFormatLite::ReadPackedEnumNoInline( + input, + &::mozilla::safebrowsing::PlatformType_IsValid, + this->mutable_platform_types()))); + } else { + goto handle_unusual; + } + if (input->ExpectTag(16)) goto parse_platform_types; + if (input->ExpectTag(26)) goto parse_threat_entries; + break; + } + + // repeated .mozilla.safebrowsing.ThreatEntry threat_entries = 3; + case 3: { + if (tag == 26) { + parse_threat_entries: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, add_threat_entries())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(26)) goto parse_threat_entries; + if (input->ExpectTag(32)) goto parse_threat_entry_types; + break; + } + + // repeated .mozilla.safebrowsing.ThreatEntryType threat_entry_types = 4; + case 4: { + if (tag == 32) { + parse_threat_entry_types: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::ThreatEntryType_IsValid(value)) { + add_threat_entry_types(static_cast< ::mozilla::safebrowsing::ThreatEntryType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else if (tag == 34) { + DO_((::google::protobuf::internal::WireFormatLite::ReadPackedEnumNoInline( + input, + &::mozilla::safebrowsing::ThreatEntryType_IsValid, + this->mutable_threat_entry_types()))); + } else { + goto handle_unusual; + } + if (input->ExpectTag(32)) goto parse_threat_entry_types; + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.ThreatInfo) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.ThreatInfo) + return false; +#undef DO_ +} + +void ThreatInfo::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.ThreatInfo) + // repeated .mozilla.safebrowsing.ThreatType threat_types = 1; + for (int i = 0; i < this->threat_types_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 1, this->threat_types(i), output); + } + + // repeated .mozilla.safebrowsing.PlatformType platform_types = 2; + for (int i = 0; i < this->platform_types_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 2, this->platform_types(i), output); + } + + // repeated .mozilla.safebrowsing.ThreatEntry threat_entries = 3; + for (int i = 0; i < this->threat_entries_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 3, this->threat_entries(i), output); + } + + // repeated .mozilla.safebrowsing.ThreatEntryType threat_entry_types = 4; + for (int i = 0; i < this->threat_entry_types_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 4, this->threat_entry_types(i), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.ThreatInfo) +} + +int ThreatInfo::ByteSize() const { + int total_size = 0; + + // repeated .mozilla.safebrowsing.ThreatType threat_types = 1; + { + int data_size = 0; + for (int i = 0; i < this->threat_types_size(); i++) { + data_size += ::google::protobuf::internal::WireFormatLite::EnumSize( + this->threat_types(i)); + } + total_size += 1 * this->threat_types_size() + data_size; + } + + // repeated .mozilla.safebrowsing.PlatformType platform_types = 2; + { + int data_size = 0; + for (int i = 0; i < this->platform_types_size(); i++) { + data_size += ::google::protobuf::internal::WireFormatLite::EnumSize( + this->platform_types(i)); + } + total_size += 1 * this->platform_types_size() + data_size; + } + + // repeated .mozilla.safebrowsing.ThreatEntryType threat_entry_types = 4; + { + int data_size = 0; + for (int i = 0; i < this->threat_entry_types_size(); i++) { + data_size += ::google::protobuf::internal::WireFormatLite::EnumSize( + this->threat_entry_types(i)); + } + total_size += 1 * this->threat_entry_types_size() + data_size; + } + + // repeated .mozilla.safebrowsing.ThreatEntry threat_entries = 3; + total_size += 1 * this->threat_entries_size(); + for (int i = 0; i < this->threat_entries_size(); i++) { + total_size += + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->threat_entries(i)); + } + + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void ThreatInfo::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const ThreatInfo*>(&from)); +} + +void ThreatInfo::MergeFrom(const ThreatInfo& from) { + GOOGLE_CHECK_NE(&from, this); + threat_types_.MergeFrom(from.threat_types_); + platform_types_.MergeFrom(from.platform_types_); + threat_entry_types_.MergeFrom(from.threat_entry_types_); + threat_entries_.MergeFrom(from.threat_entries_); + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void ThreatInfo::CopyFrom(const ThreatInfo& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool ThreatInfo::IsInitialized() const { + + return true; +} + +void ThreatInfo::Swap(ThreatInfo* other) { + if (other != this) { + threat_types_.Swap(&other->threat_types_); + platform_types_.Swap(&other->platform_types_); + threat_entry_types_.Swap(&other->threat_entry_types_); + threat_entries_.Swap(&other->threat_entries_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string ThreatInfo::GetTypeName() const { + return "mozilla.safebrowsing.ThreatInfo"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int ThreatMatch::kThreatTypeFieldNumber; +const int ThreatMatch::kPlatformTypeFieldNumber; +const int ThreatMatch::kThreatEntryTypeFieldNumber; +const int ThreatMatch::kThreatFieldNumber; +const int ThreatMatch::kThreatEntryMetadataFieldNumber; +const int ThreatMatch::kCacheDurationFieldNumber; +#endif // !_MSC_VER + +ThreatMatch::ThreatMatch() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.ThreatMatch) +} + +void ThreatMatch::InitAsDefaultInstance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + threat_ = const_cast< ::mozilla::safebrowsing::ThreatEntry*>( + ::mozilla::safebrowsing::ThreatEntry::internal_default_instance()); +#else + threat_ = const_cast< ::mozilla::safebrowsing::ThreatEntry*>(&::mozilla::safebrowsing::ThreatEntry::default_instance()); +#endif +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + threat_entry_metadata_ = const_cast< ::mozilla::safebrowsing::ThreatEntryMetadata*>( + ::mozilla::safebrowsing::ThreatEntryMetadata::internal_default_instance()); +#else + threat_entry_metadata_ = const_cast< ::mozilla::safebrowsing::ThreatEntryMetadata*>(&::mozilla::safebrowsing::ThreatEntryMetadata::default_instance()); +#endif +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + cache_duration_ = const_cast< ::mozilla::safebrowsing::Duration*>( + ::mozilla::safebrowsing::Duration::internal_default_instance()); +#else + cache_duration_ = const_cast< ::mozilla::safebrowsing::Duration*>(&::mozilla::safebrowsing::Duration::default_instance()); +#endif +} + +ThreatMatch::ThreatMatch(const ThreatMatch& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.ThreatMatch) +} + +void ThreatMatch::SharedCtor() { + _cached_size_ = 0; + threat_type_ = 0; + platform_type_ = 0; + threat_entry_type_ = 0; + threat_ = NULL; + threat_entry_metadata_ = NULL; + cache_duration_ = NULL; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +ThreatMatch::~ThreatMatch() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.ThreatMatch) + SharedDtor(); +} + +void ThreatMatch::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + delete threat_; + delete threat_entry_metadata_; + delete cache_duration_; + } +} + +void ThreatMatch::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ThreatMatch& ThreatMatch::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +ThreatMatch* ThreatMatch::default_instance_ = NULL; + +ThreatMatch* ThreatMatch::New() const { + return new ThreatMatch; +} + +void ThreatMatch::Clear() { +#define OFFSET_OF_FIELD_(f) (reinterpret_cast<char*>( \ + &reinterpret_cast<ThreatMatch*>(16)->f) - \ + reinterpret_cast<char*>(16)) + +#define ZR_(first, last) do { \ + size_t f = OFFSET_OF_FIELD_(first); \ + size_t n = OFFSET_OF_FIELD_(last) - f + sizeof(last); \ + ::memset(&first, 0, n); \ + } while (0) + + if (_has_bits_[0 / 32] & 63) { + ZR_(threat_type_, platform_type_); + threat_entry_type_ = 0; + if (has_threat()) { + if (threat_ != NULL) threat_->::mozilla::safebrowsing::ThreatEntry::Clear(); + } + if (has_threat_entry_metadata()) { + if (threat_entry_metadata_ != NULL) threat_entry_metadata_->::mozilla::safebrowsing::ThreatEntryMetadata::Clear(); + } + if (has_cache_duration()) { + if (cache_duration_ != NULL) cache_duration_->::mozilla::safebrowsing::Duration::Clear(); + } + } + +#undef OFFSET_OF_FIELD_ +#undef ZR_ + + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool ThreatMatch::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.ThreatMatch) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + case 1: { + if (tag == 8) { + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::ThreatType_IsValid(value)) { + set_threat_type(static_cast< ::mozilla::safebrowsing::ThreatType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectTag(16)) goto parse_platform_type; + break; + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + case 2: { + if (tag == 16) { + parse_platform_type: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::PlatformType_IsValid(value)) { + set_platform_type(static_cast< ::mozilla::safebrowsing::PlatformType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectTag(26)) goto parse_threat; + break; + } + + // optional .mozilla.safebrowsing.ThreatEntry threat = 3; + case 3: { + if (tag == 26) { + parse_threat: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_threat())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(34)) goto parse_threat_entry_metadata; + break; + } + + // optional .mozilla.safebrowsing.ThreatEntryMetadata threat_entry_metadata = 4; + case 4: { + if (tag == 34) { + parse_threat_entry_metadata: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_threat_entry_metadata())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(42)) goto parse_cache_duration; + break; + } + + // optional .mozilla.safebrowsing.Duration cache_duration = 5; + case 5: { + if (tag == 42) { + parse_cache_duration: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_cache_duration())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(48)) goto parse_threat_entry_type; + break; + } + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 6; + case 6: { + if (tag == 48) { + parse_threat_entry_type: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::ThreatEntryType_IsValid(value)) { + set_threat_entry_type(static_cast< ::mozilla::safebrowsing::ThreatEntryType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.ThreatMatch) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.ThreatMatch) + return false; +#undef DO_ +} + +void ThreatMatch::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.ThreatMatch) + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + if (has_threat_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 1, this->threat_type(), output); + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + if (has_platform_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 2, this->platform_type(), output); + } + + // optional .mozilla.safebrowsing.ThreatEntry threat = 3; + if (has_threat()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 3, this->threat(), output); + } + + // optional .mozilla.safebrowsing.ThreatEntryMetadata threat_entry_metadata = 4; + if (has_threat_entry_metadata()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 4, this->threat_entry_metadata(), output); + } + + // optional .mozilla.safebrowsing.Duration cache_duration = 5; + if (has_cache_duration()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 5, this->cache_duration(), output); + } + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 6; + if (has_threat_entry_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 6, this->threat_entry_type(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.ThreatMatch) +} + +int ThreatMatch::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + if (has_threat_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->threat_type()); + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + if (has_platform_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->platform_type()); + } + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 6; + if (has_threat_entry_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->threat_entry_type()); + } + + // optional .mozilla.safebrowsing.ThreatEntry threat = 3; + if (has_threat()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->threat()); + } + + // optional .mozilla.safebrowsing.ThreatEntryMetadata threat_entry_metadata = 4; + if (has_threat_entry_metadata()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->threat_entry_metadata()); + } + + // optional .mozilla.safebrowsing.Duration cache_duration = 5; + if (has_cache_duration()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->cache_duration()); + } + + } + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void ThreatMatch::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const ThreatMatch*>(&from)); +} + +void ThreatMatch::MergeFrom(const ThreatMatch& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_threat_type()) { + set_threat_type(from.threat_type()); + } + if (from.has_platform_type()) { + set_platform_type(from.platform_type()); + } + if (from.has_threat_entry_type()) { + set_threat_entry_type(from.threat_entry_type()); + } + if (from.has_threat()) { + mutable_threat()->::mozilla::safebrowsing::ThreatEntry::MergeFrom(from.threat()); + } + if (from.has_threat_entry_metadata()) { + mutable_threat_entry_metadata()->::mozilla::safebrowsing::ThreatEntryMetadata::MergeFrom(from.threat_entry_metadata()); + } + if (from.has_cache_duration()) { + mutable_cache_duration()->::mozilla::safebrowsing::Duration::MergeFrom(from.cache_duration()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void ThreatMatch::CopyFrom(const ThreatMatch& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool ThreatMatch::IsInitialized() const { + + return true; +} + +void ThreatMatch::Swap(ThreatMatch* other) { + if (other != this) { + std::swap(threat_type_, other->threat_type_); + std::swap(platform_type_, other->platform_type_); + std::swap(threat_entry_type_, other->threat_entry_type_); + std::swap(threat_, other->threat_); + std::swap(threat_entry_metadata_, other->threat_entry_metadata_); + std::swap(cache_duration_, other->cache_duration_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string ThreatMatch::GetTypeName() const { + return "mozilla.safebrowsing.ThreatMatch"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int FindThreatMatchesRequest::kClientFieldNumber; +const int FindThreatMatchesRequest::kThreatInfoFieldNumber; +#endif // !_MSC_VER + +FindThreatMatchesRequest::FindThreatMatchesRequest() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.FindThreatMatchesRequest) +} + +void FindThreatMatchesRequest::InitAsDefaultInstance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + client_ = const_cast< ::mozilla::safebrowsing::ClientInfo*>( + ::mozilla::safebrowsing::ClientInfo::internal_default_instance()); +#else + client_ = const_cast< ::mozilla::safebrowsing::ClientInfo*>(&::mozilla::safebrowsing::ClientInfo::default_instance()); +#endif +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + threat_info_ = const_cast< ::mozilla::safebrowsing::ThreatInfo*>( + ::mozilla::safebrowsing::ThreatInfo::internal_default_instance()); +#else + threat_info_ = const_cast< ::mozilla::safebrowsing::ThreatInfo*>(&::mozilla::safebrowsing::ThreatInfo::default_instance()); +#endif +} + +FindThreatMatchesRequest::FindThreatMatchesRequest(const FindThreatMatchesRequest& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.FindThreatMatchesRequest) +} + +void FindThreatMatchesRequest::SharedCtor() { + _cached_size_ = 0; + client_ = NULL; + threat_info_ = NULL; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +FindThreatMatchesRequest::~FindThreatMatchesRequest() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.FindThreatMatchesRequest) + SharedDtor(); +} + +void FindThreatMatchesRequest::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + delete client_; + delete threat_info_; + } +} + +void FindThreatMatchesRequest::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const FindThreatMatchesRequest& FindThreatMatchesRequest::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +FindThreatMatchesRequest* FindThreatMatchesRequest::default_instance_ = NULL; + +FindThreatMatchesRequest* FindThreatMatchesRequest::New() const { + return new FindThreatMatchesRequest; +} + +void FindThreatMatchesRequest::Clear() { + if (_has_bits_[0 / 32] & 3) { + if (has_client()) { + if (client_ != NULL) client_->::mozilla::safebrowsing::ClientInfo::Clear(); + } + if (has_threat_info()) { + if (threat_info_ != NULL) threat_info_->::mozilla::safebrowsing::ThreatInfo::Clear(); + } + } + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool FindThreatMatchesRequest::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.FindThreatMatchesRequest) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional .mozilla.safebrowsing.ClientInfo client = 1; + case 1: { + if (tag == 10) { + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_client())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(18)) goto parse_threat_info; + break; + } + + // optional .mozilla.safebrowsing.ThreatInfo threat_info = 2; + case 2: { + if (tag == 18) { + parse_threat_info: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_threat_info())); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.FindThreatMatchesRequest) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.FindThreatMatchesRequest) + return false; +#undef DO_ +} + +void FindThreatMatchesRequest::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.FindThreatMatchesRequest) + // optional .mozilla.safebrowsing.ClientInfo client = 1; + if (has_client()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 1, this->client(), output); + } + + // optional .mozilla.safebrowsing.ThreatInfo threat_info = 2; + if (has_threat_info()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 2, this->threat_info(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.FindThreatMatchesRequest) +} + +int FindThreatMatchesRequest::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional .mozilla.safebrowsing.ClientInfo client = 1; + if (has_client()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->client()); + } + + // optional .mozilla.safebrowsing.ThreatInfo threat_info = 2; + if (has_threat_info()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->threat_info()); + } + + } + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void FindThreatMatchesRequest::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const FindThreatMatchesRequest*>(&from)); +} + +void FindThreatMatchesRequest::MergeFrom(const FindThreatMatchesRequest& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_client()) { + mutable_client()->::mozilla::safebrowsing::ClientInfo::MergeFrom(from.client()); + } + if (from.has_threat_info()) { + mutable_threat_info()->::mozilla::safebrowsing::ThreatInfo::MergeFrom(from.threat_info()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void FindThreatMatchesRequest::CopyFrom(const FindThreatMatchesRequest& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool FindThreatMatchesRequest::IsInitialized() const { + + return true; +} + +void FindThreatMatchesRequest::Swap(FindThreatMatchesRequest* other) { + if (other != this) { + std::swap(client_, other->client_); + std::swap(threat_info_, other->threat_info_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string FindThreatMatchesRequest::GetTypeName() const { + return "mozilla.safebrowsing.FindThreatMatchesRequest"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int FindThreatMatchesResponse::kMatchesFieldNumber; +#endif // !_MSC_VER + +FindThreatMatchesResponse::FindThreatMatchesResponse() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.FindThreatMatchesResponse) +} + +void FindThreatMatchesResponse::InitAsDefaultInstance() { +} + +FindThreatMatchesResponse::FindThreatMatchesResponse(const FindThreatMatchesResponse& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.FindThreatMatchesResponse) +} + +void FindThreatMatchesResponse::SharedCtor() { + _cached_size_ = 0; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +FindThreatMatchesResponse::~FindThreatMatchesResponse() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.FindThreatMatchesResponse) + SharedDtor(); +} + +void FindThreatMatchesResponse::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void FindThreatMatchesResponse::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const FindThreatMatchesResponse& FindThreatMatchesResponse::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +FindThreatMatchesResponse* FindThreatMatchesResponse::default_instance_ = NULL; + +FindThreatMatchesResponse* FindThreatMatchesResponse::New() const { + return new FindThreatMatchesResponse; +} + +void FindThreatMatchesResponse::Clear() { + matches_.Clear(); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool FindThreatMatchesResponse::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.FindThreatMatchesResponse) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // repeated .mozilla.safebrowsing.ThreatMatch matches = 1; + case 1: { + if (tag == 10) { + parse_matches: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, add_matches())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(10)) goto parse_matches; + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.FindThreatMatchesResponse) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.FindThreatMatchesResponse) + return false; +#undef DO_ +} + +void FindThreatMatchesResponse::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.FindThreatMatchesResponse) + // repeated .mozilla.safebrowsing.ThreatMatch matches = 1; + for (int i = 0; i < this->matches_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 1, this->matches(i), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.FindThreatMatchesResponse) +} + +int FindThreatMatchesResponse::ByteSize() const { + int total_size = 0; + + // repeated .mozilla.safebrowsing.ThreatMatch matches = 1; + total_size += 1 * this->matches_size(); + for (int i = 0; i < this->matches_size(); i++) { + total_size += + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->matches(i)); + } + + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void FindThreatMatchesResponse::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const FindThreatMatchesResponse*>(&from)); +} + +void FindThreatMatchesResponse::MergeFrom(const FindThreatMatchesResponse& from) { + GOOGLE_CHECK_NE(&from, this); + matches_.MergeFrom(from.matches_); + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void FindThreatMatchesResponse::CopyFrom(const FindThreatMatchesResponse& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool FindThreatMatchesResponse::IsInitialized() const { + + return true; +} + +void FindThreatMatchesResponse::Swap(FindThreatMatchesResponse* other) { + if (other != this) { + matches_.Swap(&other->matches_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string FindThreatMatchesResponse::GetTypeName() const { + return "mozilla.safebrowsing.FindThreatMatchesResponse"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::kMaxUpdateEntriesFieldNumber; +const int FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::kMaxDatabaseEntriesFieldNumber; +const int FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::kRegionFieldNumber; +const int FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::kSupportedCompressionsFieldNumber; +#endif // !_MSC_VER + +FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints) +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::InitAsDefaultInstance() { +} + +FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints(const FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints) +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::SharedCtor() { + ::google::protobuf::internal::GetEmptyString(); + _cached_size_ = 0; + max_update_entries_ = 0; + max_database_entries_ = 0; + region_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::~FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints) + SharedDtor(); +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::SharedDtor() { + if (region_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete region_; + } + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints& FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::default_instance_ = NULL; + +FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::New() const { + return new FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints; +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::Clear() { +#define OFFSET_OF_FIELD_(f) (reinterpret_cast<char*>( \ + &reinterpret_cast<FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints*>(16)->f) - \ + reinterpret_cast<char*>(16)) + +#define ZR_(first, last) do { \ + size_t f = OFFSET_OF_FIELD_(first); \ + size_t n = OFFSET_OF_FIELD_(last) - f + sizeof(last); \ + ::memset(&first, 0, n); \ + } while (0) + + if (_has_bits_[0 / 32] & 7) { + ZR_(max_update_entries_, max_database_entries_); + if (has_region()) { + if (region_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + region_->clear(); + } + } + } + +#undef OFFSET_OF_FIELD_ +#undef ZR_ + + supported_compressions_.Clear(); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional int32 max_update_entries = 1; + case 1: { + if (tag == 8) { + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + input, &max_update_entries_))); + set_has_max_update_entries(); + } else { + goto handle_unusual; + } + if (input->ExpectTag(16)) goto parse_max_database_entries; + break; + } + + // optional int32 max_database_entries = 2; + case 2: { + if (tag == 16) { + parse_max_database_entries: + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + input, &max_database_entries_))); + set_has_max_database_entries(); + } else { + goto handle_unusual; + } + if (input->ExpectTag(26)) goto parse_region; + break; + } + + // optional string region = 3; + case 3: { + if (tag == 26) { + parse_region: + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_region())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(32)) goto parse_supported_compressions; + break; + } + + // repeated .mozilla.safebrowsing.CompressionType supported_compressions = 4; + case 4: { + if (tag == 32) { + parse_supported_compressions: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::CompressionType_IsValid(value)) { + add_supported_compressions(static_cast< ::mozilla::safebrowsing::CompressionType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else if (tag == 34) { + DO_((::google::protobuf::internal::WireFormatLite::ReadPackedEnumNoInline( + input, + &::mozilla::safebrowsing::CompressionType_IsValid, + this->mutable_supported_compressions()))); + } else { + goto handle_unusual; + } + if (input->ExpectTag(32)) goto parse_supported_compressions; + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints) + return false; +#undef DO_ +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints) + // optional int32 max_update_entries = 1; + if (has_max_update_entries()) { + ::google::protobuf::internal::WireFormatLite::WriteInt32(1, this->max_update_entries(), output); + } + + // optional int32 max_database_entries = 2; + if (has_max_database_entries()) { + ::google::protobuf::internal::WireFormatLite::WriteInt32(2, this->max_database_entries(), output); + } + + // optional string region = 3; + if (has_region()) { + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 3, this->region(), output); + } + + // repeated .mozilla.safebrowsing.CompressionType supported_compressions = 4; + for (int i = 0; i < this->supported_compressions_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 4, this->supported_compressions(i), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints) +} + +int FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional int32 max_update_entries = 1; + if (has_max_update_entries()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::Int32Size( + this->max_update_entries()); + } + + // optional int32 max_database_entries = 2; + if (has_max_database_entries()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::Int32Size( + this->max_database_entries()); + } + + // optional string region = 3; + if (has_region()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->region()); + } + + } + // repeated .mozilla.safebrowsing.CompressionType supported_compressions = 4; + { + int data_size = 0; + for (int i = 0; i < this->supported_compressions_size(); i++) { + data_size += ::google::protobuf::internal::WireFormatLite::EnumSize( + this->supported_compressions(i)); + } + total_size += 1 * this->supported_compressions_size() + data_size; + } + + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints*>(&from)); +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::MergeFrom(const FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints& from) { + GOOGLE_CHECK_NE(&from, this); + supported_compressions_.MergeFrom(from.supported_compressions_); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_max_update_entries()) { + set_max_update_entries(from.max_update_entries()); + } + if (from.has_max_database_entries()) { + set_max_database_entries(from.max_database_entries()); + } + if (from.has_region()) { + set_region(from.region()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::CopyFrom(const FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::IsInitialized() const { + + return true; +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::Swap(FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* other) { + if (other != this) { + std::swap(max_update_entries_, other->max_update_entries_); + std::swap(max_database_entries_, other->max_database_entries_); + std::swap(region_, other->region_); + supported_compressions_.Swap(&other->supported_compressions_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::GetTypeName() const { + return "mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints"; +} + + +// ------------------------------------------------------------------- + +#ifndef _MSC_VER +const int FetchThreatListUpdatesRequest_ListUpdateRequest::kThreatTypeFieldNumber; +const int FetchThreatListUpdatesRequest_ListUpdateRequest::kPlatformTypeFieldNumber; +const int FetchThreatListUpdatesRequest_ListUpdateRequest::kThreatEntryTypeFieldNumber; +const int FetchThreatListUpdatesRequest_ListUpdateRequest::kStateFieldNumber; +const int FetchThreatListUpdatesRequest_ListUpdateRequest::kConstraintsFieldNumber; +#endif // !_MSC_VER + +FetchThreatListUpdatesRequest_ListUpdateRequest::FetchThreatListUpdatesRequest_ListUpdateRequest() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest) +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest::InitAsDefaultInstance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + constraints_ = const_cast< ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints*>( + ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::internal_default_instance()); +#else + constraints_ = const_cast< ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints*>(&::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::default_instance()); +#endif +} + +FetchThreatListUpdatesRequest_ListUpdateRequest::FetchThreatListUpdatesRequest_ListUpdateRequest(const FetchThreatListUpdatesRequest_ListUpdateRequest& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest) +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest::SharedCtor() { + ::google::protobuf::internal::GetEmptyString(); + _cached_size_ = 0; + threat_type_ = 0; + platform_type_ = 0; + threat_entry_type_ = 0; + state_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + constraints_ = NULL; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +FetchThreatListUpdatesRequest_ListUpdateRequest::~FetchThreatListUpdatesRequest_ListUpdateRequest() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest) + SharedDtor(); +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest::SharedDtor() { + if (state_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete state_; + } + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + delete constraints_; + } +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const FetchThreatListUpdatesRequest_ListUpdateRequest& FetchThreatListUpdatesRequest_ListUpdateRequest::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +FetchThreatListUpdatesRequest_ListUpdateRequest* FetchThreatListUpdatesRequest_ListUpdateRequest::default_instance_ = NULL; + +FetchThreatListUpdatesRequest_ListUpdateRequest* FetchThreatListUpdatesRequest_ListUpdateRequest::New() const { + return new FetchThreatListUpdatesRequest_ListUpdateRequest; +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest::Clear() { +#define OFFSET_OF_FIELD_(f) (reinterpret_cast<char*>( \ + &reinterpret_cast<FetchThreatListUpdatesRequest_ListUpdateRequest*>(16)->f) - \ + reinterpret_cast<char*>(16)) + +#define ZR_(first, last) do { \ + size_t f = OFFSET_OF_FIELD_(first); \ + size_t n = OFFSET_OF_FIELD_(last) - f + sizeof(last); \ + ::memset(&first, 0, n); \ + } while (0) + + if (_has_bits_[0 / 32] & 31) { + ZR_(threat_type_, platform_type_); + threat_entry_type_ = 0; + if (has_state()) { + if (state_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + state_->clear(); + } + } + if (has_constraints()) { + if (constraints_ != NULL) constraints_->::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::Clear(); + } + } + +#undef OFFSET_OF_FIELD_ +#undef ZR_ + + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool FetchThreatListUpdatesRequest_ListUpdateRequest::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + case 1: { + if (tag == 8) { + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::ThreatType_IsValid(value)) { + set_threat_type(static_cast< ::mozilla::safebrowsing::ThreatType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectTag(16)) goto parse_platform_type; + break; + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + case 2: { + if (tag == 16) { + parse_platform_type: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::PlatformType_IsValid(value)) { + set_platform_type(static_cast< ::mozilla::safebrowsing::PlatformType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectTag(26)) goto parse_state; + break; + } + + // optional bytes state = 3; + case 3: { + if (tag == 26) { + parse_state: + DO_(::google::protobuf::internal::WireFormatLite::ReadBytes( + input, this->mutable_state())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(34)) goto parse_constraints; + break; + } + + // optional .mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints constraints = 4; + case 4: { + if (tag == 34) { + parse_constraints: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_constraints())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(40)) goto parse_threat_entry_type; + break; + } + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 5; + case 5: { + if (tag == 40) { + parse_threat_entry_type: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::ThreatEntryType_IsValid(value)) { + set_threat_entry_type(static_cast< ::mozilla::safebrowsing::ThreatEntryType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest) + return false; +#undef DO_ +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest) + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + if (has_threat_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 1, this->threat_type(), output); + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + if (has_platform_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 2, this->platform_type(), output); + } + + // optional bytes state = 3; + if (has_state()) { + ::google::protobuf::internal::WireFormatLite::WriteBytesMaybeAliased( + 3, this->state(), output); + } + + // optional .mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints constraints = 4; + if (has_constraints()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 4, this->constraints(), output); + } + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 5; + if (has_threat_entry_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 5, this->threat_entry_type(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest) +} + +int FetchThreatListUpdatesRequest_ListUpdateRequest::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + if (has_threat_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->threat_type()); + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + if (has_platform_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->platform_type()); + } + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 5; + if (has_threat_entry_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->threat_entry_type()); + } + + // optional bytes state = 3; + if (has_state()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::BytesSize( + this->state()); + } + + // optional .mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints constraints = 4; + if (has_constraints()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->constraints()); + } + + } + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const FetchThreatListUpdatesRequest_ListUpdateRequest*>(&from)); +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest::MergeFrom(const FetchThreatListUpdatesRequest_ListUpdateRequest& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_threat_type()) { + set_threat_type(from.threat_type()); + } + if (from.has_platform_type()) { + set_platform_type(from.platform_type()); + } + if (from.has_threat_entry_type()) { + set_threat_entry_type(from.threat_entry_type()); + } + if (from.has_state()) { + set_state(from.state()); + } + if (from.has_constraints()) { + mutable_constraints()->::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::MergeFrom(from.constraints()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest::CopyFrom(const FetchThreatListUpdatesRequest_ListUpdateRequest& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool FetchThreatListUpdatesRequest_ListUpdateRequest::IsInitialized() const { + + return true; +} + +void FetchThreatListUpdatesRequest_ListUpdateRequest::Swap(FetchThreatListUpdatesRequest_ListUpdateRequest* other) { + if (other != this) { + std::swap(threat_type_, other->threat_type_); + std::swap(platform_type_, other->platform_type_); + std::swap(threat_entry_type_, other->threat_entry_type_); + std::swap(state_, other->state_); + std::swap(constraints_, other->constraints_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string FetchThreatListUpdatesRequest_ListUpdateRequest::GetTypeName() const { + return "mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest"; +} + + +// ------------------------------------------------------------------- + +#ifndef _MSC_VER +const int FetchThreatListUpdatesRequest::kClientFieldNumber; +const int FetchThreatListUpdatesRequest::kListUpdateRequestsFieldNumber; +#endif // !_MSC_VER + +FetchThreatListUpdatesRequest::FetchThreatListUpdatesRequest() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.FetchThreatListUpdatesRequest) +} + +void FetchThreatListUpdatesRequest::InitAsDefaultInstance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + client_ = const_cast< ::mozilla::safebrowsing::ClientInfo*>( + ::mozilla::safebrowsing::ClientInfo::internal_default_instance()); +#else + client_ = const_cast< ::mozilla::safebrowsing::ClientInfo*>(&::mozilla::safebrowsing::ClientInfo::default_instance()); +#endif +} + +FetchThreatListUpdatesRequest::FetchThreatListUpdatesRequest(const FetchThreatListUpdatesRequest& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.FetchThreatListUpdatesRequest) +} + +void FetchThreatListUpdatesRequest::SharedCtor() { + _cached_size_ = 0; + client_ = NULL; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +FetchThreatListUpdatesRequest::~FetchThreatListUpdatesRequest() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.FetchThreatListUpdatesRequest) + SharedDtor(); +} + +void FetchThreatListUpdatesRequest::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + delete client_; + } +} + +void FetchThreatListUpdatesRequest::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const FetchThreatListUpdatesRequest& FetchThreatListUpdatesRequest::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +FetchThreatListUpdatesRequest* FetchThreatListUpdatesRequest::default_instance_ = NULL; + +FetchThreatListUpdatesRequest* FetchThreatListUpdatesRequest::New() const { + return new FetchThreatListUpdatesRequest; +} + +void FetchThreatListUpdatesRequest::Clear() { + if (has_client()) { + if (client_ != NULL) client_->::mozilla::safebrowsing::ClientInfo::Clear(); + } + list_update_requests_.Clear(); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool FetchThreatListUpdatesRequest::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.FetchThreatListUpdatesRequest) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional .mozilla.safebrowsing.ClientInfo client = 1; + case 1: { + if (tag == 10) { + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_client())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(26)) goto parse_list_update_requests; + break; + } + + // repeated .mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest list_update_requests = 3; + case 3: { + if (tag == 26) { + parse_list_update_requests: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, add_list_update_requests())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(26)) goto parse_list_update_requests; + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.FetchThreatListUpdatesRequest) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.FetchThreatListUpdatesRequest) + return false; +#undef DO_ +} + +void FetchThreatListUpdatesRequest::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.FetchThreatListUpdatesRequest) + // optional .mozilla.safebrowsing.ClientInfo client = 1; + if (has_client()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 1, this->client(), output); + } + + // repeated .mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest list_update_requests = 3; + for (int i = 0; i < this->list_update_requests_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 3, this->list_update_requests(i), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.FetchThreatListUpdatesRequest) +} + +int FetchThreatListUpdatesRequest::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional .mozilla.safebrowsing.ClientInfo client = 1; + if (has_client()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->client()); + } + + } + // repeated .mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest list_update_requests = 3; + total_size += 1 * this->list_update_requests_size(); + for (int i = 0; i < this->list_update_requests_size(); i++) { + total_size += + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->list_update_requests(i)); + } + + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void FetchThreatListUpdatesRequest::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const FetchThreatListUpdatesRequest*>(&from)); +} + +void FetchThreatListUpdatesRequest::MergeFrom(const FetchThreatListUpdatesRequest& from) { + GOOGLE_CHECK_NE(&from, this); + list_update_requests_.MergeFrom(from.list_update_requests_); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_client()) { + mutable_client()->::mozilla::safebrowsing::ClientInfo::MergeFrom(from.client()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void FetchThreatListUpdatesRequest::CopyFrom(const FetchThreatListUpdatesRequest& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool FetchThreatListUpdatesRequest::IsInitialized() const { + + return true; +} + +void FetchThreatListUpdatesRequest::Swap(FetchThreatListUpdatesRequest* other) { + if (other != this) { + std::swap(client_, other->client_); + list_update_requests_.Swap(&other->list_update_requests_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string FetchThreatListUpdatesRequest::GetTypeName() const { + return "mozilla.safebrowsing.FetchThreatListUpdatesRequest"; +} + + +// =================================================================== + +bool FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_IsValid(int value) { + switch(value) { + case 0: + case 1: + case 2: + return true; + default: + return false; + } +} + +#ifndef _MSC_VER +const FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType FetchThreatListUpdatesResponse_ListUpdateResponse::RESPONSE_TYPE_UNSPECIFIED; +const FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType FetchThreatListUpdatesResponse_ListUpdateResponse::PARTIAL_UPDATE; +const FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType FetchThreatListUpdatesResponse_ListUpdateResponse::FULL_UPDATE; +const FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType FetchThreatListUpdatesResponse_ListUpdateResponse::ResponseType_MIN; +const FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType FetchThreatListUpdatesResponse_ListUpdateResponse::ResponseType_MAX; +const int FetchThreatListUpdatesResponse_ListUpdateResponse::ResponseType_ARRAYSIZE; +#endif // _MSC_VER +#ifndef _MSC_VER +const int FetchThreatListUpdatesResponse_ListUpdateResponse::kThreatTypeFieldNumber; +const int FetchThreatListUpdatesResponse_ListUpdateResponse::kThreatEntryTypeFieldNumber; +const int FetchThreatListUpdatesResponse_ListUpdateResponse::kPlatformTypeFieldNumber; +const int FetchThreatListUpdatesResponse_ListUpdateResponse::kResponseTypeFieldNumber; +const int FetchThreatListUpdatesResponse_ListUpdateResponse::kAdditionsFieldNumber; +const int FetchThreatListUpdatesResponse_ListUpdateResponse::kRemovalsFieldNumber; +const int FetchThreatListUpdatesResponse_ListUpdateResponse::kNewClientStateFieldNumber; +const int FetchThreatListUpdatesResponse_ListUpdateResponse::kChecksumFieldNumber; +#endif // !_MSC_VER + +FetchThreatListUpdatesResponse_ListUpdateResponse::FetchThreatListUpdatesResponse_ListUpdateResponse() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse) +} + +void FetchThreatListUpdatesResponse_ListUpdateResponse::InitAsDefaultInstance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + checksum_ = const_cast< ::mozilla::safebrowsing::Checksum*>( + ::mozilla::safebrowsing::Checksum::internal_default_instance()); +#else + checksum_ = const_cast< ::mozilla::safebrowsing::Checksum*>(&::mozilla::safebrowsing::Checksum::default_instance()); +#endif +} + +FetchThreatListUpdatesResponse_ListUpdateResponse::FetchThreatListUpdatesResponse_ListUpdateResponse(const FetchThreatListUpdatesResponse_ListUpdateResponse& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse) +} + +void FetchThreatListUpdatesResponse_ListUpdateResponse::SharedCtor() { + ::google::protobuf::internal::GetEmptyString(); + _cached_size_ = 0; + threat_type_ = 0; + threat_entry_type_ = 0; + platform_type_ = 0; + response_type_ = 0; + new_client_state_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + checksum_ = NULL; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +FetchThreatListUpdatesResponse_ListUpdateResponse::~FetchThreatListUpdatesResponse_ListUpdateResponse() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse) + SharedDtor(); +} + +void FetchThreatListUpdatesResponse_ListUpdateResponse::SharedDtor() { + if (new_client_state_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete new_client_state_; + } + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + delete checksum_; + } +} + +void FetchThreatListUpdatesResponse_ListUpdateResponse::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const FetchThreatListUpdatesResponse_ListUpdateResponse& FetchThreatListUpdatesResponse_ListUpdateResponse::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +FetchThreatListUpdatesResponse_ListUpdateResponse* FetchThreatListUpdatesResponse_ListUpdateResponse::default_instance_ = NULL; + +FetchThreatListUpdatesResponse_ListUpdateResponse* FetchThreatListUpdatesResponse_ListUpdateResponse::New() const { + return new FetchThreatListUpdatesResponse_ListUpdateResponse; +} + +void FetchThreatListUpdatesResponse_ListUpdateResponse::Clear() { +#define OFFSET_OF_FIELD_(f) (reinterpret_cast<char*>( \ + &reinterpret_cast<FetchThreatListUpdatesResponse_ListUpdateResponse*>(16)->f) - \ + reinterpret_cast<char*>(16)) + +#define ZR_(first, last) do { \ + size_t f = OFFSET_OF_FIELD_(first); \ + size_t n = OFFSET_OF_FIELD_(last) - f + sizeof(last); \ + ::memset(&first, 0, n); \ + } while (0) + + if (_has_bits_[0 / 32] & 207) { + ZR_(threat_type_, response_type_); + if (has_new_client_state()) { + if (new_client_state_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + new_client_state_->clear(); + } + } + if (has_checksum()) { + if (checksum_ != NULL) checksum_->::mozilla::safebrowsing::Checksum::Clear(); + } + } + +#undef OFFSET_OF_FIELD_ +#undef ZR_ + + additions_.Clear(); + removals_.Clear(); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool FetchThreatListUpdatesResponse_ListUpdateResponse::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + case 1: { + if (tag == 8) { + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::ThreatType_IsValid(value)) { + set_threat_type(static_cast< ::mozilla::safebrowsing::ThreatType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectTag(16)) goto parse_threat_entry_type; + break; + } + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 2; + case 2: { + if (tag == 16) { + parse_threat_entry_type: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::ThreatEntryType_IsValid(value)) { + set_threat_entry_type(static_cast< ::mozilla::safebrowsing::ThreatEntryType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectTag(24)) goto parse_platform_type; + break; + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 3; + case 3: { + if (tag == 24) { + parse_platform_type: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::PlatformType_IsValid(value)) { + set_platform_type(static_cast< ::mozilla::safebrowsing::PlatformType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectTag(32)) goto parse_response_type; + break; + } + + // optional .mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.ResponseType response_type = 4; + case 4: { + if (tag == 32) { + parse_response_type: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_IsValid(value)) { + set_response_type(static_cast< ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectTag(42)) goto parse_additions; + break; + } + + // repeated .mozilla.safebrowsing.ThreatEntrySet additions = 5; + case 5: { + if (tag == 42) { + parse_additions: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, add_additions())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(42)) goto parse_additions; + if (input->ExpectTag(50)) goto parse_removals; + break; + } + + // repeated .mozilla.safebrowsing.ThreatEntrySet removals = 6; + case 6: { + if (tag == 50) { + parse_removals: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, add_removals())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(50)) goto parse_removals; + if (input->ExpectTag(58)) goto parse_new_client_state; + break; + } + + // optional bytes new_client_state = 7; + case 7: { + if (tag == 58) { + parse_new_client_state: + DO_(::google::protobuf::internal::WireFormatLite::ReadBytes( + input, this->mutable_new_client_state())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(66)) goto parse_checksum; + break; + } + + // optional .mozilla.safebrowsing.Checksum checksum = 8; + case 8: { + if (tag == 66) { + parse_checksum: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_checksum())); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse) + return false; +#undef DO_ +} + +void FetchThreatListUpdatesResponse_ListUpdateResponse::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse) + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + if (has_threat_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 1, this->threat_type(), output); + } + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 2; + if (has_threat_entry_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 2, this->threat_entry_type(), output); + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 3; + if (has_platform_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 3, this->platform_type(), output); + } + + // optional .mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.ResponseType response_type = 4; + if (has_response_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 4, this->response_type(), output); + } + + // repeated .mozilla.safebrowsing.ThreatEntrySet additions = 5; + for (int i = 0; i < this->additions_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 5, this->additions(i), output); + } + + // repeated .mozilla.safebrowsing.ThreatEntrySet removals = 6; + for (int i = 0; i < this->removals_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 6, this->removals(i), output); + } + + // optional bytes new_client_state = 7; + if (has_new_client_state()) { + ::google::protobuf::internal::WireFormatLite::WriteBytesMaybeAliased( + 7, this->new_client_state(), output); + } + + // optional .mozilla.safebrowsing.Checksum checksum = 8; + if (has_checksum()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 8, this->checksum(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse) +} + +int FetchThreatListUpdatesResponse_ListUpdateResponse::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + if (has_threat_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->threat_type()); + } + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 2; + if (has_threat_entry_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->threat_entry_type()); + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 3; + if (has_platform_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->platform_type()); + } + + // optional .mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.ResponseType response_type = 4; + if (has_response_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->response_type()); + } + + // optional bytes new_client_state = 7; + if (has_new_client_state()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::BytesSize( + this->new_client_state()); + } + + // optional .mozilla.safebrowsing.Checksum checksum = 8; + if (has_checksum()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->checksum()); + } + + } + // repeated .mozilla.safebrowsing.ThreatEntrySet additions = 5; + total_size += 1 * this->additions_size(); + for (int i = 0; i < this->additions_size(); i++) { + total_size += + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->additions(i)); + } + + // repeated .mozilla.safebrowsing.ThreatEntrySet removals = 6; + total_size += 1 * this->removals_size(); + for (int i = 0; i < this->removals_size(); i++) { + total_size += + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->removals(i)); + } + + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void FetchThreatListUpdatesResponse_ListUpdateResponse::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const FetchThreatListUpdatesResponse_ListUpdateResponse*>(&from)); +} + +void FetchThreatListUpdatesResponse_ListUpdateResponse::MergeFrom(const FetchThreatListUpdatesResponse_ListUpdateResponse& from) { + GOOGLE_CHECK_NE(&from, this); + additions_.MergeFrom(from.additions_); + removals_.MergeFrom(from.removals_); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_threat_type()) { + set_threat_type(from.threat_type()); + } + if (from.has_threat_entry_type()) { + set_threat_entry_type(from.threat_entry_type()); + } + if (from.has_platform_type()) { + set_platform_type(from.platform_type()); + } + if (from.has_response_type()) { + set_response_type(from.response_type()); + } + if (from.has_new_client_state()) { + set_new_client_state(from.new_client_state()); + } + if (from.has_checksum()) { + mutable_checksum()->::mozilla::safebrowsing::Checksum::MergeFrom(from.checksum()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void FetchThreatListUpdatesResponse_ListUpdateResponse::CopyFrom(const FetchThreatListUpdatesResponse_ListUpdateResponse& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool FetchThreatListUpdatesResponse_ListUpdateResponse::IsInitialized() const { + + return true; +} + +void FetchThreatListUpdatesResponse_ListUpdateResponse::Swap(FetchThreatListUpdatesResponse_ListUpdateResponse* other) { + if (other != this) { + std::swap(threat_type_, other->threat_type_); + std::swap(threat_entry_type_, other->threat_entry_type_); + std::swap(platform_type_, other->platform_type_); + std::swap(response_type_, other->response_type_); + additions_.Swap(&other->additions_); + removals_.Swap(&other->removals_); + std::swap(new_client_state_, other->new_client_state_); + std::swap(checksum_, other->checksum_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string FetchThreatListUpdatesResponse_ListUpdateResponse::GetTypeName() const { + return "mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse"; +} + + +// ------------------------------------------------------------------- + +#ifndef _MSC_VER +const int FetchThreatListUpdatesResponse::kListUpdateResponsesFieldNumber; +const int FetchThreatListUpdatesResponse::kMinimumWaitDurationFieldNumber; +#endif // !_MSC_VER + +FetchThreatListUpdatesResponse::FetchThreatListUpdatesResponse() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.FetchThreatListUpdatesResponse) +} + +void FetchThreatListUpdatesResponse::InitAsDefaultInstance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + minimum_wait_duration_ = const_cast< ::mozilla::safebrowsing::Duration*>( + ::mozilla::safebrowsing::Duration::internal_default_instance()); +#else + minimum_wait_duration_ = const_cast< ::mozilla::safebrowsing::Duration*>(&::mozilla::safebrowsing::Duration::default_instance()); +#endif +} + +FetchThreatListUpdatesResponse::FetchThreatListUpdatesResponse(const FetchThreatListUpdatesResponse& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.FetchThreatListUpdatesResponse) +} + +void FetchThreatListUpdatesResponse::SharedCtor() { + _cached_size_ = 0; + minimum_wait_duration_ = NULL; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +FetchThreatListUpdatesResponse::~FetchThreatListUpdatesResponse() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.FetchThreatListUpdatesResponse) + SharedDtor(); +} + +void FetchThreatListUpdatesResponse::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + delete minimum_wait_duration_; + } +} + +void FetchThreatListUpdatesResponse::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const FetchThreatListUpdatesResponse& FetchThreatListUpdatesResponse::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +FetchThreatListUpdatesResponse* FetchThreatListUpdatesResponse::default_instance_ = NULL; + +FetchThreatListUpdatesResponse* FetchThreatListUpdatesResponse::New() const { + return new FetchThreatListUpdatesResponse; +} + +void FetchThreatListUpdatesResponse::Clear() { + if (has_minimum_wait_duration()) { + if (minimum_wait_duration_ != NULL) minimum_wait_duration_->::mozilla::safebrowsing::Duration::Clear(); + } + list_update_responses_.Clear(); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool FetchThreatListUpdatesResponse::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.FetchThreatListUpdatesResponse) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // repeated .mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse list_update_responses = 1; + case 1: { + if (tag == 10) { + parse_list_update_responses: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, add_list_update_responses())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(10)) goto parse_list_update_responses; + if (input->ExpectTag(18)) goto parse_minimum_wait_duration; + break; + } + + // optional .mozilla.safebrowsing.Duration minimum_wait_duration = 2; + case 2: { + if (tag == 18) { + parse_minimum_wait_duration: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_minimum_wait_duration())); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.FetchThreatListUpdatesResponse) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.FetchThreatListUpdatesResponse) + return false; +#undef DO_ +} + +void FetchThreatListUpdatesResponse::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.FetchThreatListUpdatesResponse) + // repeated .mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse list_update_responses = 1; + for (int i = 0; i < this->list_update_responses_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 1, this->list_update_responses(i), output); + } + + // optional .mozilla.safebrowsing.Duration minimum_wait_duration = 2; + if (has_minimum_wait_duration()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 2, this->minimum_wait_duration(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.FetchThreatListUpdatesResponse) +} + +int FetchThreatListUpdatesResponse::ByteSize() const { + int total_size = 0; + + if (_has_bits_[1 / 32] & (0xffu << (1 % 32))) { + // optional .mozilla.safebrowsing.Duration minimum_wait_duration = 2; + if (has_minimum_wait_duration()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->minimum_wait_duration()); + } + + } + // repeated .mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse list_update_responses = 1; + total_size += 1 * this->list_update_responses_size(); + for (int i = 0; i < this->list_update_responses_size(); i++) { + total_size += + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->list_update_responses(i)); + } + + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void FetchThreatListUpdatesResponse::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const FetchThreatListUpdatesResponse*>(&from)); +} + +void FetchThreatListUpdatesResponse::MergeFrom(const FetchThreatListUpdatesResponse& from) { + GOOGLE_CHECK_NE(&from, this); + list_update_responses_.MergeFrom(from.list_update_responses_); + if (from._has_bits_[1 / 32] & (0xffu << (1 % 32))) { + if (from.has_minimum_wait_duration()) { + mutable_minimum_wait_duration()->::mozilla::safebrowsing::Duration::MergeFrom(from.minimum_wait_duration()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void FetchThreatListUpdatesResponse::CopyFrom(const FetchThreatListUpdatesResponse& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool FetchThreatListUpdatesResponse::IsInitialized() const { + + return true; +} + +void FetchThreatListUpdatesResponse::Swap(FetchThreatListUpdatesResponse* other) { + if (other != this) { + list_update_responses_.Swap(&other->list_update_responses_); + std::swap(minimum_wait_duration_, other->minimum_wait_duration_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string FetchThreatListUpdatesResponse::GetTypeName() const { + return "mozilla.safebrowsing.FetchThreatListUpdatesResponse"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int FindFullHashesRequest::kClientFieldNumber; +const int FindFullHashesRequest::kClientStatesFieldNumber; +const int FindFullHashesRequest::kThreatInfoFieldNumber; +#endif // !_MSC_VER + +FindFullHashesRequest::FindFullHashesRequest() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.FindFullHashesRequest) +} + +void FindFullHashesRequest::InitAsDefaultInstance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + client_ = const_cast< ::mozilla::safebrowsing::ClientInfo*>( + ::mozilla::safebrowsing::ClientInfo::internal_default_instance()); +#else + client_ = const_cast< ::mozilla::safebrowsing::ClientInfo*>(&::mozilla::safebrowsing::ClientInfo::default_instance()); +#endif +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + threat_info_ = const_cast< ::mozilla::safebrowsing::ThreatInfo*>( + ::mozilla::safebrowsing::ThreatInfo::internal_default_instance()); +#else + threat_info_ = const_cast< ::mozilla::safebrowsing::ThreatInfo*>(&::mozilla::safebrowsing::ThreatInfo::default_instance()); +#endif +} + +FindFullHashesRequest::FindFullHashesRequest(const FindFullHashesRequest& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.FindFullHashesRequest) +} + +void FindFullHashesRequest::SharedCtor() { + ::google::protobuf::internal::GetEmptyString(); + _cached_size_ = 0; + client_ = NULL; + threat_info_ = NULL; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +FindFullHashesRequest::~FindFullHashesRequest() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.FindFullHashesRequest) + SharedDtor(); +} + +void FindFullHashesRequest::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + delete client_; + delete threat_info_; + } +} + +void FindFullHashesRequest::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const FindFullHashesRequest& FindFullHashesRequest::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +FindFullHashesRequest* FindFullHashesRequest::default_instance_ = NULL; + +FindFullHashesRequest* FindFullHashesRequest::New() const { + return new FindFullHashesRequest; +} + +void FindFullHashesRequest::Clear() { + if (_has_bits_[0 / 32] & 5) { + if (has_client()) { + if (client_ != NULL) client_->::mozilla::safebrowsing::ClientInfo::Clear(); + } + if (has_threat_info()) { + if (threat_info_ != NULL) threat_info_->::mozilla::safebrowsing::ThreatInfo::Clear(); + } + } + client_states_.Clear(); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool FindFullHashesRequest::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.FindFullHashesRequest) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional .mozilla.safebrowsing.ClientInfo client = 1; + case 1: { + if (tag == 10) { + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_client())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(18)) goto parse_client_states; + break; + } + + // repeated bytes client_states = 2; + case 2: { + if (tag == 18) { + parse_client_states: + DO_(::google::protobuf::internal::WireFormatLite::ReadBytes( + input, this->add_client_states())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(18)) goto parse_client_states; + if (input->ExpectTag(26)) goto parse_threat_info; + break; + } + + // optional .mozilla.safebrowsing.ThreatInfo threat_info = 3; + case 3: { + if (tag == 26) { + parse_threat_info: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_threat_info())); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.FindFullHashesRequest) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.FindFullHashesRequest) + return false; +#undef DO_ +} + +void FindFullHashesRequest::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.FindFullHashesRequest) + // optional .mozilla.safebrowsing.ClientInfo client = 1; + if (has_client()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 1, this->client(), output); + } + + // repeated bytes client_states = 2; + for (int i = 0; i < this->client_states_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteBytes( + 2, this->client_states(i), output); + } + + // optional .mozilla.safebrowsing.ThreatInfo threat_info = 3; + if (has_threat_info()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 3, this->threat_info(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.FindFullHashesRequest) +} + +int FindFullHashesRequest::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional .mozilla.safebrowsing.ClientInfo client = 1; + if (has_client()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->client()); + } + + // optional .mozilla.safebrowsing.ThreatInfo threat_info = 3; + if (has_threat_info()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->threat_info()); + } + + } + // repeated bytes client_states = 2; + total_size += 1 * this->client_states_size(); + for (int i = 0; i < this->client_states_size(); i++) { + total_size += ::google::protobuf::internal::WireFormatLite::BytesSize( + this->client_states(i)); + } + + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void FindFullHashesRequest::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const FindFullHashesRequest*>(&from)); +} + +void FindFullHashesRequest::MergeFrom(const FindFullHashesRequest& from) { + GOOGLE_CHECK_NE(&from, this); + client_states_.MergeFrom(from.client_states_); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_client()) { + mutable_client()->::mozilla::safebrowsing::ClientInfo::MergeFrom(from.client()); + } + if (from.has_threat_info()) { + mutable_threat_info()->::mozilla::safebrowsing::ThreatInfo::MergeFrom(from.threat_info()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void FindFullHashesRequest::CopyFrom(const FindFullHashesRequest& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool FindFullHashesRequest::IsInitialized() const { + + return true; +} + +void FindFullHashesRequest::Swap(FindFullHashesRequest* other) { + if (other != this) { + std::swap(client_, other->client_); + client_states_.Swap(&other->client_states_); + std::swap(threat_info_, other->threat_info_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string FindFullHashesRequest::GetTypeName() const { + return "mozilla.safebrowsing.FindFullHashesRequest"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int FindFullHashesResponse::kMatchesFieldNumber; +const int FindFullHashesResponse::kMinimumWaitDurationFieldNumber; +const int FindFullHashesResponse::kNegativeCacheDurationFieldNumber; +#endif // !_MSC_VER + +FindFullHashesResponse::FindFullHashesResponse() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.FindFullHashesResponse) +} + +void FindFullHashesResponse::InitAsDefaultInstance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + minimum_wait_duration_ = const_cast< ::mozilla::safebrowsing::Duration*>( + ::mozilla::safebrowsing::Duration::internal_default_instance()); +#else + minimum_wait_duration_ = const_cast< ::mozilla::safebrowsing::Duration*>(&::mozilla::safebrowsing::Duration::default_instance()); +#endif +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + negative_cache_duration_ = const_cast< ::mozilla::safebrowsing::Duration*>( + ::mozilla::safebrowsing::Duration::internal_default_instance()); +#else + negative_cache_duration_ = const_cast< ::mozilla::safebrowsing::Duration*>(&::mozilla::safebrowsing::Duration::default_instance()); +#endif +} + +FindFullHashesResponse::FindFullHashesResponse(const FindFullHashesResponse& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.FindFullHashesResponse) +} + +void FindFullHashesResponse::SharedCtor() { + _cached_size_ = 0; + minimum_wait_duration_ = NULL; + negative_cache_duration_ = NULL; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +FindFullHashesResponse::~FindFullHashesResponse() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.FindFullHashesResponse) + SharedDtor(); +} + +void FindFullHashesResponse::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + delete minimum_wait_duration_; + delete negative_cache_duration_; + } +} + +void FindFullHashesResponse::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const FindFullHashesResponse& FindFullHashesResponse::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +FindFullHashesResponse* FindFullHashesResponse::default_instance_ = NULL; + +FindFullHashesResponse* FindFullHashesResponse::New() const { + return new FindFullHashesResponse; +} + +void FindFullHashesResponse::Clear() { + if (_has_bits_[0 / 32] & 6) { + if (has_minimum_wait_duration()) { + if (minimum_wait_duration_ != NULL) minimum_wait_duration_->::mozilla::safebrowsing::Duration::Clear(); + } + if (has_negative_cache_duration()) { + if (negative_cache_duration_ != NULL) negative_cache_duration_->::mozilla::safebrowsing::Duration::Clear(); + } + } + matches_.Clear(); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool FindFullHashesResponse::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.FindFullHashesResponse) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // repeated .mozilla.safebrowsing.ThreatMatch matches = 1; + case 1: { + if (tag == 10) { + parse_matches: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, add_matches())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(10)) goto parse_matches; + if (input->ExpectTag(18)) goto parse_minimum_wait_duration; + break; + } + + // optional .mozilla.safebrowsing.Duration minimum_wait_duration = 2; + case 2: { + if (tag == 18) { + parse_minimum_wait_duration: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_minimum_wait_duration())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(26)) goto parse_negative_cache_duration; + break; + } + + // optional .mozilla.safebrowsing.Duration negative_cache_duration = 3; + case 3: { + if (tag == 26) { + parse_negative_cache_duration: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_negative_cache_duration())); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.FindFullHashesResponse) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.FindFullHashesResponse) + return false; +#undef DO_ +} + +void FindFullHashesResponse::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.FindFullHashesResponse) + // repeated .mozilla.safebrowsing.ThreatMatch matches = 1; + for (int i = 0; i < this->matches_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 1, this->matches(i), output); + } + + // optional .mozilla.safebrowsing.Duration minimum_wait_duration = 2; + if (has_minimum_wait_duration()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 2, this->minimum_wait_duration(), output); + } + + // optional .mozilla.safebrowsing.Duration negative_cache_duration = 3; + if (has_negative_cache_duration()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 3, this->negative_cache_duration(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.FindFullHashesResponse) +} + +int FindFullHashesResponse::ByteSize() const { + int total_size = 0; + + if (_has_bits_[1 / 32] & (0xffu << (1 % 32))) { + // optional .mozilla.safebrowsing.Duration minimum_wait_duration = 2; + if (has_minimum_wait_duration()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->minimum_wait_duration()); + } + + // optional .mozilla.safebrowsing.Duration negative_cache_duration = 3; + if (has_negative_cache_duration()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->negative_cache_duration()); + } + + } + // repeated .mozilla.safebrowsing.ThreatMatch matches = 1; + total_size += 1 * this->matches_size(); + for (int i = 0; i < this->matches_size(); i++) { + total_size += + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->matches(i)); + } + + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void FindFullHashesResponse::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const FindFullHashesResponse*>(&from)); +} + +void FindFullHashesResponse::MergeFrom(const FindFullHashesResponse& from) { + GOOGLE_CHECK_NE(&from, this); + matches_.MergeFrom(from.matches_); + if (from._has_bits_[1 / 32] & (0xffu << (1 % 32))) { + if (from.has_minimum_wait_duration()) { + mutable_minimum_wait_duration()->::mozilla::safebrowsing::Duration::MergeFrom(from.minimum_wait_duration()); + } + if (from.has_negative_cache_duration()) { + mutable_negative_cache_duration()->::mozilla::safebrowsing::Duration::MergeFrom(from.negative_cache_duration()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void FindFullHashesResponse::CopyFrom(const FindFullHashesResponse& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool FindFullHashesResponse::IsInitialized() const { + + return true; +} + +void FindFullHashesResponse::Swap(FindFullHashesResponse* other) { + if (other != this) { + matches_.Swap(&other->matches_); + std::swap(minimum_wait_duration_, other->minimum_wait_duration_); + std::swap(negative_cache_duration_, other->negative_cache_duration_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string FindFullHashesResponse::GetTypeName() const { + return "mozilla.safebrowsing.FindFullHashesResponse"; +} + + +// =================================================================== + +bool ThreatHit_ThreatSourceType_IsValid(int value) { + switch(value) { + case 0: + case 1: + case 2: + case 3: + return true; + default: + return false; + } +} + +#ifndef _MSC_VER +const ThreatHit_ThreatSourceType ThreatHit::THREAT_SOURCE_TYPE_UNSPECIFIED; +const ThreatHit_ThreatSourceType ThreatHit::MATCHING_URL; +const ThreatHit_ThreatSourceType ThreatHit::TAB_URL; +const ThreatHit_ThreatSourceType ThreatHit::TAB_REDIRECT; +const ThreatHit_ThreatSourceType ThreatHit::ThreatSourceType_MIN; +const ThreatHit_ThreatSourceType ThreatHit::ThreatSourceType_MAX; +const int ThreatHit::ThreatSourceType_ARRAYSIZE; +#endif // _MSC_VER +#ifndef _MSC_VER +const int ThreatHit_ThreatSource::kUrlFieldNumber; +const int ThreatHit_ThreatSource::kTypeFieldNumber; +const int ThreatHit_ThreatSource::kRemoteIpFieldNumber; +const int ThreatHit_ThreatSource::kReferrerFieldNumber; +#endif // !_MSC_VER + +ThreatHit_ThreatSource::ThreatHit_ThreatSource() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.ThreatHit.ThreatSource) +} + +void ThreatHit_ThreatSource::InitAsDefaultInstance() { +} + +ThreatHit_ThreatSource::ThreatHit_ThreatSource(const ThreatHit_ThreatSource& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.ThreatHit.ThreatSource) +} + +void ThreatHit_ThreatSource::SharedCtor() { + ::google::protobuf::internal::GetEmptyString(); + _cached_size_ = 0; + url_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + type_ = 0; + remote_ip_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + referrer_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +ThreatHit_ThreatSource::~ThreatHit_ThreatSource() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.ThreatHit.ThreatSource) + SharedDtor(); +} + +void ThreatHit_ThreatSource::SharedDtor() { + if (url_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete url_; + } + if (remote_ip_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete remote_ip_; + } + if (referrer_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete referrer_; + } + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void ThreatHit_ThreatSource::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ThreatHit_ThreatSource& ThreatHit_ThreatSource::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +ThreatHit_ThreatSource* ThreatHit_ThreatSource::default_instance_ = NULL; + +ThreatHit_ThreatSource* ThreatHit_ThreatSource::New() const { + return new ThreatHit_ThreatSource; +} + +void ThreatHit_ThreatSource::Clear() { + if (_has_bits_[0 / 32] & 15) { + if (has_url()) { + if (url_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + url_->clear(); + } + } + type_ = 0; + if (has_remote_ip()) { + if (remote_ip_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + remote_ip_->clear(); + } + } + if (has_referrer()) { + if (referrer_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + referrer_->clear(); + } + } + } + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool ThreatHit_ThreatSource::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.ThreatHit.ThreatSource) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional string url = 1; + case 1: { + if (tag == 10) { + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_url())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(16)) goto parse_type; + break; + } + + // optional .mozilla.safebrowsing.ThreatHit.ThreatSourceType type = 2; + case 2: { + if (tag == 16) { + parse_type: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::ThreatHit_ThreatSourceType_IsValid(value)) { + set_type(static_cast< ::mozilla::safebrowsing::ThreatHit_ThreatSourceType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectTag(26)) goto parse_remote_ip; + break; + } + + // optional string remote_ip = 3; + case 3: { + if (tag == 26) { + parse_remote_ip: + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_remote_ip())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(34)) goto parse_referrer; + break; + } + + // optional string referrer = 4; + case 4: { + if (tag == 34) { + parse_referrer: + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_referrer())); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.ThreatHit.ThreatSource) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.ThreatHit.ThreatSource) + return false; +#undef DO_ +} + +void ThreatHit_ThreatSource::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.ThreatHit.ThreatSource) + // optional string url = 1; + if (has_url()) { + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 1, this->url(), output); + } + + // optional .mozilla.safebrowsing.ThreatHit.ThreatSourceType type = 2; + if (has_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 2, this->type(), output); + } + + // optional string remote_ip = 3; + if (has_remote_ip()) { + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 3, this->remote_ip(), output); + } + + // optional string referrer = 4; + if (has_referrer()) { + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 4, this->referrer(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.ThreatHit.ThreatSource) +} + +int ThreatHit_ThreatSource::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional string url = 1; + if (has_url()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->url()); + } + + // optional .mozilla.safebrowsing.ThreatHit.ThreatSourceType type = 2; + if (has_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->type()); + } + + // optional string remote_ip = 3; + if (has_remote_ip()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->remote_ip()); + } + + // optional string referrer = 4; + if (has_referrer()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->referrer()); + } + + } + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void ThreatHit_ThreatSource::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const ThreatHit_ThreatSource*>(&from)); +} + +void ThreatHit_ThreatSource::MergeFrom(const ThreatHit_ThreatSource& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_url()) { + set_url(from.url()); + } + if (from.has_type()) { + set_type(from.type()); + } + if (from.has_remote_ip()) { + set_remote_ip(from.remote_ip()); + } + if (from.has_referrer()) { + set_referrer(from.referrer()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void ThreatHit_ThreatSource::CopyFrom(const ThreatHit_ThreatSource& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool ThreatHit_ThreatSource::IsInitialized() const { + + return true; +} + +void ThreatHit_ThreatSource::Swap(ThreatHit_ThreatSource* other) { + if (other != this) { + std::swap(url_, other->url_); + std::swap(type_, other->type_); + std::swap(remote_ip_, other->remote_ip_); + std::swap(referrer_, other->referrer_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string ThreatHit_ThreatSource::GetTypeName() const { + return "mozilla.safebrowsing.ThreatHit.ThreatSource"; +} + + +// ------------------------------------------------------------------- + +#ifndef _MSC_VER +const int ThreatHit::kThreatTypeFieldNumber; +const int ThreatHit::kPlatformTypeFieldNumber; +const int ThreatHit::kEntryFieldNumber; +const int ThreatHit::kResourcesFieldNumber; +#endif // !_MSC_VER + +ThreatHit::ThreatHit() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.ThreatHit) +} + +void ThreatHit::InitAsDefaultInstance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + entry_ = const_cast< ::mozilla::safebrowsing::ThreatEntry*>( + ::mozilla::safebrowsing::ThreatEntry::internal_default_instance()); +#else + entry_ = const_cast< ::mozilla::safebrowsing::ThreatEntry*>(&::mozilla::safebrowsing::ThreatEntry::default_instance()); +#endif +} + +ThreatHit::ThreatHit(const ThreatHit& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.ThreatHit) +} + +void ThreatHit::SharedCtor() { + _cached_size_ = 0; + threat_type_ = 0; + platform_type_ = 0; + entry_ = NULL; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +ThreatHit::~ThreatHit() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.ThreatHit) + SharedDtor(); +} + +void ThreatHit::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + delete entry_; + } +} + +void ThreatHit::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ThreatHit& ThreatHit::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +ThreatHit* ThreatHit::default_instance_ = NULL; + +ThreatHit* ThreatHit::New() const { + return new ThreatHit; +} + +void ThreatHit::Clear() { +#define OFFSET_OF_FIELD_(f) (reinterpret_cast<char*>( \ + &reinterpret_cast<ThreatHit*>(16)->f) - \ + reinterpret_cast<char*>(16)) + +#define ZR_(first, last) do { \ + size_t f = OFFSET_OF_FIELD_(first); \ + size_t n = OFFSET_OF_FIELD_(last) - f + sizeof(last); \ + ::memset(&first, 0, n); \ + } while (0) + + if (_has_bits_[0 / 32] & 7) { + ZR_(threat_type_, platform_type_); + if (has_entry()) { + if (entry_ != NULL) entry_->::mozilla::safebrowsing::ThreatEntry::Clear(); + } + } + +#undef OFFSET_OF_FIELD_ +#undef ZR_ + + resources_.Clear(); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool ThreatHit::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.ThreatHit) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + case 1: { + if (tag == 8) { + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::ThreatType_IsValid(value)) { + set_threat_type(static_cast< ::mozilla::safebrowsing::ThreatType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectTag(16)) goto parse_platform_type; + break; + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + case 2: { + if (tag == 16) { + parse_platform_type: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::PlatformType_IsValid(value)) { + set_platform_type(static_cast< ::mozilla::safebrowsing::PlatformType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectTag(26)) goto parse_entry; + break; + } + + // optional .mozilla.safebrowsing.ThreatEntry entry = 3; + case 3: { + if (tag == 26) { + parse_entry: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_entry())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(34)) goto parse_resources; + break; + } + + // repeated .mozilla.safebrowsing.ThreatHit.ThreatSource resources = 4; + case 4: { + if (tag == 34) { + parse_resources: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, add_resources())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(34)) goto parse_resources; + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.ThreatHit) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.ThreatHit) + return false; +#undef DO_ +} + +void ThreatHit::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.ThreatHit) + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + if (has_threat_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 1, this->threat_type(), output); + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + if (has_platform_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 2, this->platform_type(), output); + } + + // optional .mozilla.safebrowsing.ThreatEntry entry = 3; + if (has_entry()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 3, this->entry(), output); + } + + // repeated .mozilla.safebrowsing.ThreatHit.ThreatSource resources = 4; + for (int i = 0; i < this->resources_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 4, this->resources(i), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.ThreatHit) +} + +int ThreatHit::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + if (has_threat_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->threat_type()); + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + if (has_platform_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->platform_type()); + } + + // optional .mozilla.safebrowsing.ThreatEntry entry = 3; + if (has_entry()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->entry()); + } + + } + // repeated .mozilla.safebrowsing.ThreatHit.ThreatSource resources = 4; + total_size += 1 * this->resources_size(); + for (int i = 0; i < this->resources_size(); i++) { + total_size += + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->resources(i)); + } + + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void ThreatHit::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const ThreatHit*>(&from)); +} + +void ThreatHit::MergeFrom(const ThreatHit& from) { + GOOGLE_CHECK_NE(&from, this); + resources_.MergeFrom(from.resources_); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_threat_type()) { + set_threat_type(from.threat_type()); + } + if (from.has_platform_type()) { + set_platform_type(from.platform_type()); + } + if (from.has_entry()) { + mutable_entry()->::mozilla::safebrowsing::ThreatEntry::MergeFrom(from.entry()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void ThreatHit::CopyFrom(const ThreatHit& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool ThreatHit::IsInitialized() const { + + return true; +} + +void ThreatHit::Swap(ThreatHit* other) { + if (other != this) { + std::swap(threat_type_, other->threat_type_); + std::swap(platform_type_, other->platform_type_); + std::swap(entry_, other->entry_); + resources_.Swap(&other->resources_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string ThreatHit::GetTypeName() const { + return "mozilla.safebrowsing.ThreatHit"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int ClientInfo::kClientIdFieldNumber; +const int ClientInfo::kClientVersionFieldNumber; +#endif // !_MSC_VER + +ClientInfo::ClientInfo() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.ClientInfo) +} + +void ClientInfo::InitAsDefaultInstance() { +} + +ClientInfo::ClientInfo(const ClientInfo& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.ClientInfo) +} + +void ClientInfo::SharedCtor() { + ::google::protobuf::internal::GetEmptyString(); + _cached_size_ = 0; + client_id_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + client_version_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +ClientInfo::~ClientInfo() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.ClientInfo) + SharedDtor(); +} + +void ClientInfo::SharedDtor() { + if (client_id_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete client_id_; + } + if (client_version_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete client_version_; + } + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void ClientInfo::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ClientInfo& ClientInfo::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +ClientInfo* ClientInfo::default_instance_ = NULL; + +ClientInfo* ClientInfo::New() const { + return new ClientInfo; +} + +void ClientInfo::Clear() { + if (_has_bits_[0 / 32] & 3) { + if (has_client_id()) { + if (client_id_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + client_id_->clear(); + } + } + if (has_client_version()) { + if (client_version_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + client_version_->clear(); + } + } + } + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool ClientInfo::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.ClientInfo) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional string client_id = 1; + case 1: { + if (tag == 10) { + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_client_id())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(18)) goto parse_client_version; + break; + } + + // optional string client_version = 2; + case 2: { + if (tag == 18) { + parse_client_version: + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_client_version())); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.ClientInfo) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.ClientInfo) + return false; +#undef DO_ +} + +void ClientInfo::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.ClientInfo) + // optional string client_id = 1; + if (has_client_id()) { + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 1, this->client_id(), output); + } + + // optional string client_version = 2; + if (has_client_version()) { + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 2, this->client_version(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.ClientInfo) +} + +int ClientInfo::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional string client_id = 1; + if (has_client_id()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->client_id()); + } + + // optional string client_version = 2; + if (has_client_version()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->client_version()); + } + + } + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void ClientInfo::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const ClientInfo*>(&from)); +} + +void ClientInfo::MergeFrom(const ClientInfo& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_client_id()) { + set_client_id(from.client_id()); + } + if (from.has_client_version()) { + set_client_version(from.client_version()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void ClientInfo::CopyFrom(const ClientInfo& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool ClientInfo::IsInitialized() const { + + return true; +} + +void ClientInfo::Swap(ClientInfo* other) { + if (other != this) { + std::swap(client_id_, other->client_id_); + std::swap(client_version_, other->client_version_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string ClientInfo::GetTypeName() const { + return "mozilla.safebrowsing.ClientInfo"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int Checksum::kSha256FieldNumber; +#endif // !_MSC_VER + +Checksum::Checksum() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.Checksum) +} + +void Checksum::InitAsDefaultInstance() { +} + +Checksum::Checksum(const Checksum& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.Checksum) +} + +void Checksum::SharedCtor() { + ::google::protobuf::internal::GetEmptyString(); + _cached_size_ = 0; + sha256_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +Checksum::~Checksum() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.Checksum) + SharedDtor(); +} + +void Checksum::SharedDtor() { + if (sha256_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete sha256_; + } + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void Checksum::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const Checksum& Checksum::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +Checksum* Checksum::default_instance_ = NULL; + +Checksum* Checksum::New() const { + return new Checksum; +} + +void Checksum::Clear() { + if (has_sha256()) { + if (sha256_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + sha256_->clear(); + } + } + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool Checksum::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.Checksum) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional bytes sha256 = 1; + case 1: { + if (tag == 10) { + DO_(::google::protobuf::internal::WireFormatLite::ReadBytes( + input, this->mutable_sha256())); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.Checksum) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.Checksum) + return false; +#undef DO_ +} + +void Checksum::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.Checksum) + // optional bytes sha256 = 1; + if (has_sha256()) { + ::google::protobuf::internal::WireFormatLite::WriteBytesMaybeAliased( + 1, this->sha256(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.Checksum) +} + +int Checksum::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional bytes sha256 = 1; + if (has_sha256()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::BytesSize( + this->sha256()); + } + + } + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void Checksum::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const Checksum*>(&from)); +} + +void Checksum::MergeFrom(const Checksum& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_sha256()) { + set_sha256(from.sha256()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void Checksum::CopyFrom(const Checksum& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool Checksum::IsInitialized() const { + + return true; +} + +void Checksum::Swap(Checksum* other) { + if (other != this) { + std::swap(sha256_, other->sha256_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string Checksum::GetTypeName() const { + return "mozilla.safebrowsing.Checksum"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int ThreatEntry::kHashFieldNumber; +const int ThreatEntry::kUrlFieldNumber; +#endif // !_MSC_VER + +ThreatEntry::ThreatEntry() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.ThreatEntry) +} + +void ThreatEntry::InitAsDefaultInstance() { +} + +ThreatEntry::ThreatEntry(const ThreatEntry& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.ThreatEntry) +} + +void ThreatEntry::SharedCtor() { + ::google::protobuf::internal::GetEmptyString(); + _cached_size_ = 0; + hash_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + url_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +ThreatEntry::~ThreatEntry() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.ThreatEntry) + SharedDtor(); +} + +void ThreatEntry::SharedDtor() { + if (hash_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete hash_; + } + if (url_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete url_; + } + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void ThreatEntry::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ThreatEntry& ThreatEntry::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +ThreatEntry* ThreatEntry::default_instance_ = NULL; + +ThreatEntry* ThreatEntry::New() const { + return new ThreatEntry; +} + +void ThreatEntry::Clear() { + if (_has_bits_[0 / 32] & 3) { + if (has_hash()) { + if (hash_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + hash_->clear(); + } + } + if (has_url()) { + if (url_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + url_->clear(); + } + } + } + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool ThreatEntry::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.ThreatEntry) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional bytes hash = 1; + case 1: { + if (tag == 10) { + DO_(::google::protobuf::internal::WireFormatLite::ReadBytes( + input, this->mutable_hash())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(18)) goto parse_url; + break; + } + + // optional string url = 2; + case 2: { + if (tag == 18) { + parse_url: + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_url())); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.ThreatEntry) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.ThreatEntry) + return false; +#undef DO_ +} + +void ThreatEntry::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.ThreatEntry) + // optional bytes hash = 1; + if (has_hash()) { + ::google::protobuf::internal::WireFormatLite::WriteBytesMaybeAliased( + 1, this->hash(), output); + } + + // optional string url = 2; + if (has_url()) { + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 2, this->url(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.ThreatEntry) +} + +int ThreatEntry::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional bytes hash = 1; + if (has_hash()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::BytesSize( + this->hash()); + } + + // optional string url = 2; + if (has_url()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->url()); + } + + } + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void ThreatEntry::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const ThreatEntry*>(&from)); +} + +void ThreatEntry::MergeFrom(const ThreatEntry& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_hash()) { + set_hash(from.hash()); + } + if (from.has_url()) { + set_url(from.url()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void ThreatEntry::CopyFrom(const ThreatEntry& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool ThreatEntry::IsInitialized() const { + + return true; +} + +void ThreatEntry::Swap(ThreatEntry* other) { + if (other != this) { + std::swap(hash_, other->hash_); + std::swap(url_, other->url_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string ThreatEntry::GetTypeName() const { + return "mozilla.safebrowsing.ThreatEntry"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int ThreatEntrySet::kCompressionTypeFieldNumber; +const int ThreatEntrySet::kRawHashesFieldNumber; +const int ThreatEntrySet::kRawIndicesFieldNumber; +const int ThreatEntrySet::kRiceHashesFieldNumber; +const int ThreatEntrySet::kRiceIndicesFieldNumber; +#endif // !_MSC_VER + +ThreatEntrySet::ThreatEntrySet() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.ThreatEntrySet) +} + +void ThreatEntrySet::InitAsDefaultInstance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + raw_hashes_ = const_cast< ::mozilla::safebrowsing::RawHashes*>( + ::mozilla::safebrowsing::RawHashes::internal_default_instance()); +#else + raw_hashes_ = const_cast< ::mozilla::safebrowsing::RawHashes*>(&::mozilla::safebrowsing::RawHashes::default_instance()); +#endif +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + raw_indices_ = const_cast< ::mozilla::safebrowsing::RawIndices*>( + ::mozilla::safebrowsing::RawIndices::internal_default_instance()); +#else + raw_indices_ = const_cast< ::mozilla::safebrowsing::RawIndices*>(&::mozilla::safebrowsing::RawIndices::default_instance()); +#endif +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + rice_hashes_ = const_cast< ::mozilla::safebrowsing::RiceDeltaEncoding*>( + ::mozilla::safebrowsing::RiceDeltaEncoding::internal_default_instance()); +#else + rice_hashes_ = const_cast< ::mozilla::safebrowsing::RiceDeltaEncoding*>(&::mozilla::safebrowsing::RiceDeltaEncoding::default_instance()); +#endif +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + rice_indices_ = const_cast< ::mozilla::safebrowsing::RiceDeltaEncoding*>( + ::mozilla::safebrowsing::RiceDeltaEncoding::internal_default_instance()); +#else + rice_indices_ = const_cast< ::mozilla::safebrowsing::RiceDeltaEncoding*>(&::mozilla::safebrowsing::RiceDeltaEncoding::default_instance()); +#endif +} + +ThreatEntrySet::ThreatEntrySet(const ThreatEntrySet& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.ThreatEntrySet) +} + +void ThreatEntrySet::SharedCtor() { + _cached_size_ = 0; + compression_type_ = 0; + raw_hashes_ = NULL; + raw_indices_ = NULL; + rice_hashes_ = NULL; + rice_indices_ = NULL; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +ThreatEntrySet::~ThreatEntrySet() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.ThreatEntrySet) + SharedDtor(); +} + +void ThreatEntrySet::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + delete raw_hashes_; + delete raw_indices_; + delete rice_hashes_; + delete rice_indices_; + } +} + +void ThreatEntrySet::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ThreatEntrySet& ThreatEntrySet::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +ThreatEntrySet* ThreatEntrySet::default_instance_ = NULL; + +ThreatEntrySet* ThreatEntrySet::New() const { + return new ThreatEntrySet; +} + +void ThreatEntrySet::Clear() { + if (_has_bits_[0 / 32] & 31) { + compression_type_ = 0; + if (has_raw_hashes()) { + if (raw_hashes_ != NULL) raw_hashes_->::mozilla::safebrowsing::RawHashes::Clear(); + } + if (has_raw_indices()) { + if (raw_indices_ != NULL) raw_indices_->::mozilla::safebrowsing::RawIndices::Clear(); + } + if (has_rice_hashes()) { + if (rice_hashes_ != NULL) rice_hashes_->::mozilla::safebrowsing::RiceDeltaEncoding::Clear(); + } + if (has_rice_indices()) { + if (rice_indices_ != NULL) rice_indices_->::mozilla::safebrowsing::RiceDeltaEncoding::Clear(); + } + } + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool ThreatEntrySet::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.ThreatEntrySet) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional .mozilla.safebrowsing.CompressionType compression_type = 1; + case 1: { + if (tag == 8) { + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::CompressionType_IsValid(value)) { + set_compression_type(static_cast< ::mozilla::safebrowsing::CompressionType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectTag(18)) goto parse_raw_hashes; + break; + } + + // optional .mozilla.safebrowsing.RawHashes raw_hashes = 2; + case 2: { + if (tag == 18) { + parse_raw_hashes: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_raw_hashes())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(26)) goto parse_raw_indices; + break; + } + + // optional .mozilla.safebrowsing.RawIndices raw_indices = 3; + case 3: { + if (tag == 26) { + parse_raw_indices: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_raw_indices())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(34)) goto parse_rice_hashes; + break; + } + + // optional .mozilla.safebrowsing.RiceDeltaEncoding rice_hashes = 4; + case 4: { + if (tag == 34) { + parse_rice_hashes: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_rice_hashes())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(42)) goto parse_rice_indices; + break; + } + + // optional .mozilla.safebrowsing.RiceDeltaEncoding rice_indices = 5; + case 5: { + if (tag == 42) { + parse_rice_indices: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, mutable_rice_indices())); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.ThreatEntrySet) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.ThreatEntrySet) + return false; +#undef DO_ +} + +void ThreatEntrySet::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.ThreatEntrySet) + // optional .mozilla.safebrowsing.CompressionType compression_type = 1; + if (has_compression_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 1, this->compression_type(), output); + } + + // optional .mozilla.safebrowsing.RawHashes raw_hashes = 2; + if (has_raw_hashes()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 2, this->raw_hashes(), output); + } + + // optional .mozilla.safebrowsing.RawIndices raw_indices = 3; + if (has_raw_indices()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 3, this->raw_indices(), output); + } + + // optional .mozilla.safebrowsing.RiceDeltaEncoding rice_hashes = 4; + if (has_rice_hashes()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 4, this->rice_hashes(), output); + } + + // optional .mozilla.safebrowsing.RiceDeltaEncoding rice_indices = 5; + if (has_rice_indices()) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 5, this->rice_indices(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.ThreatEntrySet) +} + +int ThreatEntrySet::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional .mozilla.safebrowsing.CompressionType compression_type = 1; + if (has_compression_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->compression_type()); + } + + // optional .mozilla.safebrowsing.RawHashes raw_hashes = 2; + if (has_raw_hashes()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->raw_hashes()); + } + + // optional .mozilla.safebrowsing.RawIndices raw_indices = 3; + if (has_raw_indices()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->raw_indices()); + } + + // optional .mozilla.safebrowsing.RiceDeltaEncoding rice_hashes = 4; + if (has_rice_hashes()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->rice_hashes()); + } + + // optional .mozilla.safebrowsing.RiceDeltaEncoding rice_indices = 5; + if (has_rice_indices()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->rice_indices()); + } + + } + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void ThreatEntrySet::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const ThreatEntrySet*>(&from)); +} + +void ThreatEntrySet::MergeFrom(const ThreatEntrySet& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_compression_type()) { + set_compression_type(from.compression_type()); + } + if (from.has_raw_hashes()) { + mutable_raw_hashes()->::mozilla::safebrowsing::RawHashes::MergeFrom(from.raw_hashes()); + } + if (from.has_raw_indices()) { + mutable_raw_indices()->::mozilla::safebrowsing::RawIndices::MergeFrom(from.raw_indices()); + } + if (from.has_rice_hashes()) { + mutable_rice_hashes()->::mozilla::safebrowsing::RiceDeltaEncoding::MergeFrom(from.rice_hashes()); + } + if (from.has_rice_indices()) { + mutable_rice_indices()->::mozilla::safebrowsing::RiceDeltaEncoding::MergeFrom(from.rice_indices()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void ThreatEntrySet::CopyFrom(const ThreatEntrySet& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool ThreatEntrySet::IsInitialized() const { + + return true; +} + +void ThreatEntrySet::Swap(ThreatEntrySet* other) { + if (other != this) { + std::swap(compression_type_, other->compression_type_); + std::swap(raw_hashes_, other->raw_hashes_); + std::swap(raw_indices_, other->raw_indices_); + std::swap(rice_hashes_, other->rice_hashes_); + std::swap(rice_indices_, other->rice_indices_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string ThreatEntrySet::GetTypeName() const { + return "mozilla.safebrowsing.ThreatEntrySet"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int RawIndices::kIndicesFieldNumber; +#endif // !_MSC_VER + +RawIndices::RawIndices() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.RawIndices) +} + +void RawIndices::InitAsDefaultInstance() { +} + +RawIndices::RawIndices(const RawIndices& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.RawIndices) +} + +void RawIndices::SharedCtor() { + _cached_size_ = 0; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +RawIndices::~RawIndices() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.RawIndices) + SharedDtor(); +} + +void RawIndices::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void RawIndices::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const RawIndices& RawIndices::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +RawIndices* RawIndices::default_instance_ = NULL; + +RawIndices* RawIndices::New() const { + return new RawIndices; +} + +void RawIndices::Clear() { + indices_.Clear(); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool RawIndices::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.RawIndices) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // repeated int32 indices = 1; + case 1: { + if (tag == 8) { + parse_indices: + DO_((::google::protobuf::internal::WireFormatLite::ReadRepeatedPrimitive< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + 1, 8, input, this->mutable_indices()))); + } else if (tag == 10) { + DO_((::google::protobuf::internal::WireFormatLite::ReadPackedPrimitiveNoInline< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + input, this->mutable_indices()))); + } else { + goto handle_unusual; + } + if (input->ExpectTag(8)) goto parse_indices; + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.RawIndices) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.RawIndices) + return false; +#undef DO_ +} + +void RawIndices::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.RawIndices) + // repeated int32 indices = 1; + for (int i = 0; i < this->indices_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteInt32( + 1, this->indices(i), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.RawIndices) +} + +int RawIndices::ByteSize() const { + int total_size = 0; + + // repeated int32 indices = 1; + { + int data_size = 0; + for (int i = 0; i < this->indices_size(); i++) { + data_size += ::google::protobuf::internal::WireFormatLite:: + Int32Size(this->indices(i)); + } + total_size += 1 * this->indices_size() + data_size; + } + + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void RawIndices::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const RawIndices*>(&from)); +} + +void RawIndices::MergeFrom(const RawIndices& from) { + GOOGLE_CHECK_NE(&from, this); + indices_.MergeFrom(from.indices_); + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void RawIndices::CopyFrom(const RawIndices& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool RawIndices::IsInitialized() const { + + return true; +} + +void RawIndices::Swap(RawIndices* other) { + if (other != this) { + indices_.Swap(&other->indices_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string RawIndices::GetTypeName() const { + return "mozilla.safebrowsing.RawIndices"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int RawHashes::kPrefixSizeFieldNumber; +const int RawHashes::kRawHashesFieldNumber; +#endif // !_MSC_VER + +RawHashes::RawHashes() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.RawHashes) +} + +void RawHashes::InitAsDefaultInstance() { +} + +RawHashes::RawHashes(const RawHashes& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.RawHashes) +} + +void RawHashes::SharedCtor() { + ::google::protobuf::internal::GetEmptyString(); + _cached_size_ = 0; + prefix_size_ = 0; + raw_hashes_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +RawHashes::~RawHashes() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.RawHashes) + SharedDtor(); +} + +void RawHashes::SharedDtor() { + if (raw_hashes_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete raw_hashes_; + } + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void RawHashes::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const RawHashes& RawHashes::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +RawHashes* RawHashes::default_instance_ = NULL; + +RawHashes* RawHashes::New() const { + return new RawHashes; +} + +void RawHashes::Clear() { + if (_has_bits_[0 / 32] & 3) { + prefix_size_ = 0; + if (has_raw_hashes()) { + if (raw_hashes_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + raw_hashes_->clear(); + } + } + } + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool RawHashes::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.RawHashes) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional int32 prefix_size = 1; + case 1: { + if (tag == 8) { + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + input, &prefix_size_))); + set_has_prefix_size(); + } else { + goto handle_unusual; + } + if (input->ExpectTag(18)) goto parse_raw_hashes; + break; + } + + // optional bytes raw_hashes = 2; + case 2: { + if (tag == 18) { + parse_raw_hashes: + DO_(::google::protobuf::internal::WireFormatLite::ReadBytes( + input, this->mutable_raw_hashes())); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.RawHashes) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.RawHashes) + return false; +#undef DO_ +} + +void RawHashes::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.RawHashes) + // optional int32 prefix_size = 1; + if (has_prefix_size()) { + ::google::protobuf::internal::WireFormatLite::WriteInt32(1, this->prefix_size(), output); + } + + // optional bytes raw_hashes = 2; + if (has_raw_hashes()) { + ::google::protobuf::internal::WireFormatLite::WriteBytesMaybeAliased( + 2, this->raw_hashes(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.RawHashes) +} + +int RawHashes::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional int32 prefix_size = 1; + if (has_prefix_size()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::Int32Size( + this->prefix_size()); + } + + // optional bytes raw_hashes = 2; + if (has_raw_hashes()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::BytesSize( + this->raw_hashes()); + } + + } + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void RawHashes::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const RawHashes*>(&from)); +} + +void RawHashes::MergeFrom(const RawHashes& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_prefix_size()) { + set_prefix_size(from.prefix_size()); + } + if (from.has_raw_hashes()) { + set_raw_hashes(from.raw_hashes()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void RawHashes::CopyFrom(const RawHashes& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool RawHashes::IsInitialized() const { + + return true; +} + +void RawHashes::Swap(RawHashes* other) { + if (other != this) { + std::swap(prefix_size_, other->prefix_size_); + std::swap(raw_hashes_, other->raw_hashes_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string RawHashes::GetTypeName() const { + return "mozilla.safebrowsing.RawHashes"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int RiceDeltaEncoding::kFirstValueFieldNumber; +const int RiceDeltaEncoding::kRiceParameterFieldNumber; +const int RiceDeltaEncoding::kNumEntriesFieldNumber; +const int RiceDeltaEncoding::kEncodedDataFieldNumber; +#endif // !_MSC_VER + +RiceDeltaEncoding::RiceDeltaEncoding() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.RiceDeltaEncoding) +} + +void RiceDeltaEncoding::InitAsDefaultInstance() { +} + +RiceDeltaEncoding::RiceDeltaEncoding(const RiceDeltaEncoding& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.RiceDeltaEncoding) +} + +void RiceDeltaEncoding::SharedCtor() { + ::google::protobuf::internal::GetEmptyString(); + _cached_size_ = 0; + first_value_ = GOOGLE_LONGLONG(0); + rice_parameter_ = 0; + num_entries_ = 0; + encoded_data_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +RiceDeltaEncoding::~RiceDeltaEncoding() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.RiceDeltaEncoding) + SharedDtor(); +} + +void RiceDeltaEncoding::SharedDtor() { + if (encoded_data_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete encoded_data_; + } + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void RiceDeltaEncoding::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const RiceDeltaEncoding& RiceDeltaEncoding::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +RiceDeltaEncoding* RiceDeltaEncoding::default_instance_ = NULL; + +RiceDeltaEncoding* RiceDeltaEncoding::New() const { + return new RiceDeltaEncoding; +} + +void RiceDeltaEncoding::Clear() { +#define OFFSET_OF_FIELD_(f) (reinterpret_cast<char*>( \ + &reinterpret_cast<RiceDeltaEncoding*>(16)->f) - \ + reinterpret_cast<char*>(16)) + +#define ZR_(first, last) do { \ + size_t f = OFFSET_OF_FIELD_(first); \ + size_t n = OFFSET_OF_FIELD_(last) - f + sizeof(last); \ + ::memset(&first, 0, n); \ + } while (0) + + if (_has_bits_[0 / 32] & 15) { + ZR_(first_value_, num_entries_); + if (has_encoded_data()) { + if (encoded_data_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + encoded_data_->clear(); + } + } + } + +#undef OFFSET_OF_FIELD_ +#undef ZR_ + + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool RiceDeltaEncoding::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.RiceDeltaEncoding) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional int64 first_value = 1; + case 1: { + if (tag == 8) { + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + ::google::protobuf::int64, ::google::protobuf::internal::WireFormatLite::TYPE_INT64>( + input, &first_value_))); + set_has_first_value(); + } else { + goto handle_unusual; + } + if (input->ExpectTag(16)) goto parse_rice_parameter; + break; + } + + // optional int32 rice_parameter = 2; + case 2: { + if (tag == 16) { + parse_rice_parameter: + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + input, &rice_parameter_))); + set_has_rice_parameter(); + } else { + goto handle_unusual; + } + if (input->ExpectTag(24)) goto parse_num_entries; + break; + } + + // optional int32 num_entries = 3; + case 3: { + if (tag == 24) { + parse_num_entries: + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + input, &num_entries_))); + set_has_num_entries(); + } else { + goto handle_unusual; + } + if (input->ExpectTag(34)) goto parse_encoded_data; + break; + } + + // optional bytes encoded_data = 4; + case 4: { + if (tag == 34) { + parse_encoded_data: + DO_(::google::protobuf::internal::WireFormatLite::ReadBytes( + input, this->mutable_encoded_data())); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.RiceDeltaEncoding) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.RiceDeltaEncoding) + return false; +#undef DO_ +} + +void RiceDeltaEncoding::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.RiceDeltaEncoding) + // optional int64 first_value = 1; + if (has_first_value()) { + ::google::protobuf::internal::WireFormatLite::WriteInt64(1, this->first_value(), output); + } + + // optional int32 rice_parameter = 2; + if (has_rice_parameter()) { + ::google::protobuf::internal::WireFormatLite::WriteInt32(2, this->rice_parameter(), output); + } + + // optional int32 num_entries = 3; + if (has_num_entries()) { + ::google::protobuf::internal::WireFormatLite::WriteInt32(3, this->num_entries(), output); + } + + // optional bytes encoded_data = 4; + if (has_encoded_data()) { + ::google::protobuf::internal::WireFormatLite::WriteBytesMaybeAliased( + 4, this->encoded_data(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.RiceDeltaEncoding) +} + +int RiceDeltaEncoding::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional int64 first_value = 1; + if (has_first_value()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::Int64Size( + this->first_value()); + } + + // optional int32 rice_parameter = 2; + if (has_rice_parameter()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::Int32Size( + this->rice_parameter()); + } + + // optional int32 num_entries = 3; + if (has_num_entries()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::Int32Size( + this->num_entries()); + } + + // optional bytes encoded_data = 4; + if (has_encoded_data()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::BytesSize( + this->encoded_data()); + } + + } + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void RiceDeltaEncoding::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const RiceDeltaEncoding*>(&from)); +} + +void RiceDeltaEncoding::MergeFrom(const RiceDeltaEncoding& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_first_value()) { + set_first_value(from.first_value()); + } + if (from.has_rice_parameter()) { + set_rice_parameter(from.rice_parameter()); + } + if (from.has_num_entries()) { + set_num_entries(from.num_entries()); + } + if (from.has_encoded_data()) { + set_encoded_data(from.encoded_data()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void RiceDeltaEncoding::CopyFrom(const RiceDeltaEncoding& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool RiceDeltaEncoding::IsInitialized() const { + + return true; +} + +void RiceDeltaEncoding::Swap(RiceDeltaEncoding* other) { + if (other != this) { + std::swap(first_value_, other->first_value_); + std::swap(rice_parameter_, other->rice_parameter_); + std::swap(num_entries_, other->num_entries_); + std::swap(encoded_data_, other->encoded_data_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string RiceDeltaEncoding::GetTypeName() const { + return "mozilla.safebrowsing.RiceDeltaEncoding"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int ThreatEntryMetadata_MetadataEntry::kKeyFieldNumber; +const int ThreatEntryMetadata_MetadataEntry::kValueFieldNumber; +#endif // !_MSC_VER + +ThreatEntryMetadata_MetadataEntry::ThreatEntryMetadata_MetadataEntry() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry) +} + +void ThreatEntryMetadata_MetadataEntry::InitAsDefaultInstance() { +} + +ThreatEntryMetadata_MetadataEntry::ThreatEntryMetadata_MetadataEntry(const ThreatEntryMetadata_MetadataEntry& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry) +} + +void ThreatEntryMetadata_MetadataEntry::SharedCtor() { + ::google::protobuf::internal::GetEmptyString(); + _cached_size_ = 0; + key_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + value_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +ThreatEntryMetadata_MetadataEntry::~ThreatEntryMetadata_MetadataEntry() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry) + SharedDtor(); +} + +void ThreatEntryMetadata_MetadataEntry::SharedDtor() { + if (key_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete key_; + } + if (value_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete value_; + } + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void ThreatEntryMetadata_MetadataEntry::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ThreatEntryMetadata_MetadataEntry& ThreatEntryMetadata_MetadataEntry::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +ThreatEntryMetadata_MetadataEntry* ThreatEntryMetadata_MetadataEntry::default_instance_ = NULL; + +ThreatEntryMetadata_MetadataEntry* ThreatEntryMetadata_MetadataEntry::New() const { + return new ThreatEntryMetadata_MetadataEntry; +} + +void ThreatEntryMetadata_MetadataEntry::Clear() { + if (_has_bits_[0 / 32] & 3) { + if (has_key()) { + if (key_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + key_->clear(); + } + } + if (has_value()) { + if (value_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + value_->clear(); + } + } + } + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool ThreatEntryMetadata_MetadataEntry::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional bytes key = 1; + case 1: { + if (tag == 10) { + DO_(::google::protobuf::internal::WireFormatLite::ReadBytes( + input, this->mutable_key())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(18)) goto parse_value; + break; + } + + // optional bytes value = 2; + case 2: { + if (tag == 18) { + parse_value: + DO_(::google::protobuf::internal::WireFormatLite::ReadBytes( + input, this->mutable_value())); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry) + return false; +#undef DO_ +} + +void ThreatEntryMetadata_MetadataEntry::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry) + // optional bytes key = 1; + if (has_key()) { + ::google::protobuf::internal::WireFormatLite::WriteBytesMaybeAliased( + 1, this->key(), output); + } + + // optional bytes value = 2; + if (has_value()) { + ::google::protobuf::internal::WireFormatLite::WriteBytesMaybeAliased( + 2, this->value(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry) +} + +int ThreatEntryMetadata_MetadataEntry::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional bytes key = 1; + if (has_key()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::BytesSize( + this->key()); + } + + // optional bytes value = 2; + if (has_value()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::BytesSize( + this->value()); + } + + } + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void ThreatEntryMetadata_MetadataEntry::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const ThreatEntryMetadata_MetadataEntry*>(&from)); +} + +void ThreatEntryMetadata_MetadataEntry::MergeFrom(const ThreatEntryMetadata_MetadataEntry& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_key()) { + set_key(from.key()); + } + if (from.has_value()) { + set_value(from.value()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void ThreatEntryMetadata_MetadataEntry::CopyFrom(const ThreatEntryMetadata_MetadataEntry& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool ThreatEntryMetadata_MetadataEntry::IsInitialized() const { + + return true; +} + +void ThreatEntryMetadata_MetadataEntry::Swap(ThreatEntryMetadata_MetadataEntry* other) { + if (other != this) { + std::swap(key_, other->key_); + std::swap(value_, other->value_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string ThreatEntryMetadata_MetadataEntry::GetTypeName() const { + return "mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry"; +} + + +// ------------------------------------------------------------------- + +#ifndef _MSC_VER +const int ThreatEntryMetadata::kEntriesFieldNumber; +#endif // !_MSC_VER + +ThreatEntryMetadata::ThreatEntryMetadata() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.ThreatEntryMetadata) +} + +void ThreatEntryMetadata::InitAsDefaultInstance() { +} + +ThreatEntryMetadata::ThreatEntryMetadata(const ThreatEntryMetadata& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.ThreatEntryMetadata) +} + +void ThreatEntryMetadata::SharedCtor() { + _cached_size_ = 0; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +ThreatEntryMetadata::~ThreatEntryMetadata() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.ThreatEntryMetadata) + SharedDtor(); +} + +void ThreatEntryMetadata::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void ThreatEntryMetadata::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ThreatEntryMetadata& ThreatEntryMetadata::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +ThreatEntryMetadata* ThreatEntryMetadata::default_instance_ = NULL; + +ThreatEntryMetadata* ThreatEntryMetadata::New() const { + return new ThreatEntryMetadata; +} + +void ThreatEntryMetadata::Clear() { + entries_.Clear(); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool ThreatEntryMetadata::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.ThreatEntryMetadata) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // repeated .mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry entries = 1; + case 1: { + if (tag == 10) { + parse_entries: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, add_entries())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(10)) goto parse_entries; + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.ThreatEntryMetadata) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.ThreatEntryMetadata) + return false; +#undef DO_ +} + +void ThreatEntryMetadata::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.ThreatEntryMetadata) + // repeated .mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry entries = 1; + for (int i = 0; i < this->entries_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 1, this->entries(i), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.ThreatEntryMetadata) +} + +int ThreatEntryMetadata::ByteSize() const { + int total_size = 0; + + // repeated .mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry entries = 1; + total_size += 1 * this->entries_size(); + for (int i = 0; i < this->entries_size(); i++) { + total_size += + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->entries(i)); + } + + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void ThreatEntryMetadata::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const ThreatEntryMetadata*>(&from)); +} + +void ThreatEntryMetadata::MergeFrom(const ThreatEntryMetadata& from) { + GOOGLE_CHECK_NE(&from, this); + entries_.MergeFrom(from.entries_); + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void ThreatEntryMetadata::CopyFrom(const ThreatEntryMetadata& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool ThreatEntryMetadata::IsInitialized() const { + + return true; +} + +void ThreatEntryMetadata::Swap(ThreatEntryMetadata* other) { + if (other != this) { + entries_.Swap(&other->entries_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string ThreatEntryMetadata::GetTypeName() const { + return "mozilla.safebrowsing.ThreatEntryMetadata"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int ThreatListDescriptor::kThreatTypeFieldNumber; +const int ThreatListDescriptor::kPlatformTypeFieldNumber; +const int ThreatListDescriptor::kThreatEntryTypeFieldNumber; +#endif // !_MSC_VER + +ThreatListDescriptor::ThreatListDescriptor() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.ThreatListDescriptor) +} + +void ThreatListDescriptor::InitAsDefaultInstance() { +} + +ThreatListDescriptor::ThreatListDescriptor(const ThreatListDescriptor& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.ThreatListDescriptor) +} + +void ThreatListDescriptor::SharedCtor() { + _cached_size_ = 0; + threat_type_ = 0; + platform_type_ = 0; + threat_entry_type_ = 0; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +ThreatListDescriptor::~ThreatListDescriptor() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.ThreatListDescriptor) + SharedDtor(); +} + +void ThreatListDescriptor::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void ThreatListDescriptor::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ThreatListDescriptor& ThreatListDescriptor::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +ThreatListDescriptor* ThreatListDescriptor::default_instance_ = NULL; + +ThreatListDescriptor* ThreatListDescriptor::New() const { + return new ThreatListDescriptor; +} + +void ThreatListDescriptor::Clear() { +#define OFFSET_OF_FIELD_(f) (reinterpret_cast<char*>( \ + &reinterpret_cast<ThreatListDescriptor*>(16)->f) - \ + reinterpret_cast<char*>(16)) + +#define ZR_(first, last) do { \ + size_t f = OFFSET_OF_FIELD_(first); \ + size_t n = OFFSET_OF_FIELD_(last) - f + sizeof(last); \ + ::memset(&first, 0, n); \ + } while (0) + + ZR_(threat_type_, threat_entry_type_); + +#undef OFFSET_OF_FIELD_ +#undef ZR_ + + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool ThreatListDescriptor::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.ThreatListDescriptor) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + case 1: { + if (tag == 8) { + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::ThreatType_IsValid(value)) { + set_threat_type(static_cast< ::mozilla::safebrowsing::ThreatType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectTag(16)) goto parse_platform_type; + break; + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + case 2: { + if (tag == 16) { + parse_platform_type: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::PlatformType_IsValid(value)) { + set_platform_type(static_cast< ::mozilla::safebrowsing::PlatformType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectTag(24)) goto parse_threat_entry_type; + break; + } + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 3; + case 3: { + if (tag == 24) { + parse_threat_entry_type: + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + if (::mozilla::safebrowsing::ThreatEntryType_IsValid(value)) { + set_threat_entry_type(static_cast< ::mozilla::safebrowsing::ThreatEntryType >(value)); + } else { + unknown_fields_stream.WriteVarint32(tag); + unknown_fields_stream.WriteVarint32(value); + } + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.ThreatListDescriptor) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.ThreatListDescriptor) + return false; +#undef DO_ +} + +void ThreatListDescriptor::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.ThreatListDescriptor) + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + if (has_threat_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 1, this->threat_type(), output); + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + if (has_platform_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 2, this->platform_type(), output); + } + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 3; + if (has_threat_entry_type()) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 3, this->threat_entry_type(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.ThreatListDescriptor) +} + +int ThreatListDescriptor::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + if (has_threat_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->threat_type()); + } + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + if (has_platform_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->platform_type()); + } + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 3; + if (has_threat_entry_type()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->threat_entry_type()); + } + + } + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void ThreatListDescriptor::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const ThreatListDescriptor*>(&from)); +} + +void ThreatListDescriptor::MergeFrom(const ThreatListDescriptor& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_threat_type()) { + set_threat_type(from.threat_type()); + } + if (from.has_platform_type()) { + set_platform_type(from.platform_type()); + } + if (from.has_threat_entry_type()) { + set_threat_entry_type(from.threat_entry_type()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void ThreatListDescriptor::CopyFrom(const ThreatListDescriptor& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool ThreatListDescriptor::IsInitialized() const { + + return true; +} + +void ThreatListDescriptor::Swap(ThreatListDescriptor* other) { + if (other != this) { + std::swap(threat_type_, other->threat_type_); + std::swap(platform_type_, other->platform_type_); + std::swap(threat_entry_type_, other->threat_entry_type_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string ThreatListDescriptor::GetTypeName() const { + return "mozilla.safebrowsing.ThreatListDescriptor"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int ListThreatListsResponse::kThreatListsFieldNumber; +#endif // !_MSC_VER + +ListThreatListsResponse::ListThreatListsResponse() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.ListThreatListsResponse) +} + +void ListThreatListsResponse::InitAsDefaultInstance() { +} + +ListThreatListsResponse::ListThreatListsResponse(const ListThreatListsResponse& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.ListThreatListsResponse) +} + +void ListThreatListsResponse::SharedCtor() { + _cached_size_ = 0; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +ListThreatListsResponse::~ListThreatListsResponse() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.ListThreatListsResponse) + SharedDtor(); +} + +void ListThreatListsResponse::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void ListThreatListsResponse::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ListThreatListsResponse& ListThreatListsResponse::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +ListThreatListsResponse* ListThreatListsResponse::default_instance_ = NULL; + +ListThreatListsResponse* ListThreatListsResponse::New() const { + return new ListThreatListsResponse; +} + +void ListThreatListsResponse::Clear() { + threat_lists_.Clear(); + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool ListThreatListsResponse::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.ListThreatListsResponse) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // repeated .mozilla.safebrowsing.ThreatListDescriptor threat_lists = 1; + case 1: { + if (tag == 10) { + parse_threat_lists: + DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( + input, add_threat_lists())); + } else { + goto handle_unusual; + } + if (input->ExpectTag(10)) goto parse_threat_lists; + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.ListThreatListsResponse) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.ListThreatListsResponse) + return false; +#undef DO_ +} + +void ListThreatListsResponse::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.ListThreatListsResponse) + // repeated .mozilla.safebrowsing.ThreatListDescriptor threat_lists = 1; + for (int i = 0; i < this->threat_lists_size(); i++) { + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 1, this->threat_lists(i), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.ListThreatListsResponse) +} + +int ListThreatListsResponse::ByteSize() const { + int total_size = 0; + + // repeated .mozilla.safebrowsing.ThreatListDescriptor threat_lists = 1; + total_size += 1 * this->threat_lists_size(); + for (int i = 0; i < this->threat_lists_size(); i++) { + total_size += + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + this->threat_lists(i)); + } + + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void ListThreatListsResponse::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const ListThreatListsResponse*>(&from)); +} + +void ListThreatListsResponse::MergeFrom(const ListThreatListsResponse& from) { + GOOGLE_CHECK_NE(&from, this); + threat_lists_.MergeFrom(from.threat_lists_); + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void ListThreatListsResponse::CopyFrom(const ListThreatListsResponse& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool ListThreatListsResponse::IsInitialized() const { + + return true; +} + +void ListThreatListsResponse::Swap(ListThreatListsResponse* other) { + if (other != this) { + threat_lists_.Swap(&other->threat_lists_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string ListThreatListsResponse::GetTypeName() const { + return "mozilla.safebrowsing.ListThreatListsResponse"; +} + + +// =================================================================== + +#ifndef _MSC_VER +const int Duration::kSecondsFieldNumber; +const int Duration::kNanosFieldNumber; +#endif // !_MSC_VER + +Duration::Duration() + : ::google::protobuf::MessageLite() { + SharedCtor(); + // @@protoc_insertion_point(constructor:mozilla.safebrowsing.Duration) +} + +void Duration::InitAsDefaultInstance() { +} + +Duration::Duration(const Duration& from) + : ::google::protobuf::MessageLite() { + SharedCtor(); + MergeFrom(from); + // @@protoc_insertion_point(copy_constructor:mozilla.safebrowsing.Duration) +} + +void Duration::SharedCtor() { + _cached_size_ = 0; + seconds_ = GOOGLE_LONGLONG(0); + nanos_ = 0; + ::memset(_has_bits_, 0, sizeof(_has_bits_)); +} + +Duration::~Duration() { + // @@protoc_insertion_point(destructor:mozilla.safebrowsing.Duration) + SharedDtor(); +} + +void Duration::SharedDtor() { + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + if (this != &default_instance()) { + #else + if (this != default_instance_) { + #endif + } +} + +void Duration::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const Duration& Duration::default_instance() { +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + protobuf_AddDesc_safebrowsing_2eproto(); +#else + if (default_instance_ == NULL) protobuf_AddDesc_safebrowsing_2eproto(); +#endif + return *default_instance_; +} + +Duration* Duration::default_instance_ = NULL; + +Duration* Duration::New() const { + return new Duration; +} + +void Duration::Clear() { +#define OFFSET_OF_FIELD_(f) (reinterpret_cast<char*>( \ + &reinterpret_cast<Duration*>(16)->f) - \ + reinterpret_cast<char*>(16)) + +#define ZR_(first, last) do { \ + size_t f = OFFSET_OF_FIELD_(first); \ + size_t n = OFFSET_OF_FIELD_(last) - f + sizeof(last); \ + ::memset(&first, 0, n); \ + } while (0) + + ZR_(seconds_, nanos_); + +#undef OFFSET_OF_FIELD_ +#undef ZR_ + + ::memset(_has_bits_, 0, sizeof(_has_bits_)); + mutable_unknown_fields()->clear(); +} + +bool Duration::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::StringOutputStream unknown_fields_string( + mutable_unknown_fields()); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string); + // @@protoc_insertion_point(parse_start:mozilla.safebrowsing.Duration) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoff(127); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // optional int64 seconds = 1; + case 1: { + if (tag == 8) { + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + ::google::protobuf::int64, ::google::protobuf::internal::WireFormatLite::TYPE_INT64>( + input, &seconds_))); + set_has_seconds(); + } else { + goto handle_unusual; + } + if (input->ExpectTag(16)) goto parse_nanos; + break; + } + + // optional int32 nanos = 2; + case 2: { + if (tag == 16) { + parse_nanos: + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( + input, &nanos_))); + set_has_nanos(); + } else { + goto handle_unusual; + } + if (input->ExpectAtEnd()) goto success; + break; + } + + default: { + handle_unusual: + if (tag == 0 || + ::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) == + ::google::protobuf::internal::WireFormatLite::WIRETYPE_END_GROUP) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:mozilla.safebrowsing.Duration) + return true; +failure: + // @@protoc_insertion_point(parse_failure:mozilla.safebrowsing.Duration) + return false; +#undef DO_ +} + +void Duration::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:mozilla.safebrowsing.Duration) + // optional int64 seconds = 1; + if (has_seconds()) { + ::google::protobuf::internal::WireFormatLite::WriteInt64(1, this->seconds(), output); + } + + // optional int32 nanos = 2; + if (has_nanos()) { + ::google::protobuf::internal::WireFormatLite::WriteInt32(2, this->nanos(), output); + } + + output->WriteRaw(unknown_fields().data(), + unknown_fields().size()); + // @@protoc_insertion_point(serialize_end:mozilla.safebrowsing.Duration) +} + +int Duration::ByteSize() const { + int total_size = 0; + + if (_has_bits_[0 / 32] & (0xffu << (0 % 32))) { + // optional int64 seconds = 1; + if (has_seconds()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::Int64Size( + this->seconds()); + } + + // optional int32 nanos = 2; + if (has_nanos()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::Int32Size( + this->nanos()); + } + + } + total_size += unknown_fields().size(); + + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void Duration::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast<const Duration*>(&from)); +} + +void Duration::MergeFrom(const Duration& from) { + GOOGLE_CHECK_NE(&from, this); + if (from._has_bits_[0 / 32] & (0xffu << (0 % 32))) { + if (from.has_seconds()) { + set_seconds(from.seconds()); + } + if (from.has_nanos()) { + set_nanos(from.nanos()); + } + } + mutable_unknown_fields()->append(from.unknown_fields()); +} + +void Duration::CopyFrom(const Duration& from) { + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool Duration::IsInitialized() const { + + return true; +} + +void Duration::Swap(Duration* other) { + if (other != this) { + std::swap(seconds_, other->seconds_); + std::swap(nanos_, other->nanos_); + std::swap(_has_bits_[0], other->_has_bits_[0]); + _unknown_fields_.swap(other->_unknown_fields_); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::std::string Duration::GetTypeName() const { + return "mozilla.safebrowsing.Duration"; +} + + +// @@protoc_insertion_point(namespace_scope) + +} // namespace safebrowsing +} // namespace mozilla + +// @@protoc_insertion_point(global_scope) diff --git a/toolkit/components/url-classifier/protobuf/safebrowsing.pb.h b/toolkit/components/url-classifier/protobuf/safebrowsing.pb.h new file mode 100644 index 000000000..3c1b436df --- /dev/null +++ b/toolkit/components/url-classifier/protobuf/safebrowsing.pb.h @@ -0,0 +1,6283 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: safebrowsing.proto + +#ifndef PROTOBUF_safebrowsing_2eproto__INCLUDED +#define PROTOBUF_safebrowsing_2eproto__INCLUDED + +#include <string> + +#include <google/protobuf/stubs/common.h> + +#if GOOGLE_PROTOBUF_VERSION < 2006000 +#error This file was generated by a newer version of protoc which is +#error incompatible with your Protocol Buffer headers. Please update +#error your headers. +#endif +#if 2006001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION +#error This file was generated by an older version of protoc which is +#error incompatible with your Protocol Buffer headers. Please +#error regenerate this file with a newer version of protoc. +#endif + +#include <google/protobuf/generated_message_util.h> +#include <google/protobuf/message_lite.h> +#include <google/protobuf/repeated_field.h> +#include <google/protobuf/extension_set.h> +// @@protoc_insertion_point(includes) + +namespace mozilla { +namespace safebrowsing { + +// Internal implementation detail -- do not call these. +void protobuf_AddDesc_safebrowsing_2eproto(); +void protobuf_AssignDesc_safebrowsing_2eproto(); +void protobuf_ShutdownFile_safebrowsing_2eproto(); + +class ThreatInfo; +class ThreatMatch; +class FindThreatMatchesRequest; +class FindThreatMatchesResponse; +class FetchThreatListUpdatesRequest; +class FetchThreatListUpdatesRequest_ListUpdateRequest; +class FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints; +class FetchThreatListUpdatesResponse; +class FetchThreatListUpdatesResponse_ListUpdateResponse; +class FindFullHashesRequest; +class FindFullHashesResponse; +class ThreatHit; +class ThreatHit_ThreatSource; +class ClientInfo; +class Checksum; +class ThreatEntry; +class ThreatEntrySet; +class RawIndices; +class RawHashes; +class RiceDeltaEncoding; +class ThreatEntryMetadata; +class ThreatEntryMetadata_MetadataEntry; +class ThreatListDescriptor; +class ListThreatListsResponse; +class Duration; + +enum FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType { + FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_RESPONSE_TYPE_UNSPECIFIED = 0, + FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_PARTIAL_UPDATE = 1, + FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_FULL_UPDATE = 2 +}; +bool FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_IsValid(int value); +const FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_ResponseType_MIN = FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_RESPONSE_TYPE_UNSPECIFIED; +const FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_ResponseType_MAX = FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_FULL_UPDATE; +const int FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_ResponseType_ARRAYSIZE = FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_ResponseType_MAX + 1; + +enum ThreatHit_ThreatSourceType { + ThreatHit_ThreatSourceType_THREAT_SOURCE_TYPE_UNSPECIFIED = 0, + ThreatHit_ThreatSourceType_MATCHING_URL = 1, + ThreatHit_ThreatSourceType_TAB_URL = 2, + ThreatHit_ThreatSourceType_TAB_REDIRECT = 3 +}; +bool ThreatHit_ThreatSourceType_IsValid(int value); +const ThreatHit_ThreatSourceType ThreatHit_ThreatSourceType_ThreatSourceType_MIN = ThreatHit_ThreatSourceType_THREAT_SOURCE_TYPE_UNSPECIFIED; +const ThreatHit_ThreatSourceType ThreatHit_ThreatSourceType_ThreatSourceType_MAX = ThreatHit_ThreatSourceType_TAB_REDIRECT; +const int ThreatHit_ThreatSourceType_ThreatSourceType_ARRAYSIZE = ThreatHit_ThreatSourceType_ThreatSourceType_MAX + 1; + +enum ThreatType { + THREAT_TYPE_UNSPECIFIED = 0, + MALWARE_THREAT = 1, + SOCIAL_ENGINEERING_PUBLIC = 2, + UNWANTED_SOFTWARE = 3, + POTENTIALLY_HARMFUL_APPLICATION = 4, + SOCIAL_ENGINEERING = 5, + API_ABUSE = 6 +}; +bool ThreatType_IsValid(int value); +const ThreatType ThreatType_MIN = THREAT_TYPE_UNSPECIFIED; +const ThreatType ThreatType_MAX = API_ABUSE; +const int ThreatType_ARRAYSIZE = ThreatType_MAX + 1; + +enum PlatformType { + PLATFORM_TYPE_UNSPECIFIED = 0, + WINDOWS_PLATFORM = 1, + LINUX_PLATFORM = 2, + ANDROID_PLATFORM = 3, + OSX_PLATFORM = 4, + IOS_PLATFORM = 5, + ANY_PLATFORM = 6, + ALL_PLATFORMS = 7, + CHROME_PLATFORM = 8 +}; +bool PlatformType_IsValid(int value); +const PlatformType PlatformType_MIN = PLATFORM_TYPE_UNSPECIFIED; +const PlatformType PlatformType_MAX = CHROME_PLATFORM; +const int PlatformType_ARRAYSIZE = PlatformType_MAX + 1; + +enum CompressionType { + COMPRESSION_TYPE_UNSPECIFIED = 0, + RAW = 1, + RICE = 2 +}; +bool CompressionType_IsValid(int value); +const CompressionType CompressionType_MIN = COMPRESSION_TYPE_UNSPECIFIED; +const CompressionType CompressionType_MAX = RICE; +const int CompressionType_ARRAYSIZE = CompressionType_MAX + 1; + +enum ThreatEntryType { + THREAT_ENTRY_TYPE_UNSPECIFIED = 0, + URL = 1, + EXECUTABLE = 2, + IP_RANGE = 3 +}; +bool ThreatEntryType_IsValid(int value); +const ThreatEntryType ThreatEntryType_MIN = THREAT_ENTRY_TYPE_UNSPECIFIED; +const ThreatEntryType ThreatEntryType_MAX = IP_RANGE; +const int ThreatEntryType_ARRAYSIZE = ThreatEntryType_MAX + 1; + +// =================================================================== + +class ThreatInfo : public ::google::protobuf::MessageLite { + public: + ThreatInfo(); + virtual ~ThreatInfo(); + + ThreatInfo(const ThreatInfo& from); + + inline ThreatInfo& operator=(const ThreatInfo& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ThreatInfo& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const ThreatInfo* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(ThreatInfo* other); + + // implements Message ---------------------------------------------- + + ThreatInfo* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const ThreatInfo& from); + void MergeFrom(const ThreatInfo& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // repeated .mozilla.safebrowsing.ThreatType threat_types = 1; + inline int threat_types_size() const; + inline void clear_threat_types(); + static const int kThreatTypesFieldNumber = 1; + inline ::mozilla::safebrowsing::ThreatType threat_types(int index) const; + inline void set_threat_types(int index, ::mozilla::safebrowsing::ThreatType value); + inline void add_threat_types(::mozilla::safebrowsing::ThreatType value); + inline const ::google::protobuf::RepeatedField<int>& threat_types() const; + inline ::google::protobuf::RepeatedField<int>* mutable_threat_types(); + + // repeated .mozilla.safebrowsing.PlatformType platform_types = 2; + inline int platform_types_size() const; + inline void clear_platform_types(); + static const int kPlatformTypesFieldNumber = 2; + inline ::mozilla::safebrowsing::PlatformType platform_types(int index) const; + inline void set_platform_types(int index, ::mozilla::safebrowsing::PlatformType value); + inline void add_platform_types(::mozilla::safebrowsing::PlatformType value); + inline const ::google::protobuf::RepeatedField<int>& platform_types() const; + inline ::google::protobuf::RepeatedField<int>* mutable_platform_types(); + + // repeated .mozilla.safebrowsing.ThreatEntryType threat_entry_types = 4; + inline int threat_entry_types_size() const; + inline void clear_threat_entry_types(); + static const int kThreatEntryTypesFieldNumber = 4; + inline ::mozilla::safebrowsing::ThreatEntryType threat_entry_types(int index) const; + inline void set_threat_entry_types(int index, ::mozilla::safebrowsing::ThreatEntryType value); + inline void add_threat_entry_types(::mozilla::safebrowsing::ThreatEntryType value); + inline const ::google::protobuf::RepeatedField<int>& threat_entry_types() const; + inline ::google::protobuf::RepeatedField<int>* mutable_threat_entry_types(); + + // repeated .mozilla.safebrowsing.ThreatEntry threat_entries = 3; + inline int threat_entries_size() const; + inline void clear_threat_entries(); + static const int kThreatEntriesFieldNumber = 3; + inline const ::mozilla::safebrowsing::ThreatEntry& threat_entries(int index) const; + inline ::mozilla::safebrowsing::ThreatEntry* mutable_threat_entries(int index); + inline ::mozilla::safebrowsing::ThreatEntry* add_threat_entries(); + inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntry >& + threat_entries() const; + inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntry >* + mutable_threat_entries(); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.ThreatInfo) + private: + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::google::protobuf::RepeatedField<int> threat_types_; + ::google::protobuf::RepeatedField<int> platform_types_; + ::google::protobuf::RepeatedField<int> threat_entry_types_; + ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntry > threat_entries_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static ThreatInfo* default_instance_; +}; +// ------------------------------------------------------------------- + +class ThreatMatch : public ::google::protobuf::MessageLite { + public: + ThreatMatch(); + virtual ~ThreatMatch(); + + ThreatMatch(const ThreatMatch& from); + + inline ThreatMatch& operator=(const ThreatMatch& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ThreatMatch& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const ThreatMatch* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(ThreatMatch* other); + + // implements Message ---------------------------------------------- + + ThreatMatch* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const ThreatMatch& from); + void MergeFrom(const ThreatMatch& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + inline bool has_threat_type() const; + inline void clear_threat_type(); + static const int kThreatTypeFieldNumber = 1; + inline ::mozilla::safebrowsing::ThreatType threat_type() const; + inline void set_threat_type(::mozilla::safebrowsing::ThreatType value); + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + inline bool has_platform_type() const; + inline void clear_platform_type(); + static const int kPlatformTypeFieldNumber = 2; + inline ::mozilla::safebrowsing::PlatformType platform_type() const; + inline void set_platform_type(::mozilla::safebrowsing::PlatformType value); + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 6; + inline bool has_threat_entry_type() const; + inline void clear_threat_entry_type(); + static const int kThreatEntryTypeFieldNumber = 6; + inline ::mozilla::safebrowsing::ThreatEntryType threat_entry_type() const; + inline void set_threat_entry_type(::mozilla::safebrowsing::ThreatEntryType value); + + // optional .mozilla.safebrowsing.ThreatEntry threat = 3; + inline bool has_threat() const; + inline void clear_threat(); + static const int kThreatFieldNumber = 3; + inline const ::mozilla::safebrowsing::ThreatEntry& threat() const; + inline ::mozilla::safebrowsing::ThreatEntry* mutable_threat(); + inline ::mozilla::safebrowsing::ThreatEntry* release_threat(); + inline void set_allocated_threat(::mozilla::safebrowsing::ThreatEntry* threat); + + // optional .mozilla.safebrowsing.ThreatEntryMetadata threat_entry_metadata = 4; + inline bool has_threat_entry_metadata() const; + inline void clear_threat_entry_metadata(); + static const int kThreatEntryMetadataFieldNumber = 4; + inline const ::mozilla::safebrowsing::ThreatEntryMetadata& threat_entry_metadata() const; + inline ::mozilla::safebrowsing::ThreatEntryMetadata* mutable_threat_entry_metadata(); + inline ::mozilla::safebrowsing::ThreatEntryMetadata* release_threat_entry_metadata(); + inline void set_allocated_threat_entry_metadata(::mozilla::safebrowsing::ThreatEntryMetadata* threat_entry_metadata); + + // optional .mozilla.safebrowsing.Duration cache_duration = 5; + inline bool has_cache_duration() const; + inline void clear_cache_duration(); + static const int kCacheDurationFieldNumber = 5; + inline const ::mozilla::safebrowsing::Duration& cache_duration() const; + inline ::mozilla::safebrowsing::Duration* mutable_cache_duration(); + inline ::mozilla::safebrowsing::Duration* release_cache_duration(); + inline void set_allocated_cache_duration(::mozilla::safebrowsing::Duration* cache_duration); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.ThreatMatch) + private: + inline void set_has_threat_type(); + inline void clear_has_threat_type(); + inline void set_has_platform_type(); + inline void clear_has_platform_type(); + inline void set_has_threat_entry_type(); + inline void clear_has_threat_entry_type(); + inline void set_has_threat(); + inline void clear_has_threat(); + inline void set_has_threat_entry_metadata(); + inline void clear_has_threat_entry_metadata(); + inline void set_has_cache_duration(); + inline void clear_has_cache_duration(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + int threat_type_; + int platform_type_; + ::mozilla::safebrowsing::ThreatEntry* threat_; + ::mozilla::safebrowsing::ThreatEntryMetadata* threat_entry_metadata_; + ::mozilla::safebrowsing::Duration* cache_duration_; + int threat_entry_type_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static ThreatMatch* default_instance_; +}; +// ------------------------------------------------------------------- + +class FindThreatMatchesRequest : public ::google::protobuf::MessageLite { + public: + FindThreatMatchesRequest(); + virtual ~FindThreatMatchesRequest(); + + FindThreatMatchesRequest(const FindThreatMatchesRequest& from); + + inline FindThreatMatchesRequest& operator=(const FindThreatMatchesRequest& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const FindThreatMatchesRequest& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const FindThreatMatchesRequest* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(FindThreatMatchesRequest* other); + + // implements Message ---------------------------------------------- + + FindThreatMatchesRequest* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const FindThreatMatchesRequest& from); + void MergeFrom(const FindThreatMatchesRequest& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional .mozilla.safebrowsing.ClientInfo client = 1; + inline bool has_client() const; + inline void clear_client(); + static const int kClientFieldNumber = 1; + inline const ::mozilla::safebrowsing::ClientInfo& client() const; + inline ::mozilla::safebrowsing::ClientInfo* mutable_client(); + inline ::mozilla::safebrowsing::ClientInfo* release_client(); + inline void set_allocated_client(::mozilla::safebrowsing::ClientInfo* client); + + // optional .mozilla.safebrowsing.ThreatInfo threat_info = 2; + inline bool has_threat_info() const; + inline void clear_threat_info(); + static const int kThreatInfoFieldNumber = 2; + inline const ::mozilla::safebrowsing::ThreatInfo& threat_info() const; + inline ::mozilla::safebrowsing::ThreatInfo* mutable_threat_info(); + inline ::mozilla::safebrowsing::ThreatInfo* release_threat_info(); + inline void set_allocated_threat_info(::mozilla::safebrowsing::ThreatInfo* threat_info); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.FindThreatMatchesRequest) + private: + inline void set_has_client(); + inline void clear_has_client(); + inline void set_has_threat_info(); + inline void clear_has_threat_info(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::mozilla::safebrowsing::ClientInfo* client_; + ::mozilla::safebrowsing::ThreatInfo* threat_info_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static FindThreatMatchesRequest* default_instance_; +}; +// ------------------------------------------------------------------- + +class FindThreatMatchesResponse : public ::google::protobuf::MessageLite { + public: + FindThreatMatchesResponse(); + virtual ~FindThreatMatchesResponse(); + + FindThreatMatchesResponse(const FindThreatMatchesResponse& from); + + inline FindThreatMatchesResponse& operator=(const FindThreatMatchesResponse& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const FindThreatMatchesResponse& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const FindThreatMatchesResponse* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(FindThreatMatchesResponse* other); + + // implements Message ---------------------------------------------- + + FindThreatMatchesResponse* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const FindThreatMatchesResponse& from); + void MergeFrom(const FindThreatMatchesResponse& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // repeated .mozilla.safebrowsing.ThreatMatch matches = 1; + inline int matches_size() const; + inline void clear_matches(); + static const int kMatchesFieldNumber = 1; + inline const ::mozilla::safebrowsing::ThreatMatch& matches(int index) const; + inline ::mozilla::safebrowsing::ThreatMatch* mutable_matches(int index); + inline ::mozilla::safebrowsing::ThreatMatch* add_matches(); + inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatMatch >& + matches() const; + inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatMatch >* + mutable_matches(); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.FindThreatMatchesResponse) + private: + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatMatch > matches_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static FindThreatMatchesResponse* default_instance_; +}; +// ------------------------------------------------------------------- + +class FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints : public ::google::protobuf::MessageLite { + public: + FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints(); + virtual ~FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints(); + + FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints(const FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints& from); + + inline FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints& operator=(const FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* other); + + // implements Message ---------------------------------------------- + + FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints& from); + void MergeFrom(const FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional int32 max_update_entries = 1; + inline bool has_max_update_entries() const; + inline void clear_max_update_entries(); + static const int kMaxUpdateEntriesFieldNumber = 1; + inline ::google::protobuf::int32 max_update_entries() const; + inline void set_max_update_entries(::google::protobuf::int32 value); + + // optional int32 max_database_entries = 2; + inline bool has_max_database_entries() const; + inline void clear_max_database_entries(); + static const int kMaxDatabaseEntriesFieldNumber = 2; + inline ::google::protobuf::int32 max_database_entries() const; + inline void set_max_database_entries(::google::protobuf::int32 value); + + // optional string region = 3; + inline bool has_region() const; + inline void clear_region(); + static const int kRegionFieldNumber = 3; + inline const ::std::string& region() const; + inline void set_region(const ::std::string& value); + inline void set_region(const char* value); + inline void set_region(const char* value, size_t size); + inline ::std::string* mutable_region(); + inline ::std::string* release_region(); + inline void set_allocated_region(::std::string* region); + + // repeated .mozilla.safebrowsing.CompressionType supported_compressions = 4; + inline int supported_compressions_size() const; + inline void clear_supported_compressions(); + static const int kSupportedCompressionsFieldNumber = 4; + inline ::mozilla::safebrowsing::CompressionType supported_compressions(int index) const; + inline void set_supported_compressions(int index, ::mozilla::safebrowsing::CompressionType value); + inline void add_supported_compressions(::mozilla::safebrowsing::CompressionType value); + inline const ::google::protobuf::RepeatedField<int>& supported_compressions() const; + inline ::google::protobuf::RepeatedField<int>* mutable_supported_compressions(); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints) + private: + inline void set_has_max_update_entries(); + inline void clear_has_max_update_entries(); + inline void set_has_max_database_entries(); + inline void clear_has_max_database_entries(); + inline void set_has_region(); + inline void clear_has_region(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::google::protobuf::int32 max_update_entries_; + ::google::protobuf::int32 max_database_entries_; + ::std::string* region_; + ::google::protobuf::RepeatedField<int> supported_compressions_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* default_instance_; +}; +// ------------------------------------------------------------------- + +class FetchThreatListUpdatesRequest_ListUpdateRequest : public ::google::protobuf::MessageLite { + public: + FetchThreatListUpdatesRequest_ListUpdateRequest(); + virtual ~FetchThreatListUpdatesRequest_ListUpdateRequest(); + + FetchThreatListUpdatesRequest_ListUpdateRequest(const FetchThreatListUpdatesRequest_ListUpdateRequest& from); + + inline FetchThreatListUpdatesRequest_ListUpdateRequest& operator=(const FetchThreatListUpdatesRequest_ListUpdateRequest& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const FetchThreatListUpdatesRequest_ListUpdateRequest& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const FetchThreatListUpdatesRequest_ListUpdateRequest* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(FetchThreatListUpdatesRequest_ListUpdateRequest* other); + + // implements Message ---------------------------------------------- + + FetchThreatListUpdatesRequest_ListUpdateRequest* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const FetchThreatListUpdatesRequest_ListUpdateRequest& from); + void MergeFrom(const FetchThreatListUpdatesRequest_ListUpdateRequest& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + typedef FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints Constraints; + + // accessors ------------------------------------------------------- + + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + inline bool has_threat_type() const; + inline void clear_threat_type(); + static const int kThreatTypeFieldNumber = 1; + inline ::mozilla::safebrowsing::ThreatType threat_type() const; + inline void set_threat_type(::mozilla::safebrowsing::ThreatType value); + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + inline bool has_platform_type() const; + inline void clear_platform_type(); + static const int kPlatformTypeFieldNumber = 2; + inline ::mozilla::safebrowsing::PlatformType platform_type() const; + inline void set_platform_type(::mozilla::safebrowsing::PlatformType value); + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 5; + inline bool has_threat_entry_type() const; + inline void clear_threat_entry_type(); + static const int kThreatEntryTypeFieldNumber = 5; + inline ::mozilla::safebrowsing::ThreatEntryType threat_entry_type() const; + inline void set_threat_entry_type(::mozilla::safebrowsing::ThreatEntryType value); + + // optional bytes state = 3; + inline bool has_state() const; + inline void clear_state(); + static const int kStateFieldNumber = 3; + inline const ::std::string& state() const; + inline void set_state(const ::std::string& value); + inline void set_state(const char* value); + inline void set_state(const void* value, size_t size); + inline ::std::string* mutable_state(); + inline ::std::string* release_state(); + inline void set_allocated_state(::std::string* state); + + // optional .mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints constraints = 4; + inline bool has_constraints() const; + inline void clear_constraints(); + static const int kConstraintsFieldNumber = 4; + inline const ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints& constraints() const; + inline ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* mutable_constraints(); + inline ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* release_constraints(); + inline void set_allocated_constraints(::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* constraints); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest) + private: + inline void set_has_threat_type(); + inline void clear_has_threat_type(); + inline void set_has_platform_type(); + inline void clear_has_platform_type(); + inline void set_has_threat_entry_type(); + inline void clear_has_threat_entry_type(); + inline void set_has_state(); + inline void clear_has_state(); + inline void set_has_constraints(); + inline void clear_has_constraints(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + int threat_type_; + int platform_type_; + ::std::string* state_; + ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* constraints_; + int threat_entry_type_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static FetchThreatListUpdatesRequest_ListUpdateRequest* default_instance_; +}; +// ------------------------------------------------------------------- + +class FetchThreatListUpdatesRequest : public ::google::protobuf::MessageLite { + public: + FetchThreatListUpdatesRequest(); + virtual ~FetchThreatListUpdatesRequest(); + + FetchThreatListUpdatesRequest(const FetchThreatListUpdatesRequest& from); + + inline FetchThreatListUpdatesRequest& operator=(const FetchThreatListUpdatesRequest& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const FetchThreatListUpdatesRequest& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const FetchThreatListUpdatesRequest* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(FetchThreatListUpdatesRequest* other); + + // implements Message ---------------------------------------------- + + FetchThreatListUpdatesRequest* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const FetchThreatListUpdatesRequest& from); + void MergeFrom(const FetchThreatListUpdatesRequest& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + typedef FetchThreatListUpdatesRequest_ListUpdateRequest ListUpdateRequest; + + // accessors ------------------------------------------------------- + + // optional .mozilla.safebrowsing.ClientInfo client = 1; + inline bool has_client() const; + inline void clear_client(); + static const int kClientFieldNumber = 1; + inline const ::mozilla::safebrowsing::ClientInfo& client() const; + inline ::mozilla::safebrowsing::ClientInfo* mutable_client(); + inline ::mozilla::safebrowsing::ClientInfo* release_client(); + inline void set_allocated_client(::mozilla::safebrowsing::ClientInfo* client); + + // repeated .mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest list_update_requests = 3; + inline int list_update_requests_size() const; + inline void clear_list_update_requests(); + static const int kListUpdateRequestsFieldNumber = 3; + inline const ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest& list_update_requests(int index) const; + inline ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest* mutable_list_update_requests(int index); + inline ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest* add_list_update_requests(); + inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest >& + list_update_requests() const; + inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest >* + mutable_list_update_requests(); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.FetchThreatListUpdatesRequest) + private: + inline void set_has_client(); + inline void clear_has_client(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::mozilla::safebrowsing::ClientInfo* client_; + ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest > list_update_requests_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static FetchThreatListUpdatesRequest* default_instance_; +}; +// ------------------------------------------------------------------- + +class FetchThreatListUpdatesResponse_ListUpdateResponse : public ::google::protobuf::MessageLite { + public: + FetchThreatListUpdatesResponse_ListUpdateResponse(); + virtual ~FetchThreatListUpdatesResponse_ListUpdateResponse(); + + FetchThreatListUpdatesResponse_ListUpdateResponse(const FetchThreatListUpdatesResponse_ListUpdateResponse& from); + + inline FetchThreatListUpdatesResponse_ListUpdateResponse& operator=(const FetchThreatListUpdatesResponse_ListUpdateResponse& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const FetchThreatListUpdatesResponse_ListUpdateResponse& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const FetchThreatListUpdatesResponse_ListUpdateResponse* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(FetchThreatListUpdatesResponse_ListUpdateResponse* other); + + // implements Message ---------------------------------------------- + + FetchThreatListUpdatesResponse_ListUpdateResponse* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const FetchThreatListUpdatesResponse_ListUpdateResponse& from); + void MergeFrom(const FetchThreatListUpdatesResponse_ListUpdateResponse& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + typedef FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType ResponseType; + static const ResponseType RESPONSE_TYPE_UNSPECIFIED = FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_RESPONSE_TYPE_UNSPECIFIED; + static const ResponseType PARTIAL_UPDATE = FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_PARTIAL_UPDATE; + static const ResponseType FULL_UPDATE = FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_FULL_UPDATE; + static inline bool ResponseType_IsValid(int value) { + return FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_IsValid(value); + } + static const ResponseType ResponseType_MIN = + FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_ResponseType_MIN; + static const ResponseType ResponseType_MAX = + FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_ResponseType_MAX; + static const int ResponseType_ARRAYSIZE = + FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_ResponseType_ARRAYSIZE; + + // accessors ------------------------------------------------------- + + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + inline bool has_threat_type() const; + inline void clear_threat_type(); + static const int kThreatTypeFieldNumber = 1; + inline ::mozilla::safebrowsing::ThreatType threat_type() const; + inline void set_threat_type(::mozilla::safebrowsing::ThreatType value); + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 2; + inline bool has_threat_entry_type() const; + inline void clear_threat_entry_type(); + static const int kThreatEntryTypeFieldNumber = 2; + inline ::mozilla::safebrowsing::ThreatEntryType threat_entry_type() const; + inline void set_threat_entry_type(::mozilla::safebrowsing::ThreatEntryType value); + + // optional .mozilla.safebrowsing.PlatformType platform_type = 3; + inline bool has_platform_type() const; + inline void clear_platform_type(); + static const int kPlatformTypeFieldNumber = 3; + inline ::mozilla::safebrowsing::PlatformType platform_type() const; + inline void set_platform_type(::mozilla::safebrowsing::PlatformType value); + + // optional .mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.ResponseType response_type = 4; + inline bool has_response_type() const; + inline void clear_response_type(); + static const int kResponseTypeFieldNumber = 4; + inline ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType response_type() const; + inline void set_response_type(::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType value); + + // repeated .mozilla.safebrowsing.ThreatEntrySet additions = 5; + inline int additions_size() const; + inline void clear_additions(); + static const int kAdditionsFieldNumber = 5; + inline const ::mozilla::safebrowsing::ThreatEntrySet& additions(int index) const; + inline ::mozilla::safebrowsing::ThreatEntrySet* mutable_additions(int index); + inline ::mozilla::safebrowsing::ThreatEntrySet* add_additions(); + inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntrySet >& + additions() const; + inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntrySet >* + mutable_additions(); + + // repeated .mozilla.safebrowsing.ThreatEntrySet removals = 6; + inline int removals_size() const; + inline void clear_removals(); + static const int kRemovalsFieldNumber = 6; + inline const ::mozilla::safebrowsing::ThreatEntrySet& removals(int index) const; + inline ::mozilla::safebrowsing::ThreatEntrySet* mutable_removals(int index); + inline ::mozilla::safebrowsing::ThreatEntrySet* add_removals(); + inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntrySet >& + removals() const; + inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntrySet >* + mutable_removals(); + + // optional bytes new_client_state = 7; + inline bool has_new_client_state() const; + inline void clear_new_client_state(); + static const int kNewClientStateFieldNumber = 7; + inline const ::std::string& new_client_state() const; + inline void set_new_client_state(const ::std::string& value); + inline void set_new_client_state(const char* value); + inline void set_new_client_state(const void* value, size_t size); + inline ::std::string* mutable_new_client_state(); + inline ::std::string* release_new_client_state(); + inline void set_allocated_new_client_state(::std::string* new_client_state); + + // optional .mozilla.safebrowsing.Checksum checksum = 8; + inline bool has_checksum() const; + inline void clear_checksum(); + static const int kChecksumFieldNumber = 8; + inline const ::mozilla::safebrowsing::Checksum& checksum() const; + inline ::mozilla::safebrowsing::Checksum* mutable_checksum(); + inline ::mozilla::safebrowsing::Checksum* release_checksum(); + inline void set_allocated_checksum(::mozilla::safebrowsing::Checksum* checksum); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse) + private: + inline void set_has_threat_type(); + inline void clear_has_threat_type(); + inline void set_has_threat_entry_type(); + inline void clear_has_threat_entry_type(); + inline void set_has_platform_type(); + inline void clear_has_platform_type(); + inline void set_has_response_type(); + inline void clear_has_response_type(); + inline void set_has_new_client_state(); + inline void clear_has_new_client_state(); + inline void set_has_checksum(); + inline void clear_has_checksum(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + int threat_type_; + int threat_entry_type_; + int platform_type_; + int response_type_; + ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntrySet > additions_; + ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntrySet > removals_; + ::std::string* new_client_state_; + ::mozilla::safebrowsing::Checksum* checksum_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static FetchThreatListUpdatesResponse_ListUpdateResponse* default_instance_; +}; +// ------------------------------------------------------------------- + +class FetchThreatListUpdatesResponse : public ::google::protobuf::MessageLite { + public: + FetchThreatListUpdatesResponse(); + virtual ~FetchThreatListUpdatesResponse(); + + FetchThreatListUpdatesResponse(const FetchThreatListUpdatesResponse& from); + + inline FetchThreatListUpdatesResponse& operator=(const FetchThreatListUpdatesResponse& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const FetchThreatListUpdatesResponse& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const FetchThreatListUpdatesResponse* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(FetchThreatListUpdatesResponse* other); + + // implements Message ---------------------------------------------- + + FetchThreatListUpdatesResponse* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const FetchThreatListUpdatesResponse& from); + void MergeFrom(const FetchThreatListUpdatesResponse& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + typedef FetchThreatListUpdatesResponse_ListUpdateResponse ListUpdateResponse; + + // accessors ------------------------------------------------------- + + // repeated .mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse list_update_responses = 1; + inline int list_update_responses_size() const; + inline void clear_list_update_responses(); + static const int kListUpdateResponsesFieldNumber = 1; + inline const ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse& list_update_responses(int index) const; + inline ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse* mutable_list_update_responses(int index); + inline ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse* add_list_update_responses(); + inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse >& + list_update_responses() const; + inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse >* + mutable_list_update_responses(); + + // optional .mozilla.safebrowsing.Duration minimum_wait_duration = 2; + inline bool has_minimum_wait_duration() const; + inline void clear_minimum_wait_duration(); + static const int kMinimumWaitDurationFieldNumber = 2; + inline const ::mozilla::safebrowsing::Duration& minimum_wait_duration() const; + inline ::mozilla::safebrowsing::Duration* mutable_minimum_wait_duration(); + inline ::mozilla::safebrowsing::Duration* release_minimum_wait_duration(); + inline void set_allocated_minimum_wait_duration(::mozilla::safebrowsing::Duration* minimum_wait_duration); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.FetchThreatListUpdatesResponse) + private: + inline void set_has_minimum_wait_duration(); + inline void clear_has_minimum_wait_duration(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse > list_update_responses_; + ::mozilla::safebrowsing::Duration* minimum_wait_duration_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static FetchThreatListUpdatesResponse* default_instance_; +}; +// ------------------------------------------------------------------- + +class FindFullHashesRequest : public ::google::protobuf::MessageLite { + public: + FindFullHashesRequest(); + virtual ~FindFullHashesRequest(); + + FindFullHashesRequest(const FindFullHashesRequest& from); + + inline FindFullHashesRequest& operator=(const FindFullHashesRequest& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const FindFullHashesRequest& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const FindFullHashesRequest* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(FindFullHashesRequest* other); + + // implements Message ---------------------------------------------- + + FindFullHashesRequest* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const FindFullHashesRequest& from); + void MergeFrom(const FindFullHashesRequest& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional .mozilla.safebrowsing.ClientInfo client = 1; + inline bool has_client() const; + inline void clear_client(); + static const int kClientFieldNumber = 1; + inline const ::mozilla::safebrowsing::ClientInfo& client() const; + inline ::mozilla::safebrowsing::ClientInfo* mutable_client(); + inline ::mozilla::safebrowsing::ClientInfo* release_client(); + inline void set_allocated_client(::mozilla::safebrowsing::ClientInfo* client); + + // repeated bytes client_states = 2; + inline int client_states_size() const; + inline void clear_client_states(); + static const int kClientStatesFieldNumber = 2; + inline const ::std::string& client_states(int index) const; + inline ::std::string* mutable_client_states(int index); + inline void set_client_states(int index, const ::std::string& value); + inline void set_client_states(int index, const char* value); + inline void set_client_states(int index, const void* value, size_t size); + inline ::std::string* add_client_states(); + inline void add_client_states(const ::std::string& value); + inline void add_client_states(const char* value); + inline void add_client_states(const void* value, size_t size); + inline const ::google::protobuf::RepeatedPtrField< ::std::string>& client_states() const; + inline ::google::protobuf::RepeatedPtrField< ::std::string>* mutable_client_states(); + + // optional .mozilla.safebrowsing.ThreatInfo threat_info = 3; + inline bool has_threat_info() const; + inline void clear_threat_info(); + static const int kThreatInfoFieldNumber = 3; + inline const ::mozilla::safebrowsing::ThreatInfo& threat_info() const; + inline ::mozilla::safebrowsing::ThreatInfo* mutable_threat_info(); + inline ::mozilla::safebrowsing::ThreatInfo* release_threat_info(); + inline void set_allocated_threat_info(::mozilla::safebrowsing::ThreatInfo* threat_info); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.FindFullHashesRequest) + private: + inline void set_has_client(); + inline void clear_has_client(); + inline void set_has_threat_info(); + inline void clear_has_threat_info(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::mozilla::safebrowsing::ClientInfo* client_; + ::google::protobuf::RepeatedPtrField< ::std::string> client_states_; + ::mozilla::safebrowsing::ThreatInfo* threat_info_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static FindFullHashesRequest* default_instance_; +}; +// ------------------------------------------------------------------- + +class FindFullHashesResponse : public ::google::protobuf::MessageLite { + public: + FindFullHashesResponse(); + virtual ~FindFullHashesResponse(); + + FindFullHashesResponse(const FindFullHashesResponse& from); + + inline FindFullHashesResponse& operator=(const FindFullHashesResponse& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const FindFullHashesResponse& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const FindFullHashesResponse* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(FindFullHashesResponse* other); + + // implements Message ---------------------------------------------- + + FindFullHashesResponse* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const FindFullHashesResponse& from); + void MergeFrom(const FindFullHashesResponse& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // repeated .mozilla.safebrowsing.ThreatMatch matches = 1; + inline int matches_size() const; + inline void clear_matches(); + static const int kMatchesFieldNumber = 1; + inline const ::mozilla::safebrowsing::ThreatMatch& matches(int index) const; + inline ::mozilla::safebrowsing::ThreatMatch* mutable_matches(int index); + inline ::mozilla::safebrowsing::ThreatMatch* add_matches(); + inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatMatch >& + matches() const; + inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatMatch >* + mutable_matches(); + + // optional .mozilla.safebrowsing.Duration minimum_wait_duration = 2; + inline bool has_minimum_wait_duration() const; + inline void clear_minimum_wait_duration(); + static const int kMinimumWaitDurationFieldNumber = 2; + inline const ::mozilla::safebrowsing::Duration& minimum_wait_duration() const; + inline ::mozilla::safebrowsing::Duration* mutable_minimum_wait_duration(); + inline ::mozilla::safebrowsing::Duration* release_minimum_wait_duration(); + inline void set_allocated_minimum_wait_duration(::mozilla::safebrowsing::Duration* minimum_wait_duration); + + // optional .mozilla.safebrowsing.Duration negative_cache_duration = 3; + inline bool has_negative_cache_duration() const; + inline void clear_negative_cache_duration(); + static const int kNegativeCacheDurationFieldNumber = 3; + inline const ::mozilla::safebrowsing::Duration& negative_cache_duration() const; + inline ::mozilla::safebrowsing::Duration* mutable_negative_cache_duration(); + inline ::mozilla::safebrowsing::Duration* release_negative_cache_duration(); + inline void set_allocated_negative_cache_duration(::mozilla::safebrowsing::Duration* negative_cache_duration); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.FindFullHashesResponse) + private: + inline void set_has_minimum_wait_duration(); + inline void clear_has_minimum_wait_duration(); + inline void set_has_negative_cache_duration(); + inline void clear_has_negative_cache_duration(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatMatch > matches_; + ::mozilla::safebrowsing::Duration* minimum_wait_duration_; + ::mozilla::safebrowsing::Duration* negative_cache_duration_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static FindFullHashesResponse* default_instance_; +}; +// ------------------------------------------------------------------- + +class ThreatHit_ThreatSource : public ::google::protobuf::MessageLite { + public: + ThreatHit_ThreatSource(); + virtual ~ThreatHit_ThreatSource(); + + ThreatHit_ThreatSource(const ThreatHit_ThreatSource& from); + + inline ThreatHit_ThreatSource& operator=(const ThreatHit_ThreatSource& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ThreatHit_ThreatSource& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const ThreatHit_ThreatSource* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(ThreatHit_ThreatSource* other); + + // implements Message ---------------------------------------------- + + ThreatHit_ThreatSource* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const ThreatHit_ThreatSource& from); + void MergeFrom(const ThreatHit_ThreatSource& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional string url = 1; + inline bool has_url() const; + inline void clear_url(); + static const int kUrlFieldNumber = 1; + inline const ::std::string& url() const; + inline void set_url(const ::std::string& value); + inline void set_url(const char* value); + inline void set_url(const char* value, size_t size); + inline ::std::string* mutable_url(); + inline ::std::string* release_url(); + inline void set_allocated_url(::std::string* url); + + // optional .mozilla.safebrowsing.ThreatHit.ThreatSourceType type = 2; + inline bool has_type() const; + inline void clear_type(); + static const int kTypeFieldNumber = 2; + inline ::mozilla::safebrowsing::ThreatHit_ThreatSourceType type() const; + inline void set_type(::mozilla::safebrowsing::ThreatHit_ThreatSourceType value); + + // optional string remote_ip = 3; + inline bool has_remote_ip() const; + inline void clear_remote_ip(); + static const int kRemoteIpFieldNumber = 3; + inline const ::std::string& remote_ip() const; + inline void set_remote_ip(const ::std::string& value); + inline void set_remote_ip(const char* value); + inline void set_remote_ip(const char* value, size_t size); + inline ::std::string* mutable_remote_ip(); + inline ::std::string* release_remote_ip(); + inline void set_allocated_remote_ip(::std::string* remote_ip); + + // optional string referrer = 4; + inline bool has_referrer() const; + inline void clear_referrer(); + static const int kReferrerFieldNumber = 4; + inline const ::std::string& referrer() const; + inline void set_referrer(const ::std::string& value); + inline void set_referrer(const char* value); + inline void set_referrer(const char* value, size_t size); + inline ::std::string* mutable_referrer(); + inline ::std::string* release_referrer(); + inline void set_allocated_referrer(::std::string* referrer); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.ThreatHit.ThreatSource) + private: + inline void set_has_url(); + inline void clear_has_url(); + inline void set_has_type(); + inline void clear_has_type(); + inline void set_has_remote_ip(); + inline void clear_has_remote_ip(); + inline void set_has_referrer(); + inline void clear_has_referrer(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::std::string* url_; + ::std::string* remote_ip_; + ::std::string* referrer_; + int type_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static ThreatHit_ThreatSource* default_instance_; +}; +// ------------------------------------------------------------------- + +class ThreatHit : public ::google::protobuf::MessageLite { + public: + ThreatHit(); + virtual ~ThreatHit(); + + ThreatHit(const ThreatHit& from); + + inline ThreatHit& operator=(const ThreatHit& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ThreatHit& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const ThreatHit* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(ThreatHit* other); + + // implements Message ---------------------------------------------- + + ThreatHit* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const ThreatHit& from); + void MergeFrom(const ThreatHit& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + typedef ThreatHit_ThreatSource ThreatSource; + + typedef ThreatHit_ThreatSourceType ThreatSourceType; + static const ThreatSourceType THREAT_SOURCE_TYPE_UNSPECIFIED = ThreatHit_ThreatSourceType_THREAT_SOURCE_TYPE_UNSPECIFIED; + static const ThreatSourceType MATCHING_URL = ThreatHit_ThreatSourceType_MATCHING_URL; + static const ThreatSourceType TAB_URL = ThreatHit_ThreatSourceType_TAB_URL; + static const ThreatSourceType TAB_REDIRECT = ThreatHit_ThreatSourceType_TAB_REDIRECT; + static inline bool ThreatSourceType_IsValid(int value) { + return ThreatHit_ThreatSourceType_IsValid(value); + } + static const ThreatSourceType ThreatSourceType_MIN = + ThreatHit_ThreatSourceType_ThreatSourceType_MIN; + static const ThreatSourceType ThreatSourceType_MAX = + ThreatHit_ThreatSourceType_ThreatSourceType_MAX; + static const int ThreatSourceType_ARRAYSIZE = + ThreatHit_ThreatSourceType_ThreatSourceType_ARRAYSIZE; + + // accessors ------------------------------------------------------- + + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + inline bool has_threat_type() const; + inline void clear_threat_type(); + static const int kThreatTypeFieldNumber = 1; + inline ::mozilla::safebrowsing::ThreatType threat_type() const; + inline void set_threat_type(::mozilla::safebrowsing::ThreatType value); + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + inline bool has_platform_type() const; + inline void clear_platform_type(); + static const int kPlatformTypeFieldNumber = 2; + inline ::mozilla::safebrowsing::PlatformType platform_type() const; + inline void set_platform_type(::mozilla::safebrowsing::PlatformType value); + + // optional .mozilla.safebrowsing.ThreatEntry entry = 3; + inline bool has_entry() const; + inline void clear_entry(); + static const int kEntryFieldNumber = 3; + inline const ::mozilla::safebrowsing::ThreatEntry& entry() const; + inline ::mozilla::safebrowsing::ThreatEntry* mutable_entry(); + inline ::mozilla::safebrowsing::ThreatEntry* release_entry(); + inline void set_allocated_entry(::mozilla::safebrowsing::ThreatEntry* entry); + + // repeated .mozilla.safebrowsing.ThreatHit.ThreatSource resources = 4; + inline int resources_size() const; + inline void clear_resources(); + static const int kResourcesFieldNumber = 4; + inline const ::mozilla::safebrowsing::ThreatHit_ThreatSource& resources(int index) const; + inline ::mozilla::safebrowsing::ThreatHit_ThreatSource* mutable_resources(int index); + inline ::mozilla::safebrowsing::ThreatHit_ThreatSource* add_resources(); + inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatHit_ThreatSource >& + resources() const; + inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatHit_ThreatSource >* + mutable_resources(); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.ThreatHit) + private: + inline void set_has_threat_type(); + inline void clear_has_threat_type(); + inline void set_has_platform_type(); + inline void clear_has_platform_type(); + inline void set_has_entry(); + inline void clear_has_entry(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + int threat_type_; + int platform_type_; + ::mozilla::safebrowsing::ThreatEntry* entry_; + ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatHit_ThreatSource > resources_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static ThreatHit* default_instance_; +}; +// ------------------------------------------------------------------- + +class ClientInfo : public ::google::protobuf::MessageLite { + public: + ClientInfo(); + virtual ~ClientInfo(); + + ClientInfo(const ClientInfo& from); + + inline ClientInfo& operator=(const ClientInfo& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ClientInfo& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const ClientInfo* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(ClientInfo* other); + + // implements Message ---------------------------------------------- + + ClientInfo* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const ClientInfo& from); + void MergeFrom(const ClientInfo& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional string client_id = 1; + inline bool has_client_id() const; + inline void clear_client_id(); + static const int kClientIdFieldNumber = 1; + inline const ::std::string& client_id() const; + inline void set_client_id(const ::std::string& value); + inline void set_client_id(const char* value); + inline void set_client_id(const char* value, size_t size); + inline ::std::string* mutable_client_id(); + inline ::std::string* release_client_id(); + inline void set_allocated_client_id(::std::string* client_id); + + // optional string client_version = 2; + inline bool has_client_version() const; + inline void clear_client_version(); + static const int kClientVersionFieldNumber = 2; + inline const ::std::string& client_version() const; + inline void set_client_version(const ::std::string& value); + inline void set_client_version(const char* value); + inline void set_client_version(const char* value, size_t size); + inline ::std::string* mutable_client_version(); + inline ::std::string* release_client_version(); + inline void set_allocated_client_version(::std::string* client_version); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.ClientInfo) + private: + inline void set_has_client_id(); + inline void clear_has_client_id(); + inline void set_has_client_version(); + inline void clear_has_client_version(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::std::string* client_id_; + ::std::string* client_version_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static ClientInfo* default_instance_; +}; +// ------------------------------------------------------------------- + +class Checksum : public ::google::protobuf::MessageLite { + public: + Checksum(); + virtual ~Checksum(); + + Checksum(const Checksum& from); + + inline Checksum& operator=(const Checksum& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const Checksum& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const Checksum* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(Checksum* other); + + // implements Message ---------------------------------------------- + + Checksum* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const Checksum& from); + void MergeFrom(const Checksum& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional bytes sha256 = 1; + inline bool has_sha256() const; + inline void clear_sha256(); + static const int kSha256FieldNumber = 1; + inline const ::std::string& sha256() const; + inline void set_sha256(const ::std::string& value); + inline void set_sha256(const char* value); + inline void set_sha256(const void* value, size_t size); + inline ::std::string* mutable_sha256(); + inline ::std::string* release_sha256(); + inline void set_allocated_sha256(::std::string* sha256); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.Checksum) + private: + inline void set_has_sha256(); + inline void clear_has_sha256(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::std::string* sha256_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static Checksum* default_instance_; +}; +// ------------------------------------------------------------------- + +class ThreatEntry : public ::google::protobuf::MessageLite { + public: + ThreatEntry(); + virtual ~ThreatEntry(); + + ThreatEntry(const ThreatEntry& from); + + inline ThreatEntry& operator=(const ThreatEntry& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ThreatEntry& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const ThreatEntry* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(ThreatEntry* other); + + // implements Message ---------------------------------------------- + + ThreatEntry* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const ThreatEntry& from); + void MergeFrom(const ThreatEntry& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional bytes hash = 1; + inline bool has_hash() const; + inline void clear_hash(); + static const int kHashFieldNumber = 1; + inline const ::std::string& hash() const; + inline void set_hash(const ::std::string& value); + inline void set_hash(const char* value); + inline void set_hash(const void* value, size_t size); + inline ::std::string* mutable_hash(); + inline ::std::string* release_hash(); + inline void set_allocated_hash(::std::string* hash); + + // optional string url = 2; + inline bool has_url() const; + inline void clear_url(); + static const int kUrlFieldNumber = 2; + inline const ::std::string& url() const; + inline void set_url(const ::std::string& value); + inline void set_url(const char* value); + inline void set_url(const char* value, size_t size); + inline ::std::string* mutable_url(); + inline ::std::string* release_url(); + inline void set_allocated_url(::std::string* url); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.ThreatEntry) + private: + inline void set_has_hash(); + inline void clear_has_hash(); + inline void set_has_url(); + inline void clear_has_url(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::std::string* hash_; + ::std::string* url_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static ThreatEntry* default_instance_; +}; +// ------------------------------------------------------------------- + +class ThreatEntrySet : public ::google::protobuf::MessageLite { + public: + ThreatEntrySet(); + virtual ~ThreatEntrySet(); + + ThreatEntrySet(const ThreatEntrySet& from); + + inline ThreatEntrySet& operator=(const ThreatEntrySet& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ThreatEntrySet& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const ThreatEntrySet* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(ThreatEntrySet* other); + + // implements Message ---------------------------------------------- + + ThreatEntrySet* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const ThreatEntrySet& from); + void MergeFrom(const ThreatEntrySet& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional .mozilla.safebrowsing.CompressionType compression_type = 1; + inline bool has_compression_type() const; + inline void clear_compression_type(); + static const int kCompressionTypeFieldNumber = 1; + inline ::mozilla::safebrowsing::CompressionType compression_type() const; + inline void set_compression_type(::mozilla::safebrowsing::CompressionType value); + + // optional .mozilla.safebrowsing.RawHashes raw_hashes = 2; + inline bool has_raw_hashes() const; + inline void clear_raw_hashes(); + static const int kRawHashesFieldNumber = 2; + inline const ::mozilla::safebrowsing::RawHashes& raw_hashes() const; + inline ::mozilla::safebrowsing::RawHashes* mutable_raw_hashes(); + inline ::mozilla::safebrowsing::RawHashes* release_raw_hashes(); + inline void set_allocated_raw_hashes(::mozilla::safebrowsing::RawHashes* raw_hashes); + + // optional .mozilla.safebrowsing.RawIndices raw_indices = 3; + inline bool has_raw_indices() const; + inline void clear_raw_indices(); + static const int kRawIndicesFieldNumber = 3; + inline const ::mozilla::safebrowsing::RawIndices& raw_indices() const; + inline ::mozilla::safebrowsing::RawIndices* mutable_raw_indices(); + inline ::mozilla::safebrowsing::RawIndices* release_raw_indices(); + inline void set_allocated_raw_indices(::mozilla::safebrowsing::RawIndices* raw_indices); + + // optional .mozilla.safebrowsing.RiceDeltaEncoding rice_hashes = 4; + inline bool has_rice_hashes() const; + inline void clear_rice_hashes(); + static const int kRiceHashesFieldNumber = 4; + inline const ::mozilla::safebrowsing::RiceDeltaEncoding& rice_hashes() const; + inline ::mozilla::safebrowsing::RiceDeltaEncoding* mutable_rice_hashes(); + inline ::mozilla::safebrowsing::RiceDeltaEncoding* release_rice_hashes(); + inline void set_allocated_rice_hashes(::mozilla::safebrowsing::RiceDeltaEncoding* rice_hashes); + + // optional .mozilla.safebrowsing.RiceDeltaEncoding rice_indices = 5; + inline bool has_rice_indices() const; + inline void clear_rice_indices(); + static const int kRiceIndicesFieldNumber = 5; + inline const ::mozilla::safebrowsing::RiceDeltaEncoding& rice_indices() const; + inline ::mozilla::safebrowsing::RiceDeltaEncoding* mutable_rice_indices(); + inline ::mozilla::safebrowsing::RiceDeltaEncoding* release_rice_indices(); + inline void set_allocated_rice_indices(::mozilla::safebrowsing::RiceDeltaEncoding* rice_indices); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.ThreatEntrySet) + private: + inline void set_has_compression_type(); + inline void clear_has_compression_type(); + inline void set_has_raw_hashes(); + inline void clear_has_raw_hashes(); + inline void set_has_raw_indices(); + inline void clear_has_raw_indices(); + inline void set_has_rice_hashes(); + inline void clear_has_rice_hashes(); + inline void set_has_rice_indices(); + inline void clear_has_rice_indices(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::mozilla::safebrowsing::RawHashes* raw_hashes_; + ::mozilla::safebrowsing::RawIndices* raw_indices_; + ::mozilla::safebrowsing::RiceDeltaEncoding* rice_hashes_; + ::mozilla::safebrowsing::RiceDeltaEncoding* rice_indices_; + int compression_type_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static ThreatEntrySet* default_instance_; +}; +// ------------------------------------------------------------------- + +class RawIndices : public ::google::protobuf::MessageLite { + public: + RawIndices(); + virtual ~RawIndices(); + + RawIndices(const RawIndices& from); + + inline RawIndices& operator=(const RawIndices& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const RawIndices& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const RawIndices* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(RawIndices* other); + + // implements Message ---------------------------------------------- + + RawIndices* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const RawIndices& from); + void MergeFrom(const RawIndices& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // repeated int32 indices = 1; + inline int indices_size() const; + inline void clear_indices(); + static const int kIndicesFieldNumber = 1; + inline ::google::protobuf::int32 indices(int index) const; + inline void set_indices(int index, ::google::protobuf::int32 value); + inline void add_indices(::google::protobuf::int32 value); + inline const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >& + indices() const; + inline ::google::protobuf::RepeatedField< ::google::protobuf::int32 >* + mutable_indices(); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.RawIndices) + private: + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::google::protobuf::RepeatedField< ::google::protobuf::int32 > indices_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static RawIndices* default_instance_; +}; +// ------------------------------------------------------------------- + +class RawHashes : public ::google::protobuf::MessageLite { + public: + RawHashes(); + virtual ~RawHashes(); + + RawHashes(const RawHashes& from); + + inline RawHashes& operator=(const RawHashes& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const RawHashes& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const RawHashes* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(RawHashes* other); + + // implements Message ---------------------------------------------- + + RawHashes* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const RawHashes& from); + void MergeFrom(const RawHashes& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional int32 prefix_size = 1; + inline bool has_prefix_size() const; + inline void clear_prefix_size(); + static const int kPrefixSizeFieldNumber = 1; + inline ::google::protobuf::int32 prefix_size() const; + inline void set_prefix_size(::google::protobuf::int32 value); + + // optional bytes raw_hashes = 2; + inline bool has_raw_hashes() const; + inline void clear_raw_hashes(); + static const int kRawHashesFieldNumber = 2; + inline const ::std::string& raw_hashes() const; + inline void set_raw_hashes(const ::std::string& value); + inline void set_raw_hashes(const char* value); + inline void set_raw_hashes(const void* value, size_t size); + inline ::std::string* mutable_raw_hashes(); + inline ::std::string* release_raw_hashes(); + inline void set_allocated_raw_hashes(::std::string* raw_hashes); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.RawHashes) + private: + inline void set_has_prefix_size(); + inline void clear_has_prefix_size(); + inline void set_has_raw_hashes(); + inline void clear_has_raw_hashes(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::std::string* raw_hashes_; + ::google::protobuf::int32 prefix_size_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static RawHashes* default_instance_; +}; +// ------------------------------------------------------------------- + +class RiceDeltaEncoding : public ::google::protobuf::MessageLite { + public: + RiceDeltaEncoding(); + virtual ~RiceDeltaEncoding(); + + RiceDeltaEncoding(const RiceDeltaEncoding& from); + + inline RiceDeltaEncoding& operator=(const RiceDeltaEncoding& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const RiceDeltaEncoding& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const RiceDeltaEncoding* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(RiceDeltaEncoding* other); + + // implements Message ---------------------------------------------- + + RiceDeltaEncoding* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const RiceDeltaEncoding& from); + void MergeFrom(const RiceDeltaEncoding& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional int64 first_value = 1; + inline bool has_first_value() const; + inline void clear_first_value(); + static const int kFirstValueFieldNumber = 1; + inline ::google::protobuf::int64 first_value() const; + inline void set_first_value(::google::protobuf::int64 value); + + // optional int32 rice_parameter = 2; + inline bool has_rice_parameter() const; + inline void clear_rice_parameter(); + static const int kRiceParameterFieldNumber = 2; + inline ::google::protobuf::int32 rice_parameter() const; + inline void set_rice_parameter(::google::protobuf::int32 value); + + // optional int32 num_entries = 3; + inline bool has_num_entries() const; + inline void clear_num_entries(); + static const int kNumEntriesFieldNumber = 3; + inline ::google::protobuf::int32 num_entries() const; + inline void set_num_entries(::google::protobuf::int32 value); + + // optional bytes encoded_data = 4; + inline bool has_encoded_data() const; + inline void clear_encoded_data(); + static const int kEncodedDataFieldNumber = 4; + inline const ::std::string& encoded_data() const; + inline void set_encoded_data(const ::std::string& value); + inline void set_encoded_data(const char* value); + inline void set_encoded_data(const void* value, size_t size); + inline ::std::string* mutable_encoded_data(); + inline ::std::string* release_encoded_data(); + inline void set_allocated_encoded_data(::std::string* encoded_data); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.RiceDeltaEncoding) + private: + inline void set_has_first_value(); + inline void clear_has_first_value(); + inline void set_has_rice_parameter(); + inline void clear_has_rice_parameter(); + inline void set_has_num_entries(); + inline void clear_has_num_entries(); + inline void set_has_encoded_data(); + inline void clear_has_encoded_data(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::google::protobuf::int64 first_value_; + ::google::protobuf::int32 rice_parameter_; + ::google::protobuf::int32 num_entries_; + ::std::string* encoded_data_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static RiceDeltaEncoding* default_instance_; +}; +// ------------------------------------------------------------------- + +class ThreatEntryMetadata_MetadataEntry : public ::google::protobuf::MessageLite { + public: + ThreatEntryMetadata_MetadataEntry(); + virtual ~ThreatEntryMetadata_MetadataEntry(); + + ThreatEntryMetadata_MetadataEntry(const ThreatEntryMetadata_MetadataEntry& from); + + inline ThreatEntryMetadata_MetadataEntry& operator=(const ThreatEntryMetadata_MetadataEntry& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ThreatEntryMetadata_MetadataEntry& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const ThreatEntryMetadata_MetadataEntry* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(ThreatEntryMetadata_MetadataEntry* other); + + // implements Message ---------------------------------------------- + + ThreatEntryMetadata_MetadataEntry* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const ThreatEntryMetadata_MetadataEntry& from); + void MergeFrom(const ThreatEntryMetadata_MetadataEntry& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional bytes key = 1; + inline bool has_key() const; + inline void clear_key(); + static const int kKeyFieldNumber = 1; + inline const ::std::string& key() const; + inline void set_key(const ::std::string& value); + inline void set_key(const char* value); + inline void set_key(const void* value, size_t size); + inline ::std::string* mutable_key(); + inline ::std::string* release_key(); + inline void set_allocated_key(::std::string* key); + + // optional bytes value = 2; + inline bool has_value() const; + inline void clear_value(); + static const int kValueFieldNumber = 2; + inline const ::std::string& value() const; + inline void set_value(const ::std::string& value); + inline void set_value(const char* value); + inline void set_value(const void* value, size_t size); + inline ::std::string* mutable_value(); + inline ::std::string* release_value(); + inline void set_allocated_value(::std::string* value); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry) + private: + inline void set_has_key(); + inline void clear_has_key(); + inline void set_has_value(); + inline void clear_has_value(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::std::string* key_; + ::std::string* value_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static ThreatEntryMetadata_MetadataEntry* default_instance_; +}; +// ------------------------------------------------------------------- + +class ThreatEntryMetadata : public ::google::protobuf::MessageLite { + public: + ThreatEntryMetadata(); + virtual ~ThreatEntryMetadata(); + + ThreatEntryMetadata(const ThreatEntryMetadata& from); + + inline ThreatEntryMetadata& operator=(const ThreatEntryMetadata& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ThreatEntryMetadata& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const ThreatEntryMetadata* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(ThreatEntryMetadata* other); + + // implements Message ---------------------------------------------- + + ThreatEntryMetadata* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const ThreatEntryMetadata& from); + void MergeFrom(const ThreatEntryMetadata& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + typedef ThreatEntryMetadata_MetadataEntry MetadataEntry; + + // accessors ------------------------------------------------------- + + // repeated .mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry entries = 1; + inline int entries_size() const; + inline void clear_entries(); + static const int kEntriesFieldNumber = 1; + inline const ::mozilla::safebrowsing::ThreatEntryMetadata_MetadataEntry& entries(int index) const; + inline ::mozilla::safebrowsing::ThreatEntryMetadata_MetadataEntry* mutable_entries(int index); + inline ::mozilla::safebrowsing::ThreatEntryMetadata_MetadataEntry* add_entries(); + inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntryMetadata_MetadataEntry >& + entries() const; + inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntryMetadata_MetadataEntry >* + mutable_entries(); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.ThreatEntryMetadata) + private: + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntryMetadata_MetadataEntry > entries_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static ThreatEntryMetadata* default_instance_; +}; +// ------------------------------------------------------------------- + +class ThreatListDescriptor : public ::google::protobuf::MessageLite { + public: + ThreatListDescriptor(); + virtual ~ThreatListDescriptor(); + + ThreatListDescriptor(const ThreatListDescriptor& from); + + inline ThreatListDescriptor& operator=(const ThreatListDescriptor& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ThreatListDescriptor& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const ThreatListDescriptor* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(ThreatListDescriptor* other); + + // implements Message ---------------------------------------------- + + ThreatListDescriptor* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const ThreatListDescriptor& from); + void MergeFrom(const ThreatListDescriptor& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional .mozilla.safebrowsing.ThreatType threat_type = 1; + inline bool has_threat_type() const; + inline void clear_threat_type(); + static const int kThreatTypeFieldNumber = 1; + inline ::mozilla::safebrowsing::ThreatType threat_type() const; + inline void set_threat_type(::mozilla::safebrowsing::ThreatType value); + + // optional .mozilla.safebrowsing.PlatformType platform_type = 2; + inline bool has_platform_type() const; + inline void clear_platform_type(); + static const int kPlatformTypeFieldNumber = 2; + inline ::mozilla::safebrowsing::PlatformType platform_type() const; + inline void set_platform_type(::mozilla::safebrowsing::PlatformType value); + + // optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 3; + inline bool has_threat_entry_type() const; + inline void clear_threat_entry_type(); + static const int kThreatEntryTypeFieldNumber = 3; + inline ::mozilla::safebrowsing::ThreatEntryType threat_entry_type() const; + inline void set_threat_entry_type(::mozilla::safebrowsing::ThreatEntryType value); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.ThreatListDescriptor) + private: + inline void set_has_threat_type(); + inline void clear_has_threat_type(); + inline void set_has_platform_type(); + inline void clear_has_platform_type(); + inline void set_has_threat_entry_type(); + inline void clear_has_threat_entry_type(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + int threat_type_; + int platform_type_; + int threat_entry_type_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static ThreatListDescriptor* default_instance_; +}; +// ------------------------------------------------------------------- + +class ListThreatListsResponse : public ::google::protobuf::MessageLite { + public: + ListThreatListsResponse(); + virtual ~ListThreatListsResponse(); + + ListThreatListsResponse(const ListThreatListsResponse& from); + + inline ListThreatListsResponse& operator=(const ListThreatListsResponse& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const ListThreatListsResponse& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const ListThreatListsResponse* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(ListThreatListsResponse* other); + + // implements Message ---------------------------------------------- + + ListThreatListsResponse* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const ListThreatListsResponse& from); + void MergeFrom(const ListThreatListsResponse& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // repeated .mozilla.safebrowsing.ThreatListDescriptor threat_lists = 1; + inline int threat_lists_size() const; + inline void clear_threat_lists(); + static const int kThreatListsFieldNumber = 1; + inline const ::mozilla::safebrowsing::ThreatListDescriptor& threat_lists(int index) const; + inline ::mozilla::safebrowsing::ThreatListDescriptor* mutable_threat_lists(int index); + inline ::mozilla::safebrowsing::ThreatListDescriptor* add_threat_lists(); + inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatListDescriptor >& + threat_lists() const; + inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatListDescriptor >* + mutable_threat_lists(); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.ListThreatListsResponse) + private: + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatListDescriptor > threat_lists_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static ListThreatListsResponse* default_instance_; +}; +// ------------------------------------------------------------------- + +class Duration : public ::google::protobuf::MessageLite { + public: + Duration(); + virtual ~Duration(); + + Duration(const Duration& from); + + inline Duration& operator=(const Duration& from) { + CopyFrom(from); + return *this; + } + + inline const ::std::string& unknown_fields() const { + return _unknown_fields_; + } + + inline ::std::string* mutable_unknown_fields() { + return &_unknown_fields_; + } + + static const Duration& default_instance(); + + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + // Returns the internal default instance pointer. This function can + // return NULL thus should not be used by the user. This is intended + // for Protobuf internal code. Please use default_instance() declared + // above instead. + static inline const Duration* internal_default_instance() { + return default_instance_; + } + #endif + + void Swap(Duration* other); + + // implements Message ---------------------------------------------- + + Duration* New() const; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from); + void CopyFrom(const Duration& from); + void MergeFrom(const Duration& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const; + void DiscardUnknownFields(); + int GetCachedSize() const { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + public: + ::std::string GetTypeName() const; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // optional int64 seconds = 1; + inline bool has_seconds() const; + inline void clear_seconds(); + static const int kSecondsFieldNumber = 1; + inline ::google::protobuf::int64 seconds() const; + inline void set_seconds(::google::protobuf::int64 value); + + // optional int32 nanos = 2; + inline bool has_nanos() const; + inline void clear_nanos(); + static const int kNanosFieldNumber = 2; + inline ::google::protobuf::int32 nanos() const; + inline void set_nanos(::google::protobuf::int32 value); + + // @@protoc_insertion_point(class_scope:mozilla.safebrowsing.Duration) + private: + inline void set_has_seconds(); + inline void clear_has_seconds(); + inline void set_has_nanos(); + inline void clear_has_nanos(); + + ::std::string _unknown_fields_; + + ::google::protobuf::uint32 _has_bits_[1]; + mutable int _cached_size_; + ::google::protobuf::int64 seconds_; + ::google::protobuf::int32 nanos_; + #ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + friend void protobuf_AddDesc_safebrowsing_2eproto_impl(); + #else + friend void protobuf_AddDesc_safebrowsing_2eproto(); + #endif + friend void protobuf_AssignDesc_safebrowsing_2eproto(); + friend void protobuf_ShutdownFile_safebrowsing_2eproto(); + + void InitAsDefaultInstance(); + static Duration* default_instance_; +}; +// =================================================================== + + +// =================================================================== + +// ThreatInfo + +// repeated .mozilla.safebrowsing.ThreatType threat_types = 1; +inline int ThreatInfo::threat_types_size() const { + return threat_types_.size(); +} +inline void ThreatInfo::clear_threat_types() { + threat_types_.Clear(); +} +inline ::mozilla::safebrowsing::ThreatType ThreatInfo::threat_types(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatInfo.threat_types) + return static_cast< ::mozilla::safebrowsing::ThreatType >(threat_types_.Get(index)); +} +inline void ThreatInfo::set_threat_types(int index, ::mozilla::safebrowsing::ThreatType value) { + assert(::mozilla::safebrowsing::ThreatType_IsValid(value)); + threat_types_.Set(index, value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatInfo.threat_types) +} +inline void ThreatInfo::add_threat_types(::mozilla::safebrowsing::ThreatType value) { + assert(::mozilla::safebrowsing::ThreatType_IsValid(value)); + threat_types_.Add(value); + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.ThreatInfo.threat_types) +} +inline const ::google::protobuf::RepeatedField<int>& +ThreatInfo::threat_types() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.ThreatInfo.threat_types) + return threat_types_; +} +inline ::google::protobuf::RepeatedField<int>* +ThreatInfo::mutable_threat_types() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.ThreatInfo.threat_types) + return &threat_types_; +} + +// repeated .mozilla.safebrowsing.PlatformType platform_types = 2; +inline int ThreatInfo::platform_types_size() const { + return platform_types_.size(); +} +inline void ThreatInfo::clear_platform_types() { + platform_types_.Clear(); +} +inline ::mozilla::safebrowsing::PlatformType ThreatInfo::platform_types(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatInfo.platform_types) + return static_cast< ::mozilla::safebrowsing::PlatformType >(platform_types_.Get(index)); +} +inline void ThreatInfo::set_platform_types(int index, ::mozilla::safebrowsing::PlatformType value) { + assert(::mozilla::safebrowsing::PlatformType_IsValid(value)); + platform_types_.Set(index, value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatInfo.platform_types) +} +inline void ThreatInfo::add_platform_types(::mozilla::safebrowsing::PlatformType value) { + assert(::mozilla::safebrowsing::PlatformType_IsValid(value)); + platform_types_.Add(value); + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.ThreatInfo.platform_types) +} +inline const ::google::protobuf::RepeatedField<int>& +ThreatInfo::platform_types() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.ThreatInfo.platform_types) + return platform_types_; +} +inline ::google::protobuf::RepeatedField<int>* +ThreatInfo::mutable_platform_types() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.ThreatInfo.platform_types) + return &platform_types_; +} + +// repeated .mozilla.safebrowsing.ThreatEntryType threat_entry_types = 4; +inline int ThreatInfo::threat_entry_types_size() const { + return threat_entry_types_.size(); +} +inline void ThreatInfo::clear_threat_entry_types() { + threat_entry_types_.Clear(); +} +inline ::mozilla::safebrowsing::ThreatEntryType ThreatInfo::threat_entry_types(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatInfo.threat_entry_types) + return static_cast< ::mozilla::safebrowsing::ThreatEntryType >(threat_entry_types_.Get(index)); +} +inline void ThreatInfo::set_threat_entry_types(int index, ::mozilla::safebrowsing::ThreatEntryType value) { + assert(::mozilla::safebrowsing::ThreatEntryType_IsValid(value)); + threat_entry_types_.Set(index, value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatInfo.threat_entry_types) +} +inline void ThreatInfo::add_threat_entry_types(::mozilla::safebrowsing::ThreatEntryType value) { + assert(::mozilla::safebrowsing::ThreatEntryType_IsValid(value)); + threat_entry_types_.Add(value); + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.ThreatInfo.threat_entry_types) +} +inline const ::google::protobuf::RepeatedField<int>& +ThreatInfo::threat_entry_types() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.ThreatInfo.threat_entry_types) + return threat_entry_types_; +} +inline ::google::protobuf::RepeatedField<int>* +ThreatInfo::mutable_threat_entry_types() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.ThreatInfo.threat_entry_types) + return &threat_entry_types_; +} + +// repeated .mozilla.safebrowsing.ThreatEntry threat_entries = 3; +inline int ThreatInfo::threat_entries_size() const { + return threat_entries_.size(); +} +inline void ThreatInfo::clear_threat_entries() { + threat_entries_.Clear(); +} +inline const ::mozilla::safebrowsing::ThreatEntry& ThreatInfo::threat_entries(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatInfo.threat_entries) + return threat_entries_.Get(index); +} +inline ::mozilla::safebrowsing::ThreatEntry* ThreatInfo::mutable_threat_entries(int index) { + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatInfo.threat_entries) + return threat_entries_.Mutable(index); +} +inline ::mozilla::safebrowsing::ThreatEntry* ThreatInfo::add_threat_entries() { + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.ThreatInfo.threat_entries) + return threat_entries_.Add(); +} +inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntry >& +ThreatInfo::threat_entries() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.ThreatInfo.threat_entries) + return threat_entries_; +} +inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntry >* +ThreatInfo::mutable_threat_entries() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.ThreatInfo.threat_entries) + return &threat_entries_; +} + +// ------------------------------------------------------------------- + +// ThreatMatch + +// optional .mozilla.safebrowsing.ThreatType threat_type = 1; +inline bool ThreatMatch::has_threat_type() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void ThreatMatch::set_has_threat_type() { + _has_bits_[0] |= 0x00000001u; +} +inline void ThreatMatch::clear_has_threat_type() { + _has_bits_[0] &= ~0x00000001u; +} +inline void ThreatMatch::clear_threat_type() { + threat_type_ = 0; + clear_has_threat_type(); +} +inline ::mozilla::safebrowsing::ThreatType ThreatMatch::threat_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatMatch.threat_type) + return static_cast< ::mozilla::safebrowsing::ThreatType >(threat_type_); +} +inline void ThreatMatch::set_threat_type(::mozilla::safebrowsing::ThreatType value) { + assert(::mozilla::safebrowsing::ThreatType_IsValid(value)); + set_has_threat_type(); + threat_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatMatch.threat_type) +} + +// optional .mozilla.safebrowsing.PlatformType platform_type = 2; +inline bool ThreatMatch::has_platform_type() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void ThreatMatch::set_has_platform_type() { + _has_bits_[0] |= 0x00000002u; +} +inline void ThreatMatch::clear_has_platform_type() { + _has_bits_[0] &= ~0x00000002u; +} +inline void ThreatMatch::clear_platform_type() { + platform_type_ = 0; + clear_has_platform_type(); +} +inline ::mozilla::safebrowsing::PlatformType ThreatMatch::platform_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatMatch.platform_type) + return static_cast< ::mozilla::safebrowsing::PlatformType >(platform_type_); +} +inline void ThreatMatch::set_platform_type(::mozilla::safebrowsing::PlatformType value) { + assert(::mozilla::safebrowsing::PlatformType_IsValid(value)); + set_has_platform_type(); + platform_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatMatch.platform_type) +} + +// optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 6; +inline bool ThreatMatch::has_threat_entry_type() const { + return (_has_bits_[0] & 0x00000004u) != 0; +} +inline void ThreatMatch::set_has_threat_entry_type() { + _has_bits_[0] |= 0x00000004u; +} +inline void ThreatMatch::clear_has_threat_entry_type() { + _has_bits_[0] &= ~0x00000004u; +} +inline void ThreatMatch::clear_threat_entry_type() { + threat_entry_type_ = 0; + clear_has_threat_entry_type(); +} +inline ::mozilla::safebrowsing::ThreatEntryType ThreatMatch::threat_entry_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatMatch.threat_entry_type) + return static_cast< ::mozilla::safebrowsing::ThreatEntryType >(threat_entry_type_); +} +inline void ThreatMatch::set_threat_entry_type(::mozilla::safebrowsing::ThreatEntryType value) { + assert(::mozilla::safebrowsing::ThreatEntryType_IsValid(value)); + set_has_threat_entry_type(); + threat_entry_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatMatch.threat_entry_type) +} + +// optional .mozilla.safebrowsing.ThreatEntry threat = 3; +inline bool ThreatMatch::has_threat() const { + return (_has_bits_[0] & 0x00000008u) != 0; +} +inline void ThreatMatch::set_has_threat() { + _has_bits_[0] |= 0x00000008u; +} +inline void ThreatMatch::clear_has_threat() { + _has_bits_[0] &= ~0x00000008u; +} +inline void ThreatMatch::clear_threat() { + if (threat_ != NULL) threat_->::mozilla::safebrowsing::ThreatEntry::Clear(); + clear_has_threat(); +} +inline const ::mozilla::safebrowsing::ThreatEntry& ThreatMatch::threat() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatMatch.threat) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return threat_ != NULL ? *threat_ : *default_instance().threat_; +#else + return threat_ != NULL ? *threat_ : *default_instance_->threat_; +#endif +} +inline ::mozilla::safebrowsing::ThreatEntry* ThreatMatch::mutable_threat() { + set_has_threat(); + if (threat_ == NULL) threat_ = new ::mozilla::safebrowsing::ThreatEntry; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatMatch.threat) + return threat_; +} +inline ::mozilla::safebrowsing::ThreatEntry* ThreatMatch::release_threat() { + clear_has_threat(); + ::mozilla::safebrowsing::ThreatEntry* temp = threat_; + threat_ = NULL; + return temp; +} +inline void ThreatMatch::set_allocated_threat(::mozilla::safebrowsing::ThreatEntry* threat) { + delete threat_; + threat_ = threat; + if (threat) { + set_has_threat(); + } else { + clear_has_threat(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatMatch.threat) +} + +// optional .mozilla.safebrowsing.ThreatEntryMetadata threat_entry_metadata = 4; +inline bool ThreatMatch::has_threat_entry_metadata() const { + return (_has_bits_[0] & 0x00000010u) != 0; +} +inline void ThreatMatch::set_has_threat_entry_metadata() { + _has_bits_[0] |= 0x00000010u; +} +inline void ThreatMatch::clear_has_threat_entry_metadata() { + _has_bits_[0] &= ~0x00000010u; +} +inline void ThreatMatch::clear_threat_entry_metadata() { + if (threat_entry_metadata_ != NULL) threat_entry_metadata_->::mozilla::safebrowsing::ThreatEntryMetadata::Clear(); + clear_has_threat_entry_metadata(); +} +inline const ::mozilla::safebrowsing::ThreatEntryMetadata& ThreatMatch::threat_entry_metadata() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatMatch.threat_entry_metadata) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return threat_entry_metadata_ != NULL ? *threat_entry_metadata_ : *default_instance().threat_entry_metadata_; +#else + return threat_entry_metadata_ != NULL ? *threat_entry_metadata_ : *default_instance_->threat_entry_metadata_; +#endif +} +inline ::mozilla::safebrowsing::ThreatEntryMetadata* ThreatMatch::mutable_threat_entry_metadata() { + set_has_threat_entry_metadata(); + if (threat_entry_metadata_ == NULL) threat_entry_metadata_ = new ::mozilla::safebrowsing::ThreatEntryMetadata; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatMatch.threat_entry_metadata) + return threat_entry_metadata_; +} +inline ::mozilla::safebrowsing::ThreatEntryMetadata* ThreatMatch::release_threat_entry_metadata() { + clear_has_threat_entry_metadata(); + ::mozilla::safebrowsing::ThreatEntryMetadata* temp = threat_entry_metadata_; + threat_entry_metadata_ = NULL; + return temp; +} +inline void ThreatMatch::set_allocated_threat_entry_metadata(::mozilla::safebrowsing::ThreatEntryMetadata* threat_entry_metadata) { + delete threat_entry_metadata_; + threat_entry_metadata_ = threat_entry_metadata; + if (threat_entry_metadata) { + set_has_threat_entry_metadata(); + } else { + clear_has_threat_entry_metadata(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatMatch.threat_entry_metadata) +} + +// optional .mozilla.safebrowsing.Duration cache_duration = 5; +inline bool ThreatMatch::has_cache_duration() const { + return (_has_bits_[0] & 0x00000020u) != 0; +} +inline void ThreatMatch::set_has_cache_duration() { + _has_bits_[0] |= 0x00000020u; +} +inline void ThreatMatch::clear_has_cache_duration() { + _has_bits_[0] &= ~0x00000020u; +} +inline void ThreatMatch::clear_cache_duration() { + if (cache_duration_ != NULL) cache_duration_->::mozilla::safebrowsing::Duration::Clear(); + clear_has_cache_duration(); +} +inline const ::mozilla::safebrowsing::Duration& ThreatMatch::cache_duration() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatMatch.cache_duration) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return cache_duration_ != NULL ? *cache_duration_ : *default_instance().cache_duration_; +#else + return cache_duration_ != NULL ? *cache_duration_ : *default_instance_->cache_duration_; +#endif +} +inline ::mozilla::safebrowsing::Duration* ThreatMatch::mutable_cache_duration() { + set_has_cache_duration(); + if (cache_duration_ == NULL) cache_duration_ = new ::mozilla::safebrowsing::Duration; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatMatch.cache_duration) + return cache_duration_; +} +inline ::mozilla::safebrowsing::Duration* ThreatMatch::release_cache_duration() { + clear_has_cache_duration(); + ::mozilla::safebrowsing::Duration* temp = cache_duration_; + cache_duration_ = NULL; + return temp; +} +inline void ThreatMatch::set_allocated_cache_duration(::mozilla::safebrowsing::Duration* cache_duration) { + delete cache_duration_; + cache_duration_ = cache_duration; + if (cache_duration) { + set_has_cache_duration(); + } else { + clear_has_cache_duration(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatMatch.cache_duration) +} + +// ------------------------------------------------------------------- + +// FindThreatMatchesRequest + +// optional .mozilla.safebrowsing.ClientInfo client = 1; +inline bool FindThreatMatchesRequest::has_client() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void FindThreatMatchesRequest::set_has_client() { + _has_bits_[0] |= 0x00000001u; +} +inline void FindThreatMatchesRequest::clear_has_client() { + _has_bits_[0] &= ~0x00000001u; +} +inline void FindThreatMatchesRequest::clear_client() { + if (client_ != NULL) client_->::mozilla::safebrowsing::ClientInfo::Clear(); + clear_has_client(); +} +inline const ::mozilla::safebrowsing::ClientInfo& FindThreatMatchesRequest::client() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FindThreatMatchesRequest.client) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return client_ != NULL ? *client_ : *default_instance().client_; +#else + return client_ != NULL ? *client_ : *default_instance_->client_; +#endif +} +inline ::mozilla::safebrowsing::ClientInfo* FindThreatMatchesRequest::mutable_client() { + set_has_client(); + if (client_ == NULL) client_ = new ::mozilla::safebrowsing::ClientInfo; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FindThreatMatchesRequest.client) + return client_; +} +inline ::mozilla::safebrowsing::ClientInfo* FindThreatMatchesRequest::release_client() { + clear_has_client(); + ::mozilla::safebrowsing::ClientInfo* temp = client_; + client_ = NULL; + return temp; +} +inline void FindThreatMatchesRequest::set_allocated_client(::mozilla::safebrowsing::ClientInfo* client) { + delete client_; + client_ = client; + if (client) { + set_has_client(); + } else { + clear_has_client(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.FindThreatMatchesRequest.client) +} + +// optional .mozilla.safebrowsing.ThreatInfo threat_info = 2; +inline bool FindThreatMatchesRequest::has_threat_info() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void FindThreatMatchesRequest::set_has_threat_info() { + _has_bits_[0] |= 0x00000002u; +} +inline void FindThreatMatchesRequest::clear_has_threat_info() { + _has_bits_[0] &= ~0x00000002u; +} +inline void FindThreatMatchesRequest::clear_threat_info() { + if (threat_info_ != NULL) threat_info_->::mozilla::safebrowsing::ThreatInfo::Clear(); + clear_has_threat_info(); +} +inline const ::mozilla::safebrowsing::ThreatInfo& FindThreatMatchesRequest::threat_info() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FindThreatMatchesRequest.threat_info) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return threat_info_ != NULL ? *threat_info_ : *default_instance().threat_info_; +#else + return threat_info_ != NULL ? *threat_info_ : *default_instance_->threat_info_; +#endif +} +inline ::mozilla::safebrowsing::ThreatInfo* FindThreatMatchesRequest::mutable_threat_info() { + set_has_threat_info(); + if (threat_info_ == NULL) threat_info_ = new ::mozilla::safebrowsing::ThreatInfo; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FindThreatMatchesRequest.threat_info) + return threat_info_; +} +inline ::mozilla::safebrowsing::ThreatInfo* FindThreatMatchesRequest::release_threat_info() { + clear_has_threat_info(); + ::mozilla::safebrowsing::ThreatInfo* temp = threat_info_; + threat_info_ = NULL; + return temp; +} +inline void FindThreatMatchesRequest::set_allocated_threat_info(::mozilla::safebrowsing::ThreatInfo* threat_info) { + delete threat_info_; + threat_info_ = threat_info; + if (threat_info) { + set_has_threat_info(); + } else { + clear_has_threat_info(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.FindThreatMatchesRequest.threat_info) +} + +// ------------------------------------------------------------------- + +// FindThreatMatchesResponse + +// repeated .mozilla.safebrowsing.ThreatMatch matches = 1; +inline int FindThreatMatchesResponse::matches_size() const { + return matches_.size(); +} +inline void FindThreatMatchesResponse::clear_matches() { + matches_.Clear(); +} +inline const ::mozilla::safebrowsing::ThreatMatch& FindThreatMatchesResponse::matches(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FindThreatMatchesResponse.matches) + return matches_.Get(index); +} +inline ::mozilla::safebrowsing::ThreatMatch* FindThreatMatchesResponse::mutable_matches(int index) { + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FindThreatMatchesResponse.matches) + return matches_.Mutable(index); +} +inline ::mozilla::safebrowsing::ThreatMatch* FindThreatMatchesResponse::add_matches() { + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.FindThreatMatchesResponse.matches) + return matches_.Add(); +} +inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatMatch >& +FindThreatMatchesResponse::matches() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.FindThreatMatchesResponse.matches) + return matches_; +} +inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatMatch >* +FindThreatMatchesResponse::mutable_matches() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.FindThreatMatchesResponse.matches) + return &matches_; +} + +// ------------------------------------------------------------------- + +// FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints + +// optional int32 max_update_entries = 1; +inline bool FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::has_max_update_entries() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::set_has_max_update_entries() { + _has_bits_[0] |= 0x00000001u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::clear_has_max_update_entries() { + _has_bits_[0] &= ~0x00000001u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::clear_max_update_entries() { + max_update_entries_ = 0; + clear_has_max_update_entries(); +} +inline ::google::protobuf::int32 FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::max_update_entries() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.max_update_entries) + return max_update_entries_; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::set_max_update_entries(::google::protobuf::int32 value) { + set_has_max_update_entries(); + max_update_entries_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.max_update_entries) +} + +// optional int32 max_database_entries = 2; +inline bool FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::has_max_database_entries() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::set_has_max_database_entries() { + _has_bits_[0] |= 0x00000002u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::clear_has_max_database_entries() { + _has_bits_[0] &= ~0x00000002u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::clear_max_database_entries() { + max_database_entries_ = 0; + clear_has_max_database_entries(); +} +inline ::google::protobuf::int32 FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::max_database_entries() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.max_database_entries) + return max_database_entries_; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::set_max_database_entries(::google::protobuf::int32 value) { + set_has_max_database_entries(); + max_database_entries_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.max_database_entries) +} + +// optional string region = 3; +inline bool FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::has_region() const { + return (_has_bits_[0] & 0x00000004u) != 0; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::set_has_region() { + _has_bits_[0] |= 0x00000004u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::clear_has_region() { + _has_bits_[0] &= ~0x00000004u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::clear_region() { + if (region_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + region_->clear(); + } + clear_has_region(); +} +inline const ::std::string& FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::region() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.region) + return *region_; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::set_region(const ::std::string& value) { + set_has_region(); + if (region_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + region_ = new ::std::string; + } + region_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.region) +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::set_region(const char* value) { + set_has_region(); + if (region_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + region_ = new ::std::string; + } + region_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.region) +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::set_region(const char* value, size_t size) { + set_has_region(); + if (region_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + region_ = new ::std::string; + } + region_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.region) +} +inline ::std::string* FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::mutable_region() { + set_has_region(); + if (region_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + region_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.region) + return region_; +} +inline ::std::string* FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::release_region() { + clear_has_region(); + if (region_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = region_; + region_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::set_allocated_region(::std::string* region) { + if (region_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete region_; + } + if (region) { + set_has_region(); + region_ = region; + } else { + clear_has_region(); + region_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.region) +} + +// repeated .mozilla.safebrowsing.CompressionType supported_compressions = 4; +inline int FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::supported_compressions_size() const { + return supported_compressions_.size(); +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::clear_supported_compressions() { + supported_compressions_.Clear(); +} +inline ::mozilla::safebrowsing::CompressionType FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::supported_compressions(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.supported_compressions) + return static_cast< ::mozilla::safebrowsing::CompressionType >(supported_compressions_.Get(index)); +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::set_supported_compressions(int index, ::mozilla::safebrowsing::CompressionType value) { + assert(::mozilla::safebrowsing::CompressionType_IsValid(value)); + supported_compressions_.Set(index, value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.supported_compressions) +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::add_supported_compressions(::mozilla::safebrowsing::CompressionType value) { + assert(::mozilla::safebrowsing::CompressionType_IsValid(value)); + supported_compressions_.Add(value); + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.supported_compressions) +} +inline const ::google::protobuf::RepeatedField<int>& +FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::supported_compressions() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.supported_compressions) + return supported_compressions_; +} +inline ::google::protobuf::RepeatedField<int>* +FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::mutable_supported_compressions() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints.supported_compressions) + return &supported_compressions_; +} + +// ------------------------------------------------------------------- + +// FetchThreatListUpdatesRequest_ListUpdateRequest + +// optional .mozilla.safebrowsing.ThreatType threat_type = 1; +inline bool FetchThreatListUpdatesRequest_ListUpdateRequest::has_threat_type() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::set_has_threat_type() { + _has_bits_[0] |= 0x00000001u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::clear_has_threat_type() { + _has_bits_[0] &= ~0x00000001u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::clear_threat_type() { + threat_type_ = 0; + clear_has_threat_type(); +} +inline ::mozilla::safebrowsing::ThreatType FetchThreatListUpdatesRequest_ListUpdateRequest::threat_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.threat_type) + return static_cast< ::mozilla::safebrowsing::ThreatType >(threat_type_); +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::set_threat_type(::mozilla::safebrowsing::ThreatType value) { + assert(::mozilla::safebrowsing::ThreatType_IsValid(value)); + set_has_threat_type(); + threat_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.threat_type) +} + +// optional .mozilla.safebrowsing.PlatformType platform_type = 2; +inline bool FetchThreatListUpdatesRequest_ListUpdateRequest::has_platform_type() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::set_has_platform_type() { + _has_bits_[0] |= 0x00000002u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::clear_has_platform_type() { + _has_bits_[0] &= ~0x00000002u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::clear_platform_type() { + platform_type_ = 0; + clear_has_platform_type(); +} +inline ::mozilla::safebrowsing::PlatformType FetchThreatListUpdatesRequest_ListUpdateRequest::platform_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.platform_type) + return static_cast< ::mozilla::safebrowsing::PlatformType >(platform_type_); +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::set_platform_type(::mozilla::safebrowsing::PlatformType value) { + assert(::mozilla::safebrowsing::PlatformType_IsValid(value)); + set_has_platform_type(); + platform_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.platform_type) +} + +// optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 5; +inline bool FetchThreatListUpdatesRequest_ListUpdateRequest::has_threat_entry_type() const { + return (_has_bits_[0] & 0x00000004u) != 0; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::set_has_threat_entry_type() { + _has_bits_[0] |= 0x00000004u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::clear_has_threat_entry_type() { + _has_bits_[0] &= ~0x00000004u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::clear_threat_entry_type() { + threat_entry_type_ = 0; + clear_has_threat_entry_type(); +} +inline ::mozilla::safebrowsing::ThreatEntryType FetchThreatListUpdatesRequest_ListUpdateRequest::threat_entry_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.threat_entry_type) + return static_cast< ::mozilla::safebrowsing::ThreatEntryType >(threat_entry_type_); +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::set_threat_entry_type(::mozilla::safebrowsing::ThreatEntryType value) { + assert(::mozilla::safebrowsing::ThreatEntryType_IsValid(value)); + set_has_threat_entry_type(); + threat_entry_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.threat_entry_type) +} + +// optional bytes state = 3; +inline bool FetchThreatListUpdatesRequest_ListUpdateRequest::has_state() const { + return (_has_bits_[0] & 0x00000008u) != 0; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::set_has_state() { + _has_bits_[0] |= 0x00000008u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::clear_has_state() { + _has_bits_[0] &= ~0x00000008u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::clear_state() { + if (state_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + state_->clear(); + } + clear_has_state(); +} +inline const ::std::string& FetchThreatListUpdatesRequest_ListUpdateRequest::state() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.state) + return *state_; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::set_state(const ::std::string& value) { + set_has_state(); + if (state_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + state_ = new ::std::string; + } + state_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.state) +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::set_state(const char* value) { + set_has_state(); + if (state_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + state_ = new ::std::string; + } + state_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.state) +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::set_state(const void* value, size_t size) { + set_has_state(); + if (state_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + state_ = new ::std::string; + } + state_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.state) +} +inline ::std::string* FetchThreatListUpdatesRequest_ListUpdateRequest::mutable_state() { + set_has_state(); + if (state_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + state_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.state) + return state_; +} +inline ::std::string* FetchThreatListUpdatesRequest_ListUpdateRequest::release_state() { + clear_has_state(); + if (state_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = state_; + state_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::set_allocated_state(::std::string* state) { + if (state_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete state_; + } + if (state) { + set_has_state(); + state_ = state; + } else { + clear_has_state(); + state_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.state) +} + +// optional .mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.Constraints constraints = 4; +inline bool FetchThreatListUpdatesRequest_ListUpdateRequest::has_constraints() const { + return (_has_bits_[0] & 0x00000010u) != 0; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::set_has_constraints() { + _has_bits_[0] |= 0x00000010u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::clear_has_constraints() { + _has_bits_[0] &= ~0x00000010u; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::clear_constraints() { + if (constraints_ != NULL) constraints_->::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints::Clear(); + clear_has_constraints(); +} +inline const ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints& FetchThreatListUpdatesRequest_ListUpdateRequest::constraints() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.constraints) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return constraints_ != NULL ? *constraints_ : *default_instance().constraints_; +#else + return constraints_ != NULL ? *constraints_ : *default_instance_->constraints_; +#endif +} +inline ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* FetchThreatListUpdatesRequest_ListUpdateRequest::mutable_constraints() { + set_has_constraints(); + if (constraints_ == NULL) constraints_ = new ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.constraints) + return constraints_; +} +inline ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* FetchThreatListUpdatesRequest_ListUpdateRequest::release_constraints() { + clear_has_constraints(); + ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* temp = constraints_; + constraints_ = NULL; + return temp; +} +inline void FetchThreatListUpdatesRequest_ListUpdateRequest::set_allocated_constraints(::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints* constraints) { + delete constraints_; + constraints_ = constraints; + if (constraints) { + set_has_constraints(); + } else { + clear_has_constraints(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest.constraints) +} + +// ------------------------------------------------------------------- + +// FetchThreatListUpdatesRequest + +// optional .mozilla.safebrowsing.ClientInfo client = 1; +inline bool FetchThreatListUpdatesRequest::has_client() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void FetchThreatListUpdatesRequest::set_has_client() { + _has_bits_[0] |= 0x00000001u; +} +inline void FetchThreatListUpdatesRequest::clear_has_client() { + _has_bits_[0] &= ~0x00000001u; +} +inline void FetchThreatListUpdatesRequest::clear_client() { + if (client_ != NULL) client_->::mozilla::safebrowsing::ClientInfo::Clear(); + clear_has_client(); +} +inline const ::mozilla::safebrowsing::ClientInfo& FetchThreatListUpdatesRequest::client() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesRequest.client) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return client_ != NULL ? *client_ : *default_instance().client_; +#else + return client_ != NULL ? *client_ : *default_instance_->client_; +#endif +} +inline ::mozilla::safebrowsing::ClientInfo* FetchThreatListUpdatesRequest::mutable_client() { + set_has_client(); + if (client_ == NULL) client_ = new ::mozilla::safebrowsing::ClientInfo; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FetchThreatListUpdatesRequest.client) + return client_; +} +inline ::mozilla::safebrowsing::ClientInfo* FetchThreatListUpdatesRequest::release_client() { + clear_has_client(); + ::mozilla::safebrowsing::ClientInfo* temp = client_; + client_ = NULL; + return temp; +} +inline void FetchThreatListUpdatesRequest::set_allocated_client(::mozilla::safebrowsing::ClientInfo* client) { + delete client_; + client_ = client; + if (client) { + set_has_client(); + } else { + clear_has_client(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.FetchThreatListUpdatesRequest.client) +} + +// repeated .mozilla.safebrowsing.FetchThreatListUpdatesRequest.ListUpdateRequest list_update_requests = 3; +inline int FetchThreatListUpdatesRequest::list_update_requests_size() const { + return list_update_requests_.size(); +} +inline void FetchThreatListUpdatesRequest::clear_list_update_requests() { + list_update_requests_.Clear(); +} +inline const ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest& FetchThreatListUpdatesRequest::list_update_requests(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesRequest.list_update_requests) + return list_update_requests_.Get(index); +} +inline ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest* FetchThreatListUpdatesRequest::mutable_list_update_requests(int index) { + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FetchThreatListUpdatesRequest.list_update_requests) + return list_update_requests_.Mutable(index); +} +inline ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest* FetchThreatListUpdatesRequest::add_list_update_requests() { + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.FetchThreatListUpdatesRequest.list_update_requests) + return list_update_requests_.Add(); +} +inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest >& +FetchThreatListUpdatesRequest::list_update_requests() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.FetchThreatListUpdatesRequest.list_update_requests) + return list_update_requests_; +} +inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::FetchThreatListUpdatesRequest_ListUpdateRequest >* +FetchThreatListUpdatesRequest::mutable_list_update_requests() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.FetchThreatListUpdatesRequest.list_update_requests) + return &list_update_requests_; +} + +// ------------------------------------------------------------------- + +// FetchThreatListUpdatesResponse_ListUpdateResponse + +// optional .mozilla.safebrowsing.ThreatType threat_type = 1; +inline bool FetchThreatListUpdatesResponse_ListUpdateResponse::has_threat_type() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_has_threat_type() { + _has_bits_[0] |= 0x00000001u; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::clear_has_threat_type() { + _has_bits_[0] &= ~0x00000001u; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::clear_threat_type() { + threat_type_ = 0; + clear_has_threat_type(); +} +inline ::mozilla::safebrowsing::ThreatType FetchThreatListUpdatesResponse_ListUpdateResponse::threat_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.threat_type) + return static_cast< ::mozilla::safebrowsing::ThreatType >(threat_type_); +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_threat_type(::mozilla::safebrowsing::ThreatType value) { + assert(::mozilla::safebrowsing::ThreatType_IsValid(value)); + set_has_threat_type(); + threat_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.threat_type) +} + +// optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 2; +inline bool FetchThreatListUpdatesResponse_ListUpdateResponse::has_threat_entry_type() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_has_threat_entry_type() { + _has_bits_[0] |= 0x00000002u; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::clear_has_threat_entry_type() { + _has_bits_[0] &= ~0x00000002u; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::clear_threat_entry_type() { + threat_entry_type_ = 0; + clear_has_threat_entry_type(); +} +inline ::mozilla::safebrowsing::ThreatEntryType FetchThreatListUpdatesResponse_ListUpdateResponse::threat_entry_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.threat_entry_type) + return static_cast< ::mozilla::safebrowsing::ThreatEntryType >(threat_entry_type_); +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_threat_entry_type(::mozilla::safebrowsing::ThreatEntryType value) { + assert(::mozilla::safebrowsing::ThreatEntryType_IsValid(value)); + set_has_threat_entry_type(); + threat_entry_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.threat_entry_type) +} + +// optional .mozilla.safebrowsing.PlatformType platform_type = 3; +inline bool FetchThreatListUpdatesResponse_ListUpdateResponse::has_platform_type() const { + return (_has_bits_[0] & 0x00000004u) != 0; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_has_platform_type() { + _has_bits_[0] |= 0x00000004u; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::clear_has_platform_type() { + _has_bits_[0] &= ~0x00000004u; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::clear_platform_type() { + platform_type_ = 0; + clear_has_platform_type(); +} +inline ::mozilla::safebrowsing::PlatformType FetchThreatListUpdatesResponse_ListUpdateResponse::platform_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.platform_type) + return static_cast< ::mozilla::safebrowsing::PlatformType >(platform_type_); +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_platform_type(::mozilla::safebrowsing::PlatformType value) { + assert(::mozilla::safebrowsing::PlatformType_IsValid(value)); + set_has_platform_type(); + platform_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.platform_type) +} + +// optional .mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.ResponseType response_type = 4; +inline bool FetchThreatListUpdatesResponse_ListUpdateResponse::has_response_type() const { + return (_has_bits_[0] & 0x00000008u) != 0; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_has_response_type() { + _has_bits_[0] |= 0x00000008u; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::clear_has_response_type() { + _has_bits_[0] &= ~0x00000008u; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::clear_response_type() { + response_type_ = 0; + clear_has_response_type(); +} +inline ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType FetchThreatListUpdatesResponse_ListUpdateResponse::response_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.response_type) + return static_cast< ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType >(response_type_); +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_response_type(::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType value) { + assert(::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType_IsValid(value)); + set_has_response_type(); + response_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.response_type) +} + +// repeated .mozilla.safebrowsing.ThreatEntrySet additions = 5; +inline int FetchThreatListUpdatesResponse_ListUpdateResponse::additions_size() const { + return additions_.size(); +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::clear_additions() { + additions_.Clear(); +} +inline const ::mozilla::safebrowsing::ThreatEntrySet& FetchThreatListUpdatesResponse_ListUpdateResponse::additions(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.additions) + return additions_.Get(index); +} +inline ::mozilla::safebrowsing::ThreatEntrySet* FetchThreatListUpdatesResponse_ListUpdateResponse::mutable_additions(int index) { + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.additions) + return additions_.Mutable(index); +} +inline ::mozilla::safebrowsing::ThreatEntrySet* FetchThreatListUpdatesResponse_ListUpdateResponse::add_additions() { + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.additions) + return additions_.Add(); +} +inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntrySet >& +FetchThreatListUpdatesResponse_ListUpdateResponse::additions() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.additions) + return additions_; +} +inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntrySet >* +FetchThreatListUpdatesResponse_ListUpdateResponse::mutable_additions() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.additions) + return &additions_; +} + +// repeated .mozilla.safebrowsing.ThreatEntrySet removals = 6; +inline int FetchThreatListUpdatesResponse_ListUpdateResponse::removals_size() const { + return removals_.size(); +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::clear_removals() { + removals_.Clear(); +} +inline const ::mozilla::safebrowsing::ThreatEntrySet& FetchThreatListUpdatesResponse_ListUpdateResponse::removals(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.removals) + return removals_.Get(index); +} +inline ::mozilla::safebrowsing::ThreatEntrySet* FetchThreatListUpdatesResponse_ListUpdateResponse::mutable_removals(int index) { + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.removals) + return removals_.Mutable(index); +} +inline ::mozilla::safebrowsing::ThreatEntrySet* FetchThreatListUpdatesResponse_ListUpdateResponse::add_removals() { + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.removals) + return removals_.Add(); +} +inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntrySet >& +FetchThreatListUpdatesResponse_ListUpdateResponse::removals() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.removals) + return removals_; +} +inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntrySet >* +FetchThreatListUpdatesResponse_ListUpdateResponse::mutable_removals() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.removals) + return &removals_; +} + +// optional bytes new_client_state = 7; +inline bool FetchThreatListUpdatesResponse_ListUpdateResponse::has_new_client_state() const { + return (_has_bits_[0] & 0x00000040u) != 0; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_has_new_client_state() { + _has_bits_[0] |= 0x00000040u; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::clear_has_new_client_state() { + _has_bits_[0] &= ~0x00000040u; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::clear_new_client_state() { + if (new_client_state_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + new_client_state_->clear(); + } + clear_has_new_client_state(); +} +inline const ::std::string& FetchThreatListUpdatesResponse_ListUpdateResponse::new_client_state() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.new_client_state) + return *new_client_state_; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_new_client_state(const ::std::string& value) { + set_has_new_client_state(); + if (new_client_state_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + new_client_state_ = new ::std::string; + } + new_client_state_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.new_client_state) +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_new_client_state(const char* value) { + set_has_new_client_state(); + if (new_client_state_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + new_client_state_ = new ::std::string; + } + new_client_state_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.new_client_state) +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_new_client_state(const void* value, size_t size) { + set_has_new_client_state(); + if (new_client_state_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + new_client_state_ = new ::std::string; + } + new_client_state_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.new_client_state) +} +inline ::std::string* FetchThreatListUpdatesResponse_ListUpdateResponse::mutable_new_client_state() { + set_has_new_client_state(); + if (new_client_state_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + new_client_state_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.new_client_state) + return new_client_state_; +} +inline ::std::string* FetchThreatListUpdatesResponse_ListUpdateResponse::release_new_client_state() { + clear_has_new_client_state(); + if (new_client_state_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = new_client_state_; + new_client_state_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_allocated_new_client_state(::std::string* new_client_state) { + if (new_client_state_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete new_client_state_; + } + if (new_client_state) { + set_has_new_client_state(); + new_client_state_ = new_client_state; + } else { + clear_has_new_client_state(); + new_client_state_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.new_client_state) +} + +// optional .mozilla.safebrowsing.Checksum checksum = 8; +inline bool FetchThreatListUpdatesResponse_ListUpdateResponse::has_checksum() const { + return (_has_bits_[0] & 0x00000080u) != 0; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_has_checksum() { + _has_bits_[0] |= 0x00000080u; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::clear_has_checksum() { + _has_bits_[0] &= ~0x00000080u; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::clear_checksum() { + if (checksum_ != NULL) checksum_->::mozilla::safebrowsing::Checksum::Clear(); + clear_has_checksum(); +} +inline const ::mozilla::safebrowsing::Checksum& FetchThreatListUpdatesResponse_ListUpdateResponse::checksum() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.checksum) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return checksum_ != NULL ? *checksum_ : *default_instance().checksum_; +#else + return checksum_ != NULL ? *checksum_ : *default_instance_->checksum_; +#endif +} +inline ::mozilla::safebrowsing::Checksum* FetchThreatListUpdatesResponse_ListUpdateResponse::mutable_checksum() { + set_has_checksum(); + if (checksum_ == NULL) checksum_ = new ::mozilla::safebrowsing::Checksum; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.checksum) + return checksum_; +} +inline ::mozilla::safebrowsing::Checksum* FetchThreatListUpdatesResponse_ListUpdateResponse::release_checksum() { + clear_has_checksum(); + ::mozilla::safebrowsing::Checksum* temp = checksum_; + checksum_ = NULL; + return temp; +} +inline void FetchThreatListUpdatesResponse_ListUpdateResponse::set_allocated_checksum(::mozilla::safebrowsing::Checksum* checksum) { + delete checksum_; + checksum_ = checksum; + if (checksum) { + set_has_checksum(); + } else { + clear_has_checksum(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse.checksum) +} + +// ------------------------------------------------------------------- + +// FetchThreatListUpdatesResponse + +// repeated .mozilla.safebrowsing.FetchThreatListUpdatesResponse.ListUpdateResponse list_update_responses = 1; +inline int FetchThreatListUpdatesResponse::list_update_responses_size() const { + return list_update_responses_.size(); +} +inline void FetchThreatListUpdatesResponse::clear_list_update_responses() { + list_update_responses_.Clear(); +} +inline const ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse& FetchThreatListUpdatesResponse::list_update_responses(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesResponse.list_update_responses) + return list_update_responses_.Get(index); +} +inline ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse* FetchThreatListUpdatesResponse::mutable_list_update_responses(int index) { + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FetchThreatListUpdatesResponse.list_update_responses) + return list_update_responses_.Mutable(index); +} +inline ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse* FetchThreatListUpdatesResponse::add_list_update_responses() { + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.FetchThreatListUpdatesResponse.list_update_responses) + return list_update_responses_.Add(); +} +inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse >& +FetchThreatListUpdatesResponse::list_update_responses() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.FetchThreatListUpdatesResponse.list_update_responses) + return list_update_responses_; +} +inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::FetchThreatListUpdatesResponse_ListUpdateResponse >* +FetchThreatListUpdatesResponse::mutable_list_update_responses() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.FetchThreatListUpdatesResponse.list_update_responses) + return &list_update_responses_; +} + +// optional .mozilla.safebrowsing.Duration minimum_wait_duration = 2; +inline bool FetchThreatListUpdatesResponse::has_minimum_wait_duration() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void FetchThreatListUpdatesResponse::set_has_minimum_wait_duration() { + _has_bits_[0] |= 0x00000002u; +} +inline void FetchThreatListUpdatesResponse::clear_has_minimum_wait_duration() { + _has_bits_[0] &= ~0x00000002u; +} +inline void FetchThreatListUpdatesResponse::clear_minimum_wait_duration() { + if (minimum_wait_duration_ != NULL) minimum_wait_duration_->::mozilla::safebrowsing::Duration::Clear(); + clear_has_minimum_wait_duration(); +} +inline const ::mozilla::safebrowsing::Duration& FetchThreatListUpdatesResponse::minimum_wait_duration() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FetchThreatListUpdatesResponse.minimum_wait_duration) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return minimum_wait_duration_ != NULL ? *minimum_wait_duration_ : *default_instance().minimum_wait_duration_; +#else + return minimum_wait_duration_ != NULL ? *minimum_wait_duration_ : *default_instance_->minimum_wait_duration_; +#endif +} +inline ::mozilla::safebrowsing::Duration* FetchThreatListUpdatesResponse::mutable_minimum_wait_duration() { + set_has_minimum_wait_duration(); + if (minimum_wait_duration_ == NULL) minimum_wait_duration_ = new ::mozilla::safebrowsing::Duration; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FetchThreatListUpdatesResponse.minimum_wait_duration) + return minimum_wait_duration_; +} +inline ::mozilla::safebrowsing::Duration* FetchThreatListUpdatesResponse::release_minimum_wait_duration() { + clear_has_minimum_wait_duration(); + ::mozilla::safebrowsing::Duration* temp = minimum_wait_duration_; + minimum_wait_duration_ = NULL; + return temp; +} +inline void FetchThreatListUpdatesResponse::set_allocated_minimum_wait_duration(::mozilla::safebrowsing::Duration* minimum_wait_duration) { + delete minimum_wait_duration_; + minimum_wait_duration_ = minimum_wait_duration; + if (minimum_wait_duration) { + set_has_minimum_wait_duration(); + } else { + clear_has_minimum_wait_duration(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.FetchThreatListUpdatesResponse.minimum_wait_duration) +} + +// ------------------------------------------------------------------- + +// FindFullHashesRequest + +// optional .mozilla.safebrowsing.ClientInfo client = 1; +inline bool FindFullHashesRequest::has_client() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void FindFullHashesRequest::set_has_client() { + _has_bits_[0] |= 0x00000001u; +} +inline void FindFullHashesRequest::clear_has_client() { + _has_bits_[0] &= ~0x00000001u; +} +inline void FindFullHashesRequest::clear_client() { + if (client_ != NULL) client_->::mozilla::safebrowsing::ClientInfo::Clear(); + clear_has_client(); +} +inline const ::mozilla::safebrowsing::ClientInfo& FindFullHashesRequest::client() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FindFullHashesRequest.client) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return client_ != NULL ? *client_ : *default_instance().client_; +#else + return client_ != NULL ? *client_ : *default_instance_->client_; +#endif +} +inline ::mozilla::safebrowsing::ClientInfo* FindFullHashesRequest::mutable_client() { + set_has_client(); + if (client_ == NULL) client_ = new ::mozilla::safebrowsing::ClientInfo; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FindFullHashesRequest.client) + return client_; +} +inline ::mozilla::safebrowsing::ClientInfo* FindFullHashesRequest::release_client() { + clear_has_client(); + ::mozilla::safebrowsing::ClientInfo* temp = client_; + client_ = NULL; + return temp; +} +inline void FindFullHashesRequest::set_allocated_client(::mozilla::safebrowsing::ClientInfo* client) { + delete client_; + client_ = client; + if (client) { + set_has_client(); + } else { + clear_has_client(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.FindFullHashesRequest.client) +} + +// repeated bytes client_states = 2; +inline int FindFullHashesRequest::client_states_size() const { + return client_states_.size(); +} +inline void FindFullHashesRequest::clear_client_states() { + client_states_.Clear(); +} +inline const ::std::string& FindFullHashesRequest::client_states(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FindFullHashesRequest.client_states) + return client_states_.Get(index); +} +inline ::std::string* FindFullHashesRequest::mutable_client_states(int index) { + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FindFullHashesRequest.client_states) + return client_states_.Mutable(index); +} +inline void FindFullHashesRequest::set_client_states(int index, const ::std::string& value) { + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.FindFullHashesRequest.client_states) + client_states_.Mutable(index)->assign(value); +} +inline void FindFullHashesRequest::set_client_states(int index, const char* value) { + client_states_.Mutable(index)->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.FindFullHashesRequest.client_states) +} +inline void FindFullHashesRequest::set_client_states(int index, const void* value, size_t size) { + client_states_.Mutable(index)->assign( + reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.FindFullHashesRequest.client_states) +} +inline ::std::string* FindFullHashesRequest::add_client_states() { + return client_states_.Add(); +} +inline void FindFullHashesRequest::add_client_states(const ::std::string& value) { + client_states_.Add()->assign(value); + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.FindFullHashesRequest.client_states) +} +inline void FindFullHashesRequest::add_client_states(const char* value) { + client_states_.Add()->assign(value); + // @@protoc_insertion_point(field_add_char:mozilla.safebrowsing.FindFullHashesRequest.client_states) +} +inline void FindFullHashesRequest::add_client_states(const void* value, size_t size) { + client_states_.Add()->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_add_pointer:mozilla.safebrowsing.FindFullHashesRequest.client_states) +} +inline const ::google::protobuf::RepeatedPtrField< ::std::string>& +FindFullHashesRequest::client_states() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.FindFullHashesRequest.client_states) + return client_states_; +} +inline ::google::protobuf::RepeatedPtrField< ::std::string>* +FindFullHashesRequest::mutable_client_states() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.FindFullHashesRequest.client_states) + return &client_states_; +} + +// optional .mozilla.safebrowsing.ThreatInfo threat_info = 3; +inline bool FindFullHashesRequest::has_threat_info() const { + return (_has_bits_[0] & 0x00000004u) != 0; +} +inline void FindFullHashesRequest::set_has_threat_info() { + _has_bits_[0] |= 0x00000004u; +} +inline void FindFullHashesRequest::clear_has_threat_info() { + _has_bits_[0] &= ~0x00000004u; +} +inline void FindFullHashesRequest::clear_threat_info() { + if (threat_info_ != NULL) threat_info_->::mozilla::safebrowsing::ThreatInfo::Clear(); + clear_has_threat_info(); +} +inline const ::mozilla::safebrowsing::ThreatInfo& FindFullHashesRequest::threat_info() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FindFullHashesRequest.threat_info) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return threat_info_ != NULL ? *threat_info_ : *default_instance().threat_info_; +#else + return threat_info_ != NULL ? *threat_info_ : *default_instance_->threat_info_; +#endif +} +inline ::mozilla::safebrowsing::ThreatInfo* FindFullHashesRequest::mutable_threat_info() { + set_has_threat_info(); + if (threat_info_ == NULL) threat_info_ = new ::mozilla::safebrowsing::ThreatInfo; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FindFullHashesRequest.threat_info) + return threat_info_; +} +inline ::mozilla::safebrowsing::ThreatInfo* FindFullHashesRequest::release_threat_info() { + clear_has_threat_info(); + ::mozilla::safebrowsing::ThreatInfo* temp = threat_info_; + threat_info_ = NULL; + return temp; +} +inline void FindFullHashesRequest::set_allocated_threat_info(::mozilla::safebrowsing::ThreatInfo* threat_info) { + delete threat_info_; + threat_info_ = threat_info; + if (threat_info) { + set_has_threat_info(); + } else { + clear_has_threat_info(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.FindFullHashesRequest.threat_info) +} + +// ------------------------------------------------------------------- + +// FindFullHashesResponse + +// repeated .mozilla.safebrowsing.ThreatMatch matches = 1; +inline int FindFullHashesResponse::matches_size() const { + return matches_.size(); +} +inline void FindFullHashesResponse::clear_matches() { + matches_.Clear(); +} +inline const ::mozilla::safebrowsing::ThreatMatch& FindFullHashesResponse::matches(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FindFullHashesResponse.matches) + return matches_.Get(index); +} +inline ::mozilla::safebrowsing::ThreatMatch* FindFullHashesResponse::mutable_matches(int index) { + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FindFullHashesResponse.matches) + return matches_.Mutable(index); +} +inline ::mozilla::safebrowsing::ThreatMatch* FindFullHashesResponse::add_matches() { + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.FindFullHashesResponse.matches) + return matches_.Add(); +} +inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatMatch >& +FindFullHashesResponse::matches() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.FindFullHashesResponse.matches) + return matches_; +} +inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatMatch >* +FindFullHashesResponse::mutable_matches() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.FindFullHashesResponse.matches) + return &matches_; +} + +// optional .mozilla.safebrowsing.Duration minimum_wait_duration = 2; +inline bool FindFullHashesResponse::has_minimum_wait_duration() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void FindFullHashesResponse::set_has_minimum_wait_duration() { + _has_bits_[0] |= 0x00000002u; +} +inline void FindFullHashesResponse::clear_has_minimum_wait_duration() { + _has_bits_[0] &= ~0x00000002u; +} +inline void FindFullHashesResponse::clear_minimum_wait_duration() { + if (minimum_wait_duration_ != NULL) minimum_wait_duration_->::mozilla::safebrowsing::Duration::Clear(); + clear_has_minimum_wait_duration(); +} +inline const ::mozilla::safebrowsing::Duration& FindFullHashesResponse::minimum_wait_duration() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FindFullHashesResponse.minimum_wait_duration) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return minimum_wait_duration_ != NULL ? *minimum_wait_duration_ : *default_instance().minimum_wait_duration_; +#else + return minimum_wait_duration_ != NULL ? *minimum_wait_duration_ : *default_instance_->minimum_wait_duration_; +#endif +} +inline ::mozilla::safebrowsing::Duration* FindFullHashesResponse::mutable_minimum_wait_duration() { + set_has_minimum_wait_duration(); + if (minimum_wait_duration_ == NULL) minimum_wait_duration_ = new ::mozilla::safebrowsing::Duration; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FindFullHashesResponse.minimum_wait_duration) + return minimum_wait_duration_; +} +inline ::mozilla::safebrowsing::Duration* FindFullHashesResponse::release_minimum_wait_duration() { + clear_has_minimum_wait_duration(); + ::mozilla::safebrowsing::Duration* temp = minimum_wait_duration_; + minimum_wait_duration_ = NULL; + return temp; +} +inline void FindFullHashesResponse::set_allocated_minimum_wait_duration(::mozilla::safebrowsing::Duration* minimum_wait_duration) { + delete minimum_wait_duration_; + minimum_wait_duration_ = minimum_wait_duration; + if (minimum_wait_duration) { + set_has_minimum_wait_duration(); + } else { + clear_has_minimum_wait_duration(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.FindFullHashesResponse.minimum_wait_duration) +} + +// optional .mozilla.safebrowsing.Duration negative_cache_duration = 3; +inline bool FindFullHashesResponse::has_negative_cache_duration() const { + return (_has_bits_[0] & 0x00000004u) != 0; +} +inline void FindFullHashesResponse::set_has_negative_cache_duration() { + _has_bits_[0] |= 0x00000004u; +} +inline void FindFullHashesResponse::clear_has_negative_cache_duration() { + _has_bits_[0] &= ~0x00000004u; +} +inline void FindFullHashesResponse::clear_negative_cache_duration() { + if (negative_cache_duration_ != NULL) negative_cache_duration_->::mozilla::safebrowsing::Duration::Clear(); + clear_has_negative_cache_duration(); +} +inline const ::mozilla::safebrowsing::Duration& FindFullHashesResponse::negative_cache_duration() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.FindFullHashesResponse.negative_cache_duration) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return negative_cache_duration_ != NULL ? *negative_cache_duration_ : *default_instance().negative_cache_duration_; +#else + return negative_cache_duration_ != NULL ? *negative_cache_duration_ : *default_instance_->negative_cache_duration_; +#endif +} +inline ::mozilla::safebrowsing::Duration* FindFullHashesResponse::mutable_negative_cache_duration() { + set_has_negative_cache_duration(); + if (negative_cache_duration_ == NULL) negative_cache_duration_ = new ::mozilla::safebrowsing::Duration; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.FindFullHashesResponse.negative_cache_duration) + return negative_cache_duration_; +} +inline ::mozilla::safebrowsing::Duration* FindFullHashesResponse::release_negative_cache_duration() { + clear_has_negative_cache_duration(); + ::mozilla::safebrowsing::Duration* temp = negative_cache_duration_; + negative_cache_duration_ = NULL; + return temp; +} +inline void FindFullHashesResponse::set_allocated_negative_cache_duration(::mozilla::safebrowsing::Duration* negative_cache_duration) { + delete negative_cache_duration_; + negative_cache_duration_ = negative_cache_duration; + if (negative_cache_duration) { + set_has_negative_cache_duration(); + } else { + clear_has_negative_cache_duration(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.FindFullHashesResponse.negative_cache_duration) +} + +// ------------------------------------------------------------------- + +// ThreatHit_ThreatSource + +// optional string url = 1; +inline bool ThreatHit_ThreatSource::has_url() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void ThreatHit_ThreatSource::set_has_url() { + _has_bits_[0] |= 0x00000001u; +} +inline void ThreatHit_ThreatSource::clear_has_url() { + _has_bits_[0] &= ~0x00000001u; +} +inline void ThreatHit_ThreatSource::clear_url() { + if (url_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + url_->clear(); + } + clear_has_url(); +} +inline const ::std::string& ThreatHit_ThreatSource::url() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatHit.ThreatSource.url) + return *url_; +} +inline void ThreatHit_ThreatSource::set_url(const ::std::string& value) { + set_has_url(); + if (url_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + url_ = new ::std::string; + } + url_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatHit.ThreatSource.url) +} +inline void ThreatHit_ThreatSource::set_url(const char* value) { + set_has_url(); + if (url_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + url_ = new ::std::string; + } + url_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.ThreatHit.ThreatSource.url) +} +inline void ThreatHit_ThreatSource::set_url(const char* value, size_t size) { + set_has_url(); + if (url_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + url_ = new ::std::string; + } + url_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.ThreatHit.ThreatSource.url) +} +inline ::std::string* ThreatHit_ThreatSource::mutable_url() { + set_has_url(); + if (url_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + url_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatHit.ThreatSource.url) + return url_; +} +inline ::std::string* ThreatHit_ThreatSource::release_url() { + clear_has_url(); + if (url_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = url_; + url_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void ThreatHit_ThreatSource::set_allocated_url(::std::string* url) { + if (url_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete url_; + } + if (url) { + set_has_url(); + url_ = url; + } else { + clear_has_url(); + url_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatHit.ThreatSource.url) +} + +// optional .mozilla.safebrowsing.ThreatHit.ThreatSourceType type = 2; +inline bool ThreatHit_ThreatSource::has_type() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void ThreatHit_ThreatSource::set_has_type() { + _has_bits_[0] |= 0x00000002u; +} +inline void ThreatHit_ThreatSource::clear_has_type() { + _has_bits_[0] &= ~0x00000002u; +} +inline void ThreatHit_ThreatSource::clear_type() { + type_ = 0; + clear_has_type(); +} +inline ::mozilla::safebrowsing::ThreatHit_ThreatSourceType ThreatHit_ThreatSource::type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatHit.ThreatSource.type) + return static_cast< ::mozilla::safebrowsing::ThreatHit_ThreatSourceType >(type_); +} +inline void ThreatHit_ThreatSource::set_type(::mozilla::safebrowsing::ThreatHit_ThreatSourceType value) { + assert(::mozilla::safebrowsing::ThreatHit_ThreatSourceType_IsValid(value)); + set_has_type(); + type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatHit.ThreatSource.type) +} + +// optional string remote_ip = 3; +inline bool ThreatHit_ThreatSource::has_remote_ip() const { + return (_has_bits_[0] & 0x00000004u) != 0; +} +inline void ThreatHit_ThreatSource::set_has_remote_ip() { + _has_bits_[0] |= 0x00000004u; +} +inline void ThreatHit_ThreatSource::clear_has_remote_ip() { + _has_bits_[0] &= ~0x00000004u; +} +inline void ThreatHit_ThreatSource::clear_remote_ip() { + if (remote_ip_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + remote_ip_->clear(); + } + clear_has_remote_ip(); +} +inline const ::std::string& ThreatHit_ThreatSource::remote_ip() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatHit.ThreatSource.remote_ip) + return *remote_ip_; +} +inline void ThreatHit_ThreatSource::set_remote_ip(const ::std::string& value) { + set_has_remote_ip(); + if (remote_ip_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + remote_ip_ = new ::std::string; + } + remote_ip_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatHit.ThreatSource.remote_ip) +} +inline void ThreatHit_ThreatSource::set_remote_ip(const char* value) { + set_has_remote_ip(); + if (remote_ip_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + remote_ip_ = new ::std::string; + } + remote_ip_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.ThreatHit.ThreatSource.remote_ip) +} +inline void ThreatHit_ThreatSource::set_remote_ip(const char* value, size_t size) { + set_has_remote_ip(); + if (remote_ip_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + remote_ip_ = new ::std::string; + } + remote_ip_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.ThreatHit.ThreatSource.remote_ip) +} +inline ::std::string* ThreatHit_ThreatSource::mutable_remote_ip() { + set_has_remote_ip(); + if (remote_ip_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + remote_ip_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatHit.ThreatSource.remote_ip) + return remote_ip_; +} +inline ::std::string* ThreatHit_ThreatSource::release_remote_ip() { + clear_has_remote_ip(); + if (remote_ip_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = remote_ip_; + remote_ip_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void ThreatHit_ThreatSource::set_allocated_remote_ip(::std::string* remote_ip) { + if (remote_ip_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete remote_ip_; + } + if (remote_ip) { + set_has_remote_ip(); + remote_ip_ = remote_ip; + } else { + clear_has_remote_ip(); + remote_ip_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatHit.ThreatSource.remote_ip) +} + +// optional string referrer = 4; +inline bool ThreatHit_ThreatSource::has_referrer() const { + return (_has_bits_[0] & 0x00000008u) != 0; +} +inline void ThreatHit_ThreatSource::set_has_referrer() { + _has_bits_[0] |= 0x00000008u; +} +inline void ThreatHit_ThreatSource::clear_has_referrer() { + _has_bits_[0] &= ~0x00000008u; +} +inline void ThreatHit_ThreatSource::clear_referrer() { + if (referrer_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + referrer_->clear(); + } + clear_has_referrer(); +} +inline const ::std::string& ThreatHit_ThreatSource::referrer() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatHit.ThreatSource.referrer) + return *referrer_; +} +inline void ThreatHit_ThreatSource::set_referrer(const ::std::string& value) { + set_has_referrer(); + if (referrer_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + referrer_ = new ::std::string; + } + referrer_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatHit.ThreatSource.referrer) +} +inline void ThreatHit_ThreatSource::set_referrer(const char* value) { + set_has_referrer(); + if (referrer_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + referrer_ = new ::std::string; + } + referrer_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.ThreatHit.ThreatSource.referrer) +} +inline void ThreatHit_ThreatSource::set_referrer(const char* value, size_t size) { + set_has_referrer(); + if (referrer_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + referrer_ = new ::std::string; + } + referrer_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.ThreatHit.ThreatSource.referrer) +} +inline ::std::string* ThreatHit_ThreatSource::mutable_referrer() { + set_has_referrer(); + if (referrer_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + referrer_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatHit.ThreatSource.referrer) + return referrer_; +} +inline ::std::string* ThreatHit_ThreatSource::release_referrer() { + clear_has_referrer(); + if (referrer_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = referrer_; + referrer_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void ThreatHit_ThreatSource::set_allocated_referrer(::std::string* referrer) { + if (referrer_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete referrer_; + } + if (referrer) { + set_has_referrer(); + referrer_ = referrer; + } else { + clear_has_referrer(); + referrer_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatHit.ThreatSource.referrer) +} + +// ------------------------------------------------------------------- + +// ThreatHit + +// optional .mozilla.safebrowsing.ThreatType threat_type = 1; +inline bool ThreatHit::has_threat_type() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void ThreatHit::set_has_threat_type() { + _has_bits_[0] |= 0x00000001u; +} +inline void ThreatHit::clear_has_threat_type() { + _has_bits_[0] &= ~0x00000001u; +} +inline void ThreatHit::clear_threat_type() { + threat_type_ = 0; + clear_has_threat_type(); +} +inline ::mozilla::safebrowsing::ThreatType ThreatHit::threat_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatHit.threat_type) + return static_cast< ::mozilla::safebrowsing::ThreatType >(threat_type_); +} +inline void ThreatHit::set_threat_type(::mozilla::safebrowsing::ThreatType value) { + assert(::mozilla::safebrowsing::ThreatType_IsValid(value)); + set_has_threat_type(); + threat_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatHit.threat_type) +} + +// optional .mozilla.safebrowsing.PlatformType platform_type = 2; +inline bool ThreatHit::has_platform_type() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void ThreatHit::set_has_platform_type() { + _has_bits_[0] |= 0x00000002u; +} +inline void ThreatHit::clear_has_platform_type() { + _has_bits_[0] &= ~0x00000002u; +} +inline void ThreatHit::clear_platform_type() { + platform_type_ = 0; + clear_has_platform_type(); +} +inline ::mozilla::safebrowsing::PlatformType ThreatHit::platform_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatHit.platform_type) + return static_cast< ::mozilla::safebrowsing::PlatformType >(platform_type_); +} +inline void ThreatHit::set_platform_type(::mozilla::safebrowsing::PlatformType value) { + assert(::mozilla::safebrowsing::PlatformType_IsValid(value)); + set_has_platform_type(); + platform_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatHit.platform_type) +} + +// optional .mozilla.safebrowsing.ThreatEntry entry = 3; +inline bool ThreatHit::has_entry() const { + return (_has_bits_[0] & 0x00000004u) != 0; +} +inline void ThreatHit::set_has_entry() { + _has_bits_[0] |= 0x00000004u; +} +inline void ThreatHit::clear_has_entry() { + _has_bits_[0] &= ~0x00000004u; +} +inline void ThreatHit::clear_entry() { + if (entry_ != NULL) entry_->::mozilla::safebrowsing::ThreatEntry::Clear(); + clear_has_entry(); +} +inline const ::mozilla::safebrowsing::ThreatEntry& ThreatHit::entry() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatHit.entry) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return entry_ != NULL ? *entry_ : *default_instance().entry_; +#else + return entry_ != NULL ? *entry_ : *default_instance_->entry_; +#endif +} +inline ::mozilla::safebrowsing::ThreatEntry* ThreatHit::mutable_entry() { + set_has_entry(); + if (entry_ == NULL) entry_ = new ::mozilla::safebrowsing::ThreatEntry; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatHit.entry) + return entry_; +} +inline ::mozilla::safebrowsing::ThreatEntry* ThreatHit::release_entry() { + clear_has_entry(); + ::mozilla::safebrowsing::ThreatEntry* temp = entry_; + entry_ = NULL; + return temp; +} +inline void ThreatHit::set_allocated_entry(::mozilla::safebrowsing::ThreatEntry* entry) { + delete entry_; + entry_ = entry; + if (entry) { + set_has_entry(); + } else { + clear_has_entry(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatHit.entry) +} + +// repeated .mozilla.safebrowsing.ThreatHit.ThreatSource resources = 4; +inline int ThreatHit::resources_size() const { + return resources_.size(); +} +inline void ThreatHit::clear_resources() { + resources_.Clear(); +} +inline const ::mozilla::safebrowsing::ThreatHit_ThreatSource& ThreatHit::resources(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatHit.resources) + return resources_.Get(index); +} +inline ::mozilla::safebrowsing::ThreatHit_ThreatSource* ThreatHit::mutable_resources(int index) { + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatHit.resources) + return resources_.Mutable(index); +} +inline ::mozilla::safebrowsing::ThreatHit_ThreatSource* ThreatHit::add_resources() { + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.ThreatHit.resources) + return resources_.Add(); +} +inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatHit_ThreatSource >& +ThreatHit::resources() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.ThreatHit.resources) + return resources_; +} +inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatHit_ThreatSource >* +ThreatHit::mutable_resources() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.ThreatHit.resources) + return &resources_; +} + +// ------------------------------------------------------------------- + +// ClientInfo + +// optional string client_id = 1; +inline bool ClientInfo::has_client_id() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void ClientInfo::set_has_client_id() { + _has_bits_[0] |= 0x00000001u; +} +inline void ClientInfo::clear_has_client_id() { + _has_bits_[0] &= ~0x00000001u; +} +inline void ClientInfo::clear_client_id() { + if (client_id_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + client_id_->clear(); + } + clear_has_client_id(); +} +inline const ::std::string& ClientInfo::client_id() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ClientInfo.client_id) + return *client_id_; +} +inline void ClientInfo::set_client_id(const ::std::string& value) { + set_has_client_id(); + if (client_id_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + client_id_ = new ::std::string; + } + client_id_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ClientInfo.client_id) +} +inline void ClientInfo::set_client_id(const char* value) { + set_has_client_id(); + if (client_id_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + client_id_ = new ::std::string; + } + client_id_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.ClientInfo.client_id) +} +inline void ClientInfo::set_client_id(const char* value, size_t size) { + set_has_client_id(); + if (client_id_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + client_id_ = new ::std::string; + } + client_id_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.ClientInfo.client_id) +} +inline ::std::string* ClientInfo::mutable_client_id() { + set_has_client_id(); + if (client_id_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + client_id_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ClientInfo.client_id) + return client_id_; +} +inline ::std::string* ClientInfo::release_client_id() { + clear_has_client_id(); + if (client_id_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = client_id_; + client_id_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void ClientInfo::set_allocated_client_id(::std::string* client_id) { + if (client_id_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete client_id_; + } + if (client_id) { + set_has_client_id(); + client_id_ = client_id; + } else { + clear_has_client_id(); + client_id_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ClientInfo.client_id) +} + +// optional string client_version = 2; +inline bool ClientInfo::has_client_version() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void ClientInfo::set_has_client_version() { + _has_bits_[0] |= 0x00000002u; +} +inline void ClientInfo::clear_has_client_version() { + _has_bits_[0] &= ~0x00000002u; +} +inline void ClientInfo::clear_client_version() { + if (client_version_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + client_version_->clear(); + } + clear_has_client_version(); +} +inline const ::std::string& ClientInfo::client_version() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ClientInfo.client_version) + return *client_version_; +} +inline void ClientInfo::set_client_version(const ::std::string& value) { + set_has_client_version(); + if (client_version_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + client_version_ = new ::std::string; + } + client_version_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ClientInfo.client_version) +} +inline void ClientInfo::set_client_version(const char* value) { + set_has_client_version(); + if (client_version_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + client_version_ = new ::std::string; + } + client_version_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.ClientInfo.client_version) +} +inline void ClientInfo::set_client_version(const char* value, size_t size) { + set_has_client_version(); + if (client_version_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + client_version_ = new ::std::string; + } + client_version_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.ClientInfo.client_version) +} +inline ::std::string* ClientInfo::mutable_client_version() { + set_has_client_version(); + if (client_version_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + client_version_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ClientInfo.client_version) + return client_version_; +} +inline ::std::string* ClientInfo::release_client_version() { + clear_has_client_version(); + if (client_version_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = client_version_; + client_version_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void ClientInfo::set_allocated_client_version(::std::string* client_version) { + if (client_version_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete client_version_; + } + if (client_version) { + set_has_client_version(); + client_version_ = client_version; + } else { + clear_has_client_version(); + client_version_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ClientInfo.client_version) +} + +// ------------------------------------------------------------------- + +// Checksum + +// optional bytes sha256 = 1; +inline bool Checksum::has_sha256() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void Checksum::set_has_sha256() { + _has_bits_[0] |= 0x00000001u; +} +inline void Checksum::clear_has_sha256() { + _has_bits_[0] &= ~0x00000001u; +} +inline void Checksum::clear_sha256() { + if (sha256_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + sha256_->clear(); + } + clear_has_sha256(); +} +inline const ::std::string& Checksum::sha256() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.Checksum.sha256) + return *sha256_; +} +inline void Checksum::set_sha256(const ::std::string& value) { + set_has_sha256(); + if (sha256_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + sha256_ = new ::std::string; + } + sha256_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.Checksum.sha256) +} +inline void Checksum::set_sha256(const char* value) { + set_has_sha256(); + if (sha256_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + sha256_ = new ::std::string; + } + sha256_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.Checksum.sha256) +} +inline void Checksum::set_sha256(const void* value, size_t size) { + set_has_sha256(); + if (sha256_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + sha256_ = new ::std::string; + } + sha256_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.Checksum.sha256) +} +inline ::std::string* Checksum::mutable_sha256() { + set_has_sha256(); + if (sha256_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + sha256_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.Checksum.sha256) + return sha256_; +} +inline ::std::string* Checksum::release_sha256() { + clear_has_sha256(); + if (sha256_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = sha256_; + sha256_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void Checksum::set_allocated_sha256(::std::string* sha256) { + if (sha256_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete sha256_; + } + if (sha256) { + set_has_sha256(); + sha256_ = sha256; + } else { + clear_has_sha256(); + sha256_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.Checksum.sha256) +} + +// ------------------------------------------------------------------- + +// ThreatEntry + +// optional bytes hash = 1; +inline bool ThreatEntry::has_hash() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void ThreatEntry::set_has_hash() { + _has_bits_[0] |= 0x00000001u; +} +inline void ThreatEntry::clear_has_hash() { + _has_bits_[0] &= ~0x00000001u; +} +inline void ThreatEntry::clear_hash() { + if (hash_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + hash_->clear(); + } + clear_has_hash(); +} +inline const ::std::string& ThreatEntry::hash() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatEntry.hash) + return *hash_; +} +inline void ThreatEntry::set_hash(const ::std::string& value) { + set_has_hash(); + if (hash_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + hash_ = new ::std::string; + } + hash_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatEntry.hash) +} +inline void ThreatEntry::set_hash(const char* value) { + set_has_hash(); + if (hash_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + hash_ = new ::std::string; + } + hash_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.ThreatEntry.hash) +} +inline void ThreatEntry::set_hash(const void* value, size_t size) { + set_has_hash(); + if (hash_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + hash_ = new ::std::string; + } + hash_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.ThreatEntry.hash) +} +inline ::std::string* ThreatEntry::mutable_hash() { + set_has_hash(); + if (hash_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + hash_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatEntry.hash) + return hash_; +} +inline ::std::string* ThreatEntry::release_hash() { + clear_has_hash(); + if (hash_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = hash_; + hash_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void ThreatEntry::set_allocated_hash(::std::string* hash) { + if (hash_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete hash_; + } + if (hash) { + set_has_hash(); + hash_ = hash; + } else { + clear_has_hash(); + hash_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatEntry.hash) +} + +// optional string url = 2; +inline bool ThreatEntry::has_url() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void ThreatEntry::set_has_url() { + _has_bits_[0] |= 0x00000002u; +} +inline void ThreatEntry::clear_has_url() { + _has_bits_[0] &= ~0x00000002u; +} +inline void ThreatEntry::clear_url() { + if (url_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + url_->clear(); + } + clear_has_url(); +} +inline const ::std::string& ThreatEntry::url() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatEntry.url) + return *url_; +} +inline void ThreatEntry::set_url(const ::std::string& value) { + set_has_url(); + if (url_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + url_ = new ::std::string; + } + url_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatEntry.url) +} +inline void ThreatEntry::set_url(const char* value) { + set_has_url(); + if (url_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + url_ = new ::std::string; + } + url_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.ThreatEntry.url) +} +inline void ThreatEntry::set_url(const char* value, size_t size) { + set_has_url(); + if (url_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + url_ = new ::std::string; + } + url_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.ThreatEntry.url) +} +inline ::std::string* ThreatEntry::mutable_url() { + set_has_url(); + if (url_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + url_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatEntry.url) + return url_; +} +inline ::std::string* ThreatEntry::release_url() { + clear_has_url(); + if (url_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = url_; + url_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void ThreatEntry::set_allocated_url(::std::string* url) { + if (url_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete url_; + } + if (url) { + set_has_url(); + url_ = url; + } else { + clear_has_url(); + url_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatEntry.url) +} + +// ------------------------------------------------------------------- + +// ThreatEntrySet + +// optional .mozilla.safebrowsing.CompressionType compression_type = 1; +inline bool ThreatEntrySet::has_compression_type() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void ThreatEntrySet::set_has_compression_type() { + _has_bits_[0] |= 0x00000001u; +} +inline void ThreatEntrySet::clear_has_compression_type() { + _has_bits_[0] &= ~0x00000001u; +} +inline void ThreatEntrySet::clear_compression_type() { + compression_type_ = 0; + clear_has_compression_type(); +} +inline ::mozilla::safebrowsing::CompressionType ThreatEntrySet::compression_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatEntrySet.compression_type) + return static_cast< ::mozilla::safebrowsing::CompressionType >(compression_type_); +} +inline void ThreatEntrySet::set_compression_type(::mozilla::safebrowsing::CompressionType value) { + assert(::mozilla::safebrowsing::CompressionType_IsValid(value)); + set_has_compression_type(); + compression_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatEntrySet.compression_type) +} + +// optional .mozilla.safebrowsing.RawHashes raw_hashes = 2; +inline bool ThreatEntrySet::has_raw_hashes() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void ThreatEntrySet::set_has_raw_hashes() { + _has_bits_[0] |= 0x00000002u; +} +inline void ThreatEntrySet::clear_has_raw_hashes() { + _has_bits_[0] &= ~0x00000002u; +} +inline void ThreatEntrySet::clear_raw_hashes() { + if (raw_hashes_ != NULL) raw_hashes_->::mozilla::safebrowsing::RawHashes::Clear(); + clear_has_raw_hashes(); +} +inline const ::mozilla::safebrowsing::RawHashes& ThreatEntrySet::raw_hashes() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatEntrySet.raw_hashes) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return raw_hashes_ != NULL ? *raw_hashes_ : *default_instance().raw_hashes_; +#else + return raw_hashes_ != NULL ? *raw_hashes_ : *default_instance_->raw_hashes_; +#endif +} +inline ::mozilla::safebrowsing::RawHashes* ThreatEntrySet::mutable_raw_hashes() { + set_has_raw_hashes(); + if (raw_hashes_ == NULL) raw_hashes_ = new ::mozilla::safebrowsing::RawHashes; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatEntrySet.raw_hashes) + return raw_hashes_; +} +inline ::mozilla::safebrowsing::RawHashes* ThreatEntrySet::release_raw_hashes() { + clear_has_raw_hashes(); + ::mozilla::safebrowsing::RawHashes* temp = raw_hashes_; + raw_hashes_ = NULL; + return temp; +} +inline void ThreatEntrySet::set_allocated_raw_hashes(::mozilla::safebrowsing::RawHashes* raw_hashes) { + delete raw_hashes_; + raw_hashes_ = raw_hashes; + if (raw_hashes) { + set_has_raw_hashes(); + } else { + clear_has_raw_hashes(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatEntrySet.raw_hashes) +} + +// optional .mozilla.safebrowsing.RawIndices raw_indices = 3; +inline bool ThreatEntrySet::has_raw_indices() const { + return (_has_bits_[0] & 0x00000004u) != 0; +} +inline void ThreatEntrySet::set_has_raw_indices() { + _has_bits_[0] |= 0x00000004u; +} +inline void ThreatEntrySet::clear_has_raw_indices() { + _has_bits_[0] &= ~0x00000004u; +} +inline void ThreatEntrySet::clear_raw_indices() { + if (raw_indices_ != NULL) raw_indices_->::mozilla::safebrowsing::RawIndices::Clear(); + clear_has_raw_indices(); +} +inline const ::mozilla::safebrowsing::RawIndices& ThreatEntrySet::raw_indices() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatEntrySet.raw_indices) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return raw_indices_ != NULL ? *raw_indices_ : *default_instance().raw_indices_; +#else + return raw_indices_ != NULL ? *raw_indices_ : *default_instance_->raw_indices_; +#endif +} +inline ::mozilla::safebrowsing::RawIndices* ThreatEntrySet::mutable_raw_indices() { + set_has_raw_indices(); + if (raw_indices_ == NULL) raw_indices_ = new ::mozilla::safebrowsing::RawIndices; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatEntrySet.raw_indices) + return raw_indices_; +} +inline ::mozilla::safebrowsing::RawIndices* ThreatEntrySet::release_raw_indices() { + clear_has_raw_indices(); + ::mozilla::safebrowsing::RawIndices* temp = raw_indices_; + raw_indices_ = NULL; + return temp; +} +inline void ThreatEntrySet::set_allocated_raw_indices(::mozilla::safebrowsing::RawIndices* raw_indices) { + delete raw_indices_; + raw_indices_ = raw_indices; + if (raw_indices) { + set_has_raw_indices(); + } else { + clear_has_raw_indices(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatEntrySet.raw_indices) +} + +// optional .mozilla.safebrowsing.RiceDeltaEncoding rice_hashes = 4; +inline bool ThreatEntrySet::has_rice_hashes() const { + return (_has_bits_[0] & 0x00000008u) != 0; +} +inline void ThreatEntrySet::set_has_rice_hashes() { + _has_bits_[0] |= 0x00000008u; +} +inline void ThreatEntrySet::clear_has_rice_hashes() { + _has_bits_[0] &= ~0x00000008u; +} +inline void ThreatEntrySet::clear_rice_hashes() { + if (rice_hashes_ != NULL) rice_hashes_->::mozilla::safebrowsing::RiceDeltaEncoding::Clear(); + clear_has_rice_hashes(); +} +inline const ::mozilla::safebrowsing::RiceDeltaEncoding& ThreatEntrySet::rice_hashes() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatEntrySet.rice_hashes) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return rice_hashes_ != NULL ? *rice_hashes_ : *default_instance().rice_hashes_; +#else + return rice_hashes_ != NULL ? *rice_hashes_ : *default_instance_->rice_hashes_; +#endif +} +inline ::mozilla::safebrowsing::RiceDeltaEncoding* ThreatEntrySet::mutable_rice_hashes() { + set_has_rice_hashes(); + if (rice_hashes_ == NULL) rice_hashes_ = new ::mozilla::safebrowsing::RiceDeltaEncoding; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatEntrySet.rice_hashes) + return rice_hashes_; +} +inline ::mozilla::safebrowsing::RiceDeltaEncoding* ThreatEntrySet::release_rice_hashes() { + clear_has_rice_hashes(); + ::mozilla::safebrowsing::RiceDeltaEncoding* temp = rice_hashes_; + rice_hashes_ = NULL; + return temp; +} +inline void ThreatEntrySet::set_allocated_rice_hashes(::mozilla::safebrowsing::RiceDeltaEncoding* rice_hashes) { + delete rice_hashes_; + rice_hashes_ = rice_hashes; + if (rice_hashes) { + set_has_rice_hashes(); + } else { + clear_has_rice_hashes(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatEntrySet.rice_hashes) +} + +// optional .mozilla.safebrowsing.RiceDeltaEncoding rice_indices = 5; +inline bool ThreatEntrySet::has_rice_indices() const { + return (_has_bits_[0] & 0x00000010u) != 0; +} +inline void ThreatEntrySet::set_has_rice_indices() { + _has_bits_[0] |= 0x00000010u; +} +inline void ThreatEntrySet::clear_has_rice_indices() { + _has_bits_[0] &= ~0x00000010u; +} +inline void ThreatEntrySet::clear_rice_indices() { + if (rice_indices_ != NULL) rice_indices_->::mozilla::safebrowsing::RiceDeltaEncoding::Clear(); + clear_has_rice_indices(); +} +inline const ::mozilla::safebrowsing::RiceDeltaEncoding& ThreatEntrySet::rice_indices() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatEntrySet.rice_indices) +#ifdef GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER + return rice_indices_ != NULL ? *rice_indices_ : *default_instance().rice_indices_; +#else + return rice_indices_ != NULL ? *rice_indices_ : *default_instance_->rice_indices_; +#endif +} +inline ::mozilla::safebrowsing::RiceDeltaEncoding* ThreatEntrySet::mutable_rice_indices() { + set_has_rice_indices(); + if (rice_indices_ == NULL) rice_indices_ = new ::mozilla::safebrowsing::RiceDeltaEncoding; + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatEntrySet.rice_indices) + return rice_indices_; +} +inline ::mozilla::safebrowsing::RiceDeltaEncoding* ThreatEntrySet::release_rice_indices() { + clear_has_rice_indices(); + ::mozilla::safebrowsing::RiceDeltaEncoding* temp = rice_indices_; + rice_indices_ = NULL; + return temp; +} +inline void ThreatEntrySet::set_allocated_rice_indices(::mozilla::safebrowsing::RiceDeltaEncoding* rice_indices) { + delete rice_indices_; + rice_indices_ = rice_indices; + if (rice_indices) { + set_has_rice_indices(); + } else { + clear_has_rice_indices(); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatEntrySet.rice_indices) +} + +// ------------------------------------------------------------------- + +// RawIndices + +// repeated int32 indices = 1; +inline int RawIndices::indices_size() const { + return indices_.size(); +} +inline void RawIndices::clear_indices() { + indices_.Clear(); +} +inline ::google::protobuf::int32 RawIndices::indices(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.RawIndices.indices) + return indices_.Get(index); +} +inline void RawIndices::set_indices(int index, ::google::protobuf::int32 value) { + indices_.Set(index, value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.RawIndices.indices) +} +inline void RawIndices::add_indices(::google::protobuf::int32 value) { + indices_.Add(value); + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.RawIndices.indices) +} +inline const ::google::protobuf::RepeatedField< ::google::protobuf::int32 >& +RawIndices::indices() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.RawIndices.indices) + return indices_; +} +inline ::google::protobuf::RepeatedField< ::google::protobuf::int32 >* +RawIndices::mutable_indices() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.RawIndices.indices) + return &indices_; +} + +// ------------------------------------------------------------------- + +// RawHashes + +// optional int32 prefix_size = 1; +inline bool RawHashes::has_prefix_size() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void RawHashes::set_has_prefix_size() { + _has_bits_[0] |= 0x00000001u; +} +inline void RawHashes::clear_has_prefix_size() { + _has_bits_[0] &= ~0x00000001u; +} +inline void RawHashes::clear_prefix_size() { + prefix_size_ = 0; + clear_has_prefix_size(); +} +inline ::google::protobuf::int32 RawHashes::prefix_size() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.RawHashes.prefix_size) + return prefix_size_; +} +inline void RawHashes::set_prefix_size(::google::protobuf::int32 value) { + set_has_prefix_size(); + prefix_size_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.RawHashes.prefix_size) +} + +// optional bytes raw_hashes = 2; +inline bool RawHashes::has_raw_hashes() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void RawHashes::set_has_raw_hashes() { + _has_bits_[0] |= 0x00000002u; +} +inline void RawHashes::clear_has_raw_hashes() { + _has_bits_[0] &= ~0x00000002u; +} +inline void RawHashes::clear_raw_hashes() { + if (raw_hashes_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + raw_hashes_->clear(); + } + clear_has_raw_hashes(); +} +inline const ::std::string& RawHashes::raw_hashes() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.RawHashes.raw_hashes) + return *raw_hashes_; +} +inline void RawHashes::set_raw_hashes(const ::std::string& value) { + set_has_raw_hashes(); + if (raw_hashes_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + raw_hashes_ = new ::std::string; + } + raw_hashes_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.RawHashes.raw_hashes) +} +inline void RawHashes::set_raw_hashes(const char* value) { + set_has_raw_hashes(); + if (raw_hashes_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + raw_hashes_ = new ::std::string; + } + raw_hashes_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.RawHashes.raw_hashes) +} +inline void RawHashes::set_raw_hashes(const void* value, size_t size) { + set_has_raw_hashes(); + if (raw_hashes_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + raw_hashes_ = new ::std::string; + } + raw_hashes_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.RawHashes.raw_hashes) +} +inline ::std::string* RawHashes::mutable_raw_hashes() { + set_has_raw_hashes(); + if (raw_hashes_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + raw_hashes_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.RawHashes.raw_hashes) + return raw_hashes_; +} +inline ::std::string* RawHashes::release_raw_hashes() { + clear_has_raw_hashes(); + if (raw_hashes_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = raw_hashes_; + raw_hashes_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void RawHashes::set_allocated_raw_hashes(::std::string* raw_hashes) { + if (raw_hashes_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete raw_hashes_; + } + if (raw_hashes) { + set_has_raw_hashes(); + raw_hashes_ = raw_hashes; + } else { + clear_has_raw_hashes(); + raw_hashes_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.RawHashes.raw_hashes) +} + +// ------------------------------------------------------------------- + +// RiceDeltaEncoding + +// optional int64 first_value = 1; +inline bool RiceDeltaEncoding::has_first_value() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void RiceDeltaEncoding::set_has_first_value() { + _has_bits_[0] |= 0x00000001u; +} +inline void RiceDeltaEncoding::clear_has_first_value() { + _has_bits_[0] &= ~0x00000001u; +} +inline void RiceDeltaEncoding::clear_first_value() { + first_value_ = GOOGLE_LONGLONG(0); + clear_has_first_value(); +} +inline ::google::protobuf::int64 RiceDeltaEncoding::first_value() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.RiceDeltaEncoding.first_value) + return first_value_; +} +inline void RiceDeltaEncoding::set_first_value(::google::protobuf::int64 value) { + set_has_first_value(); + first_value_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.RiceDeltaEncoding.first_value) +} + +// optional int32 rice_parameter = 2; +inline bool RiceDeltaEncoding::has_rice_parameter() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void RiceDeltaEncoding::set_has_rice_parameter() { + _has_bits_[0] |= 0x00000002u; +} +inline void RiceDeltaEncoding::clear_has_rice_parameter() { + _has_bits_[0] &= ~0x00000002u; +} +inline void RiceDeltaEncoding::clear_rice_parameter() { + rice_parameter_ = 0; + clear_has_rice_parameter(); +} +inline ::google::protobuf::int32 RiceDeltaEncoding::rice_parameter() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.RiceDeltaEncoding.rice_parameter) + return rice_parameter_; +} +inline void RiceDeltaEncoding::set_rice_parameter(::google::protobuf::int32 value) { + set_has_rice_parameter(); + rice_parameter_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.RiceDeltaEncoding.rice_parameter) +} + +// optional int32 num_entries = 3; +inline bool RiceDeltaEncoding::has_num_entries() const { + return (_has_bits_[0] & 0x00000004u) != 0; +} +inline void RiceDeltaEncoding::set_has_num_entries() { + _has_bits_[0] |= 0x00000004u; +} +inline void RiceDeltaEncoding::clear_has_num_entries() { + _has_bits_[0] &= ~0x00000004u; +} +inline void RiceDeltaEncoding::clear_num_entries() { + num_entries_ = 0; + clear_has_num_entries(); +} +inline ::google::protobuf::int32 RiceDeltaEncoding::num_entries() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.RiceDeltaEncoding.num_entries) + return num_entries_; +} +inline void RiceDeltaEncoding::set_num_entries(::google::protobuf::int32 value) { + set_has_num_entries(); + num_entries_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.RiceDeltaEncoding.num_entries) +} + +// optional bytes encoded_data = 4; +inline bool RiceDeltaEncoding::has_encoded_data() const { + return (_has_bits_[0] & 0x00000008u) != 0; +} +inline void RiceDeltaEncoding::set_has_encoded_data() { + _has_bits_[0] |= 0x00000008u; +} +inline void RiceDeltaEncoding::clear_has_encoded_data() { + _has_bits_[0] &= ~0x00000008u; +} +inline void RiceDeltaEncoding::clear_encoded_data() { + if (encoded_data_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + encoded_data_->clear(); + } + clear_has_encoded_data(); +} +inline const ::std::string& RiceDeltaEncoding::encoded_data() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.RiceDeltaEncoding.encoded_data) + return *encoded_data_; +} +inline void RiceDeltaEncoding::set_encoded_data(const ::std::string& value) { + set_has_encoded_data(); + if (encoded_data_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + encoded_data_ = new ::std::string; + } + encoded_data_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.RiceDeltaEncoding.encoded_data) +} +inline void RiceDeltaEncoding::set_encoded_data(const char* value) { + set_has_encoded_data(); + if (encoded_data_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + encoded_data_ = new ::std::string; + } + encoded_data_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.RiceDeltaEncoding.encoded_data) +} +inline void RiceDeltaEncoding::set_encoded_data(const void* value, size_t size) { + set_has_encoded_data(); + if (encoded_data_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + encoded_data_ = new ::std::string; + } + encoded_data_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.RiceDeltaEncoding.encoded_data) +} +inline ::std::string* RiceDeltaEncoding::mutable_encoded_data() { + set_has_encoded_data(); + if (encoded_data_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + encoded_data_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.RiceDeltaEncoding.encoded_data) + return encoded_data_; +} +inline ::std::string* RiceDeltaEncoding::release_encoded_data() { + clear_has_encoded_data(); + if (encoded_data_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = encoded_data_; + encoded_data_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void RiceDeltaEncoding::set_allocated_encoded_data(::std::string* encoded_data) { + if (encoded_data_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete encoded_data_; + } + if (encoded_data) { + set_has_encoded_data(); + encoded_data_ = encoded_data; + } else { + clear_has_encoded_data(); + encoded_data_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.RiceDeltaEncoding.encoded_data) +} + +// ------------------------------------------------------------------- + +// ThreatEntryMetadata_MetadataEntry + +// optional bytes key = 1; +inline bool ThreatEntryMetadata_MetadataEntry::has_key() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void ThreatEntryMetadata_MetadataEntry::set_has_key() { + _has_bits_[0] |= 0x00000001u; +} +inline void ThreatEntryMetadata_MetadataEntry::clear_has_key() { + _has_bits_[0] &= ~0x00000001u; +} +inline void ThreatEntryMetadata_MetadataEntry::clear_key() { + if (key_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + key_->clear(); + } + clear_has_key(); +} +inline const ::std::string& ThreatEntryMetadata_MetadataEntry::key() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry.key) + return *key_; +} +inline void ThreatEntryMetadata_MetadataEntry::set_key(const ::std::string& value) { + set_has_key(); + if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + key_ = new ::std::string; + } + key_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry.key) +} +inline void ThreatEntryMetadata_MetadataEntry::set_key(const char* value) { + set_has_key(); + if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + key_ = new ::std::string; + } + key_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry.key) +} +inline void ThreatEntryMetadata_MetadataEntry::set_key(const void* value, size_t size) { + set_has_key(); + if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + key_ = new ::std::string; + } + key_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry.key) +} +inline ::std::string* ThreatEntryMetadata_MetadataEntry::mutable_key() { + set_has_key(); + if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + key_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry.key) + return key_; +} +inline ::std::string* ThreatEntryMetadata_MetadataEntry::release_key() { + clear_has_key(); + if (key_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = key_; + key_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void ThreatEntryMetadata_MetadataEntry::set_allocated_key(::std::string* key) { + if (key_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete key_; + } + if (key) { + set_has_key(); + key_ = key; + } else { + clear_has_key(); + key_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry.key) +} + +// optional bytes value = 2; +inline bool ThreatEntryMetadata_MetadataEntry::has_value() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void ThreatEntryMetadata_MetadataEntry::set_has_value() { + _has_bits_[0] |= 0x00000002u; +} +inline void ThreatEntryMetadata_MetadataEntry::clear_has_value() { + _has_bits_[0] &= ~0x00000002u; +} +inline void ThreatEntryMetadata_MetadataEntry::clear_value() { + if (value_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + value_->clear(); + } + clear_has_value(); +} +inline const ::std::string& ThreatEntryMetadata_MetadataEntry::value() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry.value) + return *value_; +} +inline void ThreatEntryMetadata_MetadataEntry::set_value(const ::std::string& value) { + set_has_value(); + if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + value_ = new ::std::string; + } + value_->assign(value); + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry.value) +} +inline void ThreatEntryMetadata_MetadataEntry::set_value(const char* value) { + set_has_value(); + if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + value_ = new ::std::string; + } + value_->assign(value); + // @@protoc_insertion_point(field_set_char:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry.value) +} +inline void ThreatEntryMetadata_MetadataEntry::set_value(const void* value, size_t size) { + set_has_value(); + if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + value_ = new ::std::string; + } + value_->assign(reinterpret_cast<const char*>(value), size); + // @@protoc_insertion_point(field_set_pointer:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry.value) +} +inline ::std::string* ThreatEntryMetadata_MetadataEntry::mutable_value() { + set_has_value(); + if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + value_ = new ::std::string; + } + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry.value) + return value_; +} +inline ::std::string* ThreatEntryMetadata_MetadataEntry::release_value() { + clear_has_value(); + if (value_ == &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + return NULL; + } else { + ::std::string* temp = value_; + value_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return temp; + } +} +inline void ThreatEntryMetadata_MetadataEntry::set_allocated_value(::std::string* value) { + if (value_ != &::google::protobuf::internal::GetEmptyStringAlreadyInited()) { + delete value_; + } + if (value) { + set_has_value(); + value_ = value; + } else { + clear_has_value(); + value_ = const_cast< ::std::string*>(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + // @@protoc_insertion_point(field_set_allocated:mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry.value) +} + +// ------------------------------------------------------------------- + +// ThreatEntryMetadata + +// repeated .mozilla.safebrowsing.ThreatEntryMetadata.MetadataEntry entries = 1; +inline int ThreatEntryMetadata::entries_size() const { + return entries_.size(); +} +inline void ThreatEntryMetadata::clear_entries() { + entries_.Clear(); +} +inline const ::mozilla::safebrowsing::ThreatEntryMetadata_MetadataEntry& ThreatEntryMetadata::entries(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatEntryMetadata.entries) + return entries_.Get(index); +} +inline ::mozilla::safebrowsing::ThreatEntryMetadata_MetadataEntry* ThreatEntryMetadata::mutable_entries(int index) { + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ThreatEntryMetadata.entries) + return entries_.Mutable(index); +} +inline ::mozilla::safebrowsing::ThreatEntryMetadata_MetadataEntry* ThreatEntryMetadata::add_entries() { + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.ThreatEntryMetadata.entries) + return entries_.Add(); +} +inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntryMetadata_MetadataEntry >& +ThreatEntryMetadata::entries() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.ThreatEntryMetadata.entries) + return entries_; +} +inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatEntryMetadata_MetadataEntry >* +ThreatEntryMetadata::mutable_entries() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.ThreatEntryMetadata.entries) + return &entries_; +} + +// ------------------------------------------------------------------- + +// ThreatListDescriptor + +// optional .mozilla.safebrowsing.ThreatType threat_type = 1; +inline bool ThreatListDescriptor::has_threat_type() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void ThreatListDescriptor::set_has_threat_type() { + _has_bits_[0] |= 0x00000001u; +} +inline void ThreatListDescriptor::clear_has_threat_type() { + _has_bits_[0] &= ~0x00000001u; +} +inline void ThreatListDescriptor::clear_threat_type() { + threat_type_ = 0; + clear_has_threat_type(); +} +inline ::mozilla::safebrowsing::ThreatType ThreatListDescriptor::threat_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatListDescriptor.threat_type) + return static_cast< ::mozilla::safebrowsing::ThreatType >(threat_type_); +} +inline void ThreatListDescriptor::set_threat_type(::mozilla::safebrowsing::ThreatType value) { + assert(::mozilla::safebrowsing::ThreatType_IsValid(value)); + set_has_threat_type(); + threat_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatListDescriptor.threat_type) +} + +// optional .mozilla.safebrowsing.PlatformType platform_type = 2; +inline bool ThreatListDescriptor::has_platform_type() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void ThreatListDescriptor::set_has_platform_type() { + _has_bits_[0] |= 0x00000002u; +} +inline void ThreatListDescriptor::clear_has_platform_type() { + _has_bits_[0] &= ~0x00000002u; +} +inline void ThreatListDescriptor::clear_platform_type() { + platform_type_ = 0; + clear_has_platform_type(); +} +inline ::mozilla::safebrowsing::PlatformType ThreatListDescriptor::platform_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatListDescriptor.platform_type) + return static_cast< ::mozilla::safebrowsing::PlatformType >(platform_type_); +} +inline void ThreatListDescriptor::set_platform_type(::mozilla::safebrowsing::PlatformType value) { + assert(::mozilla::safebrowsing::PlatformType_IsValid(value)); + set_has_platform_type(); + platform_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatListDescriptor.platform_type) +} + +// optional .mozilla.safebrowsing.ThreatEntryType threat_entry_type = 3; +inline bool ThreatListDescriptor::has_threat_entry_type() const { + return (_has_bits_[0] & 0x00000004u) != 0; +} +inline void ThreatListDescriptor::set_has_threat_entry_type() { + _has_bits_[0] |= 0x00000004u; +} +inline void ThreatListDescriptor::clear_has_threat_entry_type() { + _has_bits_[0] &= ~0x00000004u; +} +inline void ThreatListDescriptor::clear_threat_entry_type() { + threat_entry_type_ = 0; + clear_has_threat_entry_type(); +} +inline ::mozilla::safebrowsing::ThreatEntryType ThreatListDescriptor::threat_entry_type() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ThreatListDescriptor.threat_entry_type) + return static_cast< ::mozilla::safebrowsing::ThreatEntryType >(threat_entry_type_); +} +inline void ThreatListDescriptor::set_threat_entry_type(::mozilla::safebrowsing::ThreatEntryType value) { + assert(::mozilla::safebrowsing::ThreatEntryType_IsValid(value)); + set_has_threat_entry_type(); + threat_entry_type_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.ThreatListDescriptor.threat_entry_type) +} + +// ------------------------------------------------------------------- + +// ListThreatListsResponse + +// repeated .mozilla.safebrowsing.ThreatListDescriptor threat_lists = 1; +inline int ListThreatListsResponse::threat_lists_size() const { + return threat_lists_.size(); +} +inline void ListThreatListsResponse::clear_threat_lists() { + threat_lists_.Clear(); +} +inline const ::mozilla::safebrowsing::ThreatListDescriptor& ListThreatListsResponse::threat_lists(int index) const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.ListThreatListsResponse.threat_lists) + return threat_lists_.Get(index); +} +inline ::mozilla::safebrowsing::ThreatListDescriptor* ListThreatListsResponse::mutable_threat_lists(int index) { + // @@protoc_insertion_point(field_mutable:mozilla.safebrowsing.ListThreatListsResponse.threat_lists) + return threat_lists_.Mutable(index); +} +inline ::mozilla::safebrowsing::ThreatListDescriptor* ListThreatListsResponse::add_threat_lists() { + // @@protoc_insertion_point(field_add:mozilla.safebrowsing.ListThreatListsResponse.threat_lists) + return threat_lists_.Add(); +} +inline const ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatListDescriptor >& +ListThreatListsResponse::threat_lists() const { + // @@protoc_insertion_point(field_list:mozilla.safebrowsing.ListThreatListsResponse.threat_lists) + return threat_lists_; +} +inline ::google::protobuf::RepeatedPtrField< ::mozilla::safebrowsing::ThreatListDescriptor >* +ListThreatListsResponse::mutable_threat_lists() { + // @@protoc_insertion_point(field_mutable_list:mozilla.safebrowsing.ListThreatListsResponse.threat_lists) + return &threat_lists_; +} + +// ------------------------------------------------------------------- + +// Duration + +// optional int64 seconds = 1; +inline bool Duration::has_seconds() const { + return (_has_bits_[0] & 0x00000001u) != 0; +} +inline void Duration::set_has_seconds() { + _has_bits_[0] |= 0x00000001u; +} +inline void Duration::clear_has_seconds() { + _has_bits_[0] &= ~0x00000001u; +} +inline void Duration::clear_seconds() { + seconds_ = GOOGLE_LONGLONG(0); + clear_has_seconds(); +} +inline ::google::protobuf::int64 Duration::seconds() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.Duration.seconds) + return seconds_; +} +inline void Duration::set_seconds(::google::protobuf::int64 value) { + set_has_seconds(); + seconds_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.Duration.seconds) +} + +// optional int32 nanos = 2; +inline bool Duration::has_nanos() const { + return (_has_bits_[0] & 0x00000002u) != 0; +} +inline void Duration::set_has_nanos() { + _has_bits_[0] |= 0x00000002u; +} +inline void Duration::clear_has_nanos() { + _has_bits_[0] &= ~0x00000002u; +} +inline void Duration::clear_nanos() { + nanos_ = 0; + clear_has_nanos(); +} +inline ::google::protobuf::int32 Duration::nanos() const { + // @@protoc_insertion_point(field_get:mozilla.safebrowsing.Duration.nanos) + return nanos_; +} +inline void Duration::set_nanos(::google::protobuf::int32 value) { + set_has_nanos(); + nanos_ = value; + // @@protoc_insertion_point(field_set:mozilla.safebrowsing.Duration.nanos) +} + + +// @@protoc_insertion_point(namespace_scope) + +} // namespace safebrowsing +} // namespace mozilla + +// @@protoc_insertion_point(global_scope) + +#endif // PROTOBUF_safebrowsing_2eproto__INCLUDED diff --git a/toolkit/components/url-classifier/tests/UrlClassifierTestUtils.jsm b/toolkit/components/url-classifier/tests/UrlClassifierTestUtils.jsm new file mode 100644 index 000000000..615769473 --- /dev/null +++ b/toolkit/components/url-classifier/tests/UrlClassifierTestUtils.jsm @@ -0,0 +1,98 @@ +"use strict"; + +this.EXPORTED_SYMBOLS = ["UrlClassifierTestUtils"]; + +const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; + +const TRACKING_TABLE_NAME = "mochitest-track-simple"; +const TRACKING_TABLE_PREF = "urlclassifier.trackingTable"; +const WHITELIST_TABLE_NAME = "mochitest-trackwhite-simple"; +const WHITELIST_TABLE_PREF = "urlclassifier.trackingWhitelistTable"; + +Cu.import("resource://gre/modules/Services.jsm"); + +this.UrlClassifierTestUtils = { + + addTestTrackers() { + // Add some URLs to the tracking databases + let trackingURL1 = "tracking.example.com/"; + let trackingURL2 = "itisatracker.org/"; + let trackingURL3 = "trackertest.org/"; + let whitelistedURL = "itisatrap.org/?resource=itisatracker.org"; + + let trackingUpdate = + "n:1000\ni:" + TRACKING_TABLE_NAME + "\nad:3\n" + + "a:1:32:" + trackingURL1.length + "\n" + + trackingURL1 + "\n" + + "a:2:32:" + trackingURL2.length + "\n" + + trackingURL2 + "\n" + + "a:3:32:" + trackingURL3.length + "\n" + + trackingURL3 + "\n"; + let whitelistUpdate = + "n:1000\ni:" + WHITELIST_TABLE_NAME + "\nad:1\n" + + "a:1:32:" + whitelistedURL.length + "\n" + + whitelistedURL + "\n"; + + var tables = [ + { + pref: TRACKING_TABLE_PREF, + name: TRACKING_TABLE_NAME, + update: trackingUpdate + }, + { + pref: WHITELIST_TABLE_PREF, + name: WHITELIST_TABLE_NAME, + update: whitelistUpdate + } + ]; + + return this.useTestDatabase(tables); + }, + + cleanupTestTrackers() { + Services.prefs.clearUserPref(TRACKING_TABLE_PREF); + Services.prefs.clearUserPref(WHITELIST_TABLE_PREF); + }, + + /** + * Add some entries to a test tracking protection database, and resets + * back to the default database after the test ends. + * + * @return {Promise} + */ + useTestDatabase(tables) { + for (var table of tables) { + Services.prefs.setCharPref(table.pref, table.name); + } + + return new Promise((resolve, reject) => { + let dbService = Cc["@mozilla.org/url-classifier/dbservice;1"]. + getService(Ci.nsIUrlClassifierDBService); + let listener = { + QueryInterface: iid => { + if (iid.equals(Ci.nsISupports) || + iid.equals(Ci.nsIUrlClassifierUpdateObserver)) + return listener; + + throw Cr.NS_ERROR_NO_INTERFACE; + }, + updateUrlRequested: url => { }, + streamFinished: status => { }, + updateError: errorCode => { + reject("Couldn't update classifier."); + }, + updateSuccess: requestedTimeout => { + resolve(); + } + }; + + for (var table of tables) { + dbService.beginUpdate(listener, table.name, ""); + dbService.beginStream("", ""); + dbService.updateStream(table.update); + dbService.finishStream(); + dbService.finishUpdate(); + } + }); + }, +}; diff --git a/toolkit/components/url-classifier/tests/gtest/Common.cpp b/toolkit/components/url-classifier/tests/gtest/Common.cpp new file mode 100644 index 000000000..b5f024b38 --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/Common.cpp @@ -0,0 +1,78 @@ +#include "Common.h" +#include "HashStore.h" +#include "Classifier.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsTArray.h" +#include "nsIThread.h" +#include "nsThreadUtils.h" +#include "nsUrlClassifierUtils.h" + +using namespace mozilla; +using namespace mozilla::safebrowsing; + +template<typename Function> +void RunTestInNewThread(Function&& aFunction) { + nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction(mozilla::Forward<Function>(aFunction)); + nsCOMPtr<nsIThread> testingThread; + nsresult rv = NS_NewThread(getter_AddRefs(testingThread), r); + ASSERT_EQ(rv, NS_OK); + testingThread->Shutdown(); +} + +already_AddRefed<nsIFile> +GetFile(const nsTArray<nsString>& path) +{ + nsCOMPtr<nsIFile> file; + nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file)); + if (NS_WARN_IF(NS_FAILED(rv))) { + return nullptr; + } + + for (uint32_t i = 0; i < path.Length(); i++) { + file->Append(path[i]); + } + return file.forget(); +} + +void ApplyUpdate(nsTArray<TableUpdate*>& updates) +{ + nsCOMPtr<nsIFile> file; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file)); + + UniquePtr<Classifier> classifier(new Classifier()); + classifier->Open(*file); + + { + // Force nsIUrlClassifierUtils loading on main thread + // because nsIUrlClassifierDBService will not run in advance + // in gtest. + nsresult rv; + nsCOMPtr<nsIUrlClassifierUtils> dummy = + do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID, &rv); + ASSERT_TRUE(NS_SUCCEEDED(rv)); + } + + RunTestInNewThread([&] () -> void { + classifier->ApplyUpdates(&updates); + }); +} + +void ApplyUpdate(TableUpdate* update) +{ + nsTArray<TableUpdate*> updates = { update }; + ApplyUpdate(updates); +} + +void +PrefixArrayToPrefixStringMap(const nsTArray<nsCString>& prefixArray, + PrefixStringMap& out) +{ + out.Clear(); + + for (uint32_t i = 0; i < prefixArray.Length(); i++) { + const nsCString& prefix = prefixArray[i]; + nsCString* prefixString = out.LookupOrAdd(prefix.Length()); + prefixString->Append(prefix.BeginReading(), prefix.Length()); + } +} + diff --git a/toolkit/components/url-classifier/tests/gtest/Common.h b/toolkit/components/url-classifier/tests/gtest/Common.h new file mode 100644 index 000000000..c9a9cdf7e --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/Common.h @@ -0,0 +1,26 @@ +#include "HashStore.h" +#include "nsIFile.h" +#include "nsTArray.h" +#include "gtest/gtest.h" + +using namespace mozilla; +using namespace mozilla::safebrowsing; + +template<typename Function> +void RunTestInNewThread(Function&& aFunction); + +// Return nsIFile with root directory - NS_APP_USER_PROFILE_50_DIR +// Sub-directories are passed in path argument. +already_AddRefed<nsIFile> +GetFile(const nsTArray<nsString>& path); + +// ApplyUpdate will call |ApplyUpdates| of Classifier within a new thread +void ApplyUpdate(nsTArray<TableUpdate*>& updates); + +void ApplyUpdate(TableUpdate* update); + +// This function converts lexigraphic-sorted prefixes to a hashtable +// which key is prefix size and value is concatenated prefix string. +void PrefixArrayToPrefixStringMap(const nsTArray<nsCString>& prefixArray, + PrefixStringMap& out); + diff --git a/toolkit/components/url-classifier/tests/gtest/TestChunkSet.cpp b/toolkit/components/url-classifier/tests/gtest/TestChunkSet.cpp new file mode 100644 index 000000000..dba2fc2c1 --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/TestChunkSet.cpp @@ -0,0 +1,279 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <stdio.h> +#include <stdlib.h> +#include <set> + +#include "gtest/gtest.h" +#include "ChunkSet.h" +#include "mozilla/ArrayUtils.h" + +TEST(UrlClassifierChunkSet, Empty) +{ + mozilla::safebrowsing::ChunkSet chunkSet; + mozilla::safebrowsing::ChunkSet removeSet; + + removeSet.Set(0); + + ASSERT_FALSE(chunkSet.Has(0)); + ASSERT_FALSE(chunkSet.Has(1)); + ASSERT_TRUE(chunkSet.Remove(removeSet) == NS_OK); + ASSERT_TRUE(chunkSet.Length() == 0); + + chunkSet.Set(0); + + ASSERT_TRUE(chunkSet.Has(0)); + ASSERT_TRUE(chunkSet.Length() == 1); + ASSERT_TRUE(chunkSet.Remove(removeSet) == NS_OK); + ASSERT_FALSE(chunkSet.Has(0)); + ASSERT_TRUE(chunkSet.Length() == 0); +} + +TEST(UrlClassifierChunkSet, Main) +{ + static int testVals[] = {2, 1, 5, 6, 8, 7, 14, 10, 12, 13}; + + mozilla::safebrowsing::ChunkSet chunkSet; + + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) { + chunkSet.Set(testVals[i]); + } + + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) { + ASSERT_TRUE(chunkSet.Has(testVals[i])); + } + + ASSERT_FALSE(chunkSet.Has(3)); + ASSERT_FALSE(chunkSet.Has(4)); + ASSERT_FALSE(chunkSet.Has(9)); + ASSERT_FALSE(chunkSet.Has(11)); + + ASSERT_TRUE(chunkSet.Length() == MOZ_ARRAY_LENGTH(testVals)); +} + +TEST(UrlClassifierChunkSet, Merge) +{ + static int testVals[] = {2, 1, 5, 6, 8, 7, 14, 10, 12, 13}; + static int mergeVals[] = {9, 3, 4, 20, 14, 16}; + + mozilla::safebrowsing::ChunkSet chunkSet; + mozilla::safebrowsing::ChunkSet mergeSet; + + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) { + chunkSet.Set(testVals[i]); + } + + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) { + mergeSet.Set(mergeVals[i]); + } + + chunkSet.Merge(mergeSet); + + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) { + ASSERT_TRUE(chunkSet.Has(testVals[i])); + } + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) { + ASSERT_TRUE(chunkSet.Has(mergeVals[i])); + } + + // -1 because 14 is duplicated in both sets + ASSERT_TRUE(chunkSet.Length() == + MOZ_ARRAY_LENGTH(testVals) + MOZ_ARRAY_LENGTH(mergeVals) - 1); + + ASSERT_FALSE(chunkSet.Has(11)); + ASSERT_FALSE(chunkSet.Has(15)); + ASSERT_FALSE(chunkSet.Has(17)); + ASSERT_FALSE(chunkSet.Has(18)); + ASSERT_FALSE(chunkSet.Has(19)); +} + +TEST(UrlClassifierChunkSet, Merge2) +{ + static int testVals[] = {2, 1, 5, 6, 8, 7, 14, 10, 12, 13}; + static int mergeVals[] = {9, 3, 4, 20, 14, 16}; + static int mergeVals2[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + + mozilla::safebrowsing::ChunkSet chunkSet; + mozilla::safebrowsing::ChunkSet mergeSet; + mozilla::safebrowsing::ChunkSet mergeSet2; + + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) { + chunkSet.Set(testVals[i]); + } + + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) { + mergeSet.Set(mergeVals[i]); + } + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals2); i++) { + mergeSet2.Set(mergeVals2[i]); + } + + chunkSet.Merge(mergeSet); + chunkSet.Merge(mergeSet2); + + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) { + ASSERT_TRUE(chunkSet.Has(testVals[i])); + } + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) { + ASSERT_TRUE(chunkSet.Has(mergeVals[i])); + } + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals2); i++) { + ASSERT_TRUE(chunkSet.Has(mergeVals2[i])); + } + + ASSERT_FALSE(chunkSet.Has(15)); + ASSERT_FALSE(chunkSet.Has(17)); + ASSERT_FALSE(chunkSet.Has(18)); + ASSERT_FALSE(chunkSet.Has(19)); +} + +TEST(UrlClassifierChunkSet, Stress) +{ + mozilla::safebrowsing::ChunkSet chunkSet; + mozilla::safebrowsing::ChunkSet mergeSet; + std::set<int> refSet; + std::set<int> refMergeSet; + static const int TEST_ITERS = 7000; + static const int REMOVE_ITERS = 3000; + static const int TEST_RANGE = 10000; + + // Construction by Set + for (int i = 0; i < TEST_ITERS; i++) { + int chunk = rand() % TEST_RANGE; + chunkSet.Set(chunk); + refSet.insert(chunk); + } + + // Same elements as reference set + for (auto it = refSet.begin(); it != refSet.end(); ++it) { + ASSERT_TRUE(chunkSet.Has(*it)); + } + + // Hole punching via Remove + for (int i = 0; i < REMOVE_ITERS; i++) { + int chunk = rand() % TEST_RANGE; + mozilla::safebrowsing::ChunkSet helpChunk; + helpChunk.Set(chunk); + + chunkSet.Remove(helpChunk); + refSet.erase(chunk); + + ASSERT_FALSE(chunkSet.Has(chunk)); + } + + // Should have chunks present in reference set + // Should not have chunks absent in reference set + for (int it = 0; it < TEST_RANGE; ++it) { + auto found = refSet.find(it); + if (chunkSet.Has(it)) { + ASSERT_FALSE(found == refSet.end()); + } else { + ASSERT_TRUE(found == refSet.end()); + } + } + + // Construct set to merge with + for (int i = 0; i < TEST_ITERS; i++) { + int chunk = rand() % TEST_RANGE; + mergeSet.Set(chunk); + refMergeSet.insert(chunk); + } + + // Merge set constructed correctly + for (auto it = refMergeSet.begin(); it != refMergeSet.end(); ++it) { + ASSERT_TRUE(mergeSet.Has(*it)); + } + + mozilla::safebrowsing::ChunkSet origSet; + origSet = chunkSet; + + chunkSet.Merge(mergeSet); + refSet.insert(refMergeSet.begin(), refMergeSet.end()); + + // Check for presence of elements from both source + // Should not have chunks absent in reference set + for (int it = 0; it < TEST_RANGE; ++it) { + auto found = refSet.find(it); + if (chunkSet.Has(it)) { + ASSERT_FALSE(found == refSet.end()); + } else { + ASSERT_TRUE(found == refSet.end()); + } + } + + // Unmerge + chunkSet.Remove(origSet); + for (int it = 0; it < TEST_RANGE; ++it) { + if (origSet.Has(it)) { + ASSERT_FALSE(chunkSet.Has(it)); + } else if (mergeSet.Has(it)) { + ASSERT_TRUE(chunkSet.Has(it)); + } + } +} + +TEST(UrlClassifierChunkSet, RemoveClear) +{ + static int testVals[] = {2, 1, 5, 6, 8, 7, 14, 10, 12, 13}; + static int mergeVals[] = {3, 4, 9, 16, 20}; + + mozilla::safebrowsing::ChunkSet chunkSet; + mozilla::safebrowsing::ChunkSet mergeSet; + mozilla::safebrowsing::ChunkSet removeSet; + + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) { + chunkSet.Set(testVals[i]); + removeSet.Set(testVals[i]); + } + + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) { + mergeSet.Set(mergeVals[i]); + } + + ASSERT_TRUE(chunkSet.Merge(mergeSet) == NS_OK); + ASSERT_TRUE(chunkSet.Remove(removeSet) == NS_OK); + + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) { + ASSERT_TRUE(chunkSet.Has(mergeVals[i])); + } + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) { + ASSERT_FALSE(chunkSet.Has(testVals[i])); + } + + chunkSet.Clear(); + + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) { + ASSERT_FALSE(chunkSet.Has(mergeVals[i])); + } +} + +TEST(UrlClassifierChunkSet, Serialize) +{ + static int testVals[] = {2, 1, 5, 6, 8, 7, 14, 10, 12, 13}; + static int mergeVals[] = {3, 4, 9, 16, 20}; + + mozilla::safebrowsing::ChunkSet chunkSet; + mozilla::safebrowsing::ChunkSet mergeSet; + + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) { + chunkSet.Set(testVals[i]); + } + + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) { + mergeSet.Set(mergeVals[i]); + } + + chunkSet.Merge(mergeSet); + + nsAutoCString mergeResult; + chunkSet.Serialize(mergeResult); + + printf("mergeResult: %s\n", mergeResult.get()); + + nsAutoCString expected(NS_LITERAL_CSTRING("1-10,12-14,16,20")); + + ASSERT_TRUE(mergeResult.Equals(expected)); +} diff --git a/toolkit/components/url-classifier/tests/gtest/TestFailUpdate.cpp b/toolkit/components/url-classifier/tests/gtest/TestFailUpdate.cpp new file mode 100644 index 000000000..bdb9eebb0 --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/TestFailUpdate.cpp @@ -0,0 +1,97 @@ +#include "HashStore.h" +#include "nsPrintfCString.h" +#include "string.h" +#include "gtest/gtest.h" +#include "mozilla/Unused.h" + +using namespace mozilla; +using namespace mozilla::safebrowsing; + +static const char* kFilesInV2[] = {".pset", ".sbstore"}; +static const char* kFilesInV4[] = {".pset", ".metadata"}; + +#define V2_TABLE "gtest-malware-simple" +#define V4_TABLE1 "goog-malware-proto" +#define V4_TABLE2 "goog-phish-proto" + +#define ROOT_DIR NS_LITERAL_STRING("safebrowsing") +#define SB_FILE(x, y) NS_ConvertUTF8toUTF16(nsPrintfCString("%s%s",x, y)) + +template<typename T, size_t N> +void CheckFileExist(const char* table, const T (&files)[N], bool expectExists) +{ + for (uint32_t i = 0; i < N; i++) { + // This is just a quick way to know if this is v4 table + NS_ConvertUTF8toUTF16 SUB_DIR(strstr(table, "-proto") ? "google4" : ""); + nsCOMPtr<nsIFile> file = + GetFile(nsTArray<nsString> { ROOT_DIR, SUB_DIR, SB_FILE(table, files[i]) }); + + bool exists; + file->Exists(&exists); + + nsAutoCString path; + file->GetNativePath(path); + ASSERT_EQ(expectExists, exists) << path.get(); + } +} + +TEST(FailUpdate, CheckTableReset) +{ + const bool FULL_UPDATE = true; + const bool PARTIAL_UPDATE = false; + + // Apply V2 update + { + auto update = new TableUpdateV2(NS_LITERAL_CSTRING(V2_TABLE)); + Unused << update->NewAddChunk(1); + + ApplyUpdate(update); + + // A successful V2 update should create .pset & .sbstore files + CheckFileExist(V2_TABLE, kFilesInV2, true); + } + + // Helper function to generate table update data + auto func = [](TableUpdateV4* update, bool full, const char* str) { + update->SetFullUpdate(full); + std::string prefix(str); + update->NewPrefixes(prefix.length(), prefix); + }; + + // Apply V4 update for table1 + { + auto update = new TableUpdateV4(NS_LITERAL_CSTRING(V4_TABLE1)); + func(update, FULL_UPDATE, "test_prefix"); + + ApplyUpdate(update); + + // A successful V4 update should create .pset & .metadata files + CheckFileExist(V4_TABLE1, kFilesInV4, true); + } + + // Apply V4 update for table2 + { + auto update = new TableUpdateV4(NS_LITERAL_CSTRING(V4_TABLE2)); + func(update, FULL_UPDATE, "test_prefix"); + + ApplyUpdate(update); + + CheckFileExist(V4_TABLE2, kFilesInV4, true); + } + + // Apply V4 update with the same prefix in previous full udpate + // This should cause an update error. + { + auto update = new TableUpdateV4(NS_LITERAL_CSTRING(V4_TABLE1)); + func(update, PARTIAL_UPDATE, "test_prefix"); + + ApplyUpdate(update); + + // A fail update should remove files for that table + CheckFileExist(V4_TABLE1, kFilesInV4, false); + + // A fail update should NOT remove files for the other tables + CheckFileExist(V2_TABLE, kFilesInV2, true); + CheckFileExist(V4_TABLE2, kFilesInV4, true); + } +} diff --git a/toolkit/components/url-classifier/tests/gtest/TestLookupCacheV4.cpp b/toolkit/components/url-classifier/tests/gtest/TestLookupCacheV4.cpp new file mode 100644 index 000000000..00525f704 --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/TestLookupCacheV4.cpp @@ -0,0 +1,88 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "LookupCacheV4.h" +#include "Common.h" + +#define GTEST_SAFEBROWSING_DIR NS_LITERAL_CSTRING("safebrowsing") +#define GTEST_TABLE NS_LITERAL_CSTRING("gtest-malware-proto") + +typedef nsCString _Fragment; +typedef nsTArray<nsCString> _PrefixArray; + +// Generate a hash prefix from string +static const nsCString +GeneratePrefix(const _Fragment& aFragment, uint8_t aLength) +{ + Completion complete; + nsCOMPtr<nsICryptoHash> cryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID); + complete.FromPlaintext(aFragment, cryptoHash); + + nsCString hash; + hash.Assign((const char *)complete.buf, aLength); + return hash; +} + +static UniquePtr<LookupCacheV4> +SetupLookupCacheV4(const _PrefixArray& prefixArray) +{ + nsCOMPtr<nsIFile> file; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file)); + + file->AppendNative(GTEST_SAFEBROWSING_DIR); + + UniquePtr<LookupCacheV4> cache = MakeUnique<LookupCacheV4>(GTEST_TABLE, EmptyCString(), file); + nsresult rv = cache->Init(); + EXPECT_EQ(rv, NS_OK); + + PrefixStringMap map; + PrefixArrayToPrefixStringMap(prefixArray, map); + rv = cache->Build(map); + EXPECT_EQ(rv, NS_OK); + + return Move(cache); +} + +void +TestHasPrefix(const _Fragment& aFragment, bool aExpectedHas, bool aExpectedComplete) +{ + _PrefixArray array = { GeneratePrefix(_Fragment("bravo.com/"), 32), + GeneratePrefix(_Fragment("browsing.com/"), 8), + GeneratePrefix(_Fragment("gound.com/"), 5), + GeneratePrefix(_Fragment("small.com/"), 4) + }; + + RunTestInNewThread([&] () -> void { + UniquePtr<LookupCache> cache = SetupLookupCacheV4(array); + + Completion lookupHash; + nsCOMPtr<nsICryptoHash> cryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID); + lookupHash.FromPlaintext(aFragment, cryptoHash); + + bool has, complete; + nsresult rv = cache->Has(lookupHash, &has, &complete); + + EXPECT_EQ(rv, NS_OK); + EXPECT_EQ(has, aExpectedHas); + EXPECT_EQ(complete, aExpectedComplete); + + cache->ClearAll(); + }); + +} + +TEST(LookupCacheV4, HasComplete) +{ + TestHasPrefix(_Fragment("bravo.com/"), true, true); +} + +TEST(LookupCacheV4, HasPrefix) +{ + TestHasPrefix(_Fragment("browsing.com/"), true, false); +} + +TEST(LookupCacheV4, Nomatch) +{ + TestHasPrefix(_Fragment("nomatch.com/"), false, false); +} diff --git a/toolkit/components/url-classifier/tests/gtest/TestPerProviderDirectory.cpp b/toolkit/components/url-classifier/tests/gtest/TestPerProviderDirectory.cpp new file mode 100644 index 000000000..72ff08a1e --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/TestPerProviderDirectory.cpp @@ -0,0 +1,98 @@ +#include "LookupCache.h" +#include "LookupCacheV4.h" +#include "HashStore.h" +#include "gtest/gtest.h" +#include "nsAppDirectoryServiceDefs.h" + +namespace mozilla { +namespace safebrowsing { + +class PerProviderDirectoryTestUtils { +public: + template<typename T> + static nsIFile* InspectStoreDirectory(const T& aT) + { + return aT.mStoreDirectory; + } +}; + +} // end of namespace safebrowsing +} // end of namespace mozilla + +using namespace mozilla; +using namespace mozilla::safebrowsing; + +template<typename T> +void VerifyPrivateStorePath(const char* aTableName, + const char* aProvider, + nsIFile* aRootDir, + bool aUsePerProviderStore) +{ + nsString rootStorePath; + nsresult rv = aRootDir->GetPath(rootStorePath); + EXPECT_EQ(rv, NS_OK); + + T target(nsCString(aTableName), nsCString(aProvider), aRootDir); + + nsIFile* privateStoreDirectory = + PerProviderDirectoryTestUtils::InspectStoreDirectory(target); + + nsString privateStorePath; + rv = privateStoreDirectory->GetPath(privateStorePath); + ASSERT_EQ(rv, NS_OK); + + nsString expectedPrivateStorePath = rootStorePath; + + if (aUsePerProviderStore) { + // Use API to append "provider" to the root directoy path + nsCOMPtr<nsIFile> expectedPrivateStoreDir; + rv = aRootDir->Clone(getter_AddRefs(expectedPrivateStoreDir)); + ASSERT_EQ(rv, NS_OK); + + expectedPrivateStoreDir->AppendNative(nsCString(aProvider)); + rv = expectedPrivateStoreDir->GetPath(expectedPrivateStorePath); + ASSERT_EQ(rv, NS_OK); + } + + printf("table: %s\nprovider: %s\nroot path: %s\nprivate path: %s\n\n", + aTableName, + aProvider, + NS_ConvertUTF16toUTF8(rootStorePath).get(), + NS_ConvertUTF16toUTF8(privateStorePath).get()); + + ASSERT_TRUE(privateStorePath == expectedPrivateStorePath); +} + +TEST(PerProviderDirectory, LookupCache) +{ + RunTestInNewThread([] () -> void { + nsCOMPtr<nsIFile> rootDir; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(rootDir)); + + // For V2 tables (NOT ending with '-proto'), root directory should be + // used as the private store. + VerifyPrivateStorePath<LookupCacheV2>("goog-phish-shavar", "google", rootDir, false); + + // For V4 tables, if provider is found, use per-provider subdirectory; + // If not found, use root directory. + VerifyPrivateStorePath<LookupCacheV4>("goog-noprovider-proto", "", rootDir, false); + VerifyPrivateStorePath<LookupCacheV4>("goog-phish-proto", "google4", rootDir, true); + }); +} + +TEST(PerProviderDirectory, HashStore) +{ + RunTestInNewThread([] () -> void { + nsCOMPtr<nsIFile> rootDir; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(rootDir)); + + // For V2 tables (NOT ending with '-proto'), root directory should be + // used as the private store. + VerifyPrivateStorePath<HashStore>("goog-phish-shavar", "google", rootDir, false); + + // For V4 tables, if provider is found, use per-provider subdirectory; + // If not found, use root directory. + VerifyPrivateStorePath<HashStore>("goog-noprovider-proto", "", rootDir, false); + VerifyPrivateStorePath<HashStore>("goog-phish-proto", "google4", rootDir, true); + }); +} diff --git a/toolkit/components/url-classifier/tests/gtest/TestProtocolParser.cpp b/toolkit/components/url-classifier/tests/gtest/TestProtocolParser.cpp new file mode 100644 index 000000000..ea6ffb5e6 --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/TestProtocolParser.cpp @@ -0,0 +1,159 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +#include "gtest/gtest.h" +#include "ProtocolParser.h" +#include "mozilla/EndianUtils.h" + +using namespace mozilla; +using namespace mozilla::safebrowsing; + +typedef FetchThreatListUpdatesResponse_ListUpdateResponse ListUpdateResponse; + +static bool +InitUpdateResponse(ListUpdateResponse* aUpdateResponse, + ThreatType aThreatType, + const nsACString& aState, + const nsACString& aChecksum, + bool isFullUpdate, + const nsTArray<uint32_t>& aFixedLengthPrefixes, + bool aDoPrefixEncoding); + +static void +DumpBinary(const nsACString& aBinary); + +TEST(ProtocolParser, UpdateWait) +{ + // Top level response which contains a list of update response + // for different lists. + FetchThreatListUpdatesResponse response; + + auto r = response.mutable_list_update_responses()->Add(); + InitUpdateResponse(r, SOCIAL_ENGINEERING_PUBLIC, + nsCString("sta\x00te", 6), + nsCString("check\x0sum", 9), + true, + {0, 1, 2, 3}, + false /* aDoPrefixEncoding */ ); + + // Set min wait duration. + auto minWaitDuration = response.mutable_minimum_wait_duration(); + minWaitDuration->set_seconds(8); + minWaitDuration->set_nanos(1 * 1000000000); + + std::string s; + response.SerializeToString(&s); + + DumpBinary(nsCString(s.c_str(), s.length())); + + ProtocolParser* p = new ProtocolParserProtobuf(); + p->AppendStream(nsCString(s.c_str(), s.length())); + p->End(); + ASSERT_EQ(p->UpdateWaitSec(), 9u); + delete p; +} + +TEST(ProtocolParser, SingleValueEncoding) +{ + // Top level response which contains a list of update response + // for different lists. + FetchThreatListUpdatesResponse response; + + auto r = response.mutable_list_update_responses()->Add(); + + const char* expectedPrefix = "\x00\x01\x02\x00"; + if (!InitUpdateResponse(r, SOCIAL_ENGINEERING_PUBLIC, + nsCString("sta\x00te", 6), + nsCString("check\x0sum", 9), + true, + // As per spec, we should interpret the prefix as uint32 + // in little endian before encoding. + {LittleEndian::readUint32(expectedPrefix)}, + true /* aDoPrefixEncoding */ )) { + printf("Failed to initialize update response."); + ASSERT_TRUE(false); + return; + } + + // Set min wait duration. + auto minWaitDuration = response.mutable_minimum_wait_duration(); + minWaitDuration->set_seconds(8); + minWaitDuration->set_nanos(1 * 1000000000); + + std::string s; + response.SerializeToString(&s); + + // Feed data to the protocol parser. + ProtocolParser* p = new ProtocolParserProtobuf(); + p->SetRequestedTables({ nsCString("googpub-phish-proto") }); + p->AppendStream(nsCString(s.c_str(), s.length())); + p->End(); + + auto& tus = p->GetTableUpdates(); + auto tuv4 = TableUpdate::Cast<TableUpdateV4>(tus[0]); + auto& prefixMap = tuv4->Prefixes(); + for (auto iter = prefixMap.Iter(); !iter.Done(); iter.Next()) { + // This prefix map should contain only a single 4-byte prefixe. + ASSERT_EQ(iter.Key(), 4u); + + // The fixed-length prefix string from ProtcolParser should + // exactly match the expected prefix string. + auto& prefix = iter.Data()->GetPrefixString(); + ASSERT_TRUE(prefix.Equals(nsCString(expectedPrefix, 4))); + } + + delete p; +} + +static bool +InitUpdateResponse(ListUpdateResponse* aUpdateResponse, + ThreatType aThreatType, + const nsACString& aState, + const nsACString& aChecksum, + bool isFullUpdate, + const nsTArray<uint32_t>& aFixedLengthPrefixes, + bool aDoPrefixEncoding) +{ + aUpdateResponse->set_threat_type(aThreatType); + aUpdateResponse->set_new_client_state(aState.BeginReading(), aState.Length()); + aUpdateResponse->mutable_checksum()->set_sha256(aChecksum.BeginReading(), aChecksum.Length()); + aUpdateResponse->set_response_type(isFullUpdate ? ListUpdateResponse::FULL_UPDATE + : ListUpdateResponse::PARTIAL_UPDATE); + + auto additions = aUpdateResponse->mutable_additions()->Add(); + + if (!aDoPrefixEncoding) { + additions->set_compression_type(RAW); + auto rawHashes = additions->mutable_raw_hashes(); + rawHashes->set_prefix_size(4); + auto prefixes = rawHashes->mutable_raw_hashes(); + for (auto p : aFixedLengthPrefixes) { + char buffer[4]; + NativeEndian::copyAndSwapToBigEndian(buffer, &p, 1); + prefixes->append(buffer, 4); + } + return true; + } + + if (1 != aFixedLengthPrefixes.Length()) { + printf("This function only supports single value encoding.\n"); + return false; + } + + uint32_t firstValue = aFixedLengthPrefixes[0]; + additions->set_compression_type(RICE); + auto riceHashes = additions->mutable_rice_hashes(); + riceHashes->set_first_value(firstValue); + riceHashes->set_num_entries(0); + + return true; +} + +static void DumpBinary(const nsACString& aBinary) +{ + nsCString s; + for (size_t i = 0; i < aBinary.Length(); i++) { + s.AppendPrintf("\\x%.2X", (uint8_t)aBinary[i]); + } + printf("%s\n", s.get()); +}
\ No newline at end of file diff --git a/toolkit/components/url-classifier/tests/gtest/TestRiceDeltaDecoder.cpp b/toolkit/components/url-classifier/tests/gtest/TestRiceDeltaDecoder.cpp new file mode 100644 index 000000000..f03d27358 --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/TestRiceDeltaDecoder.cpp @@ -0,0 +1,165 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +#include "gtest/gtest.h" +#include "RiceDeltaDecoder.h" +#include "mozilla/ArrayUtils.h" + +using namespace mozilla; +using namespace mozilla::safebrowsing; + +struct TestingData { + std::vector<uint32_t> mExpectedDecoded; + std::vector<uint8_t> mEncoded; + uint32_t mRiceParameter; +}; + +static bool runOneTest(TestingData& aData); + +TEST(RiceDeltaDecoder, SingleEncodedValue) { + TestingData td = { { 99 }, { 99 }, 0 }; + + ASSERT_TRUE(runOneTest(td)); +} + +// In this batch of tests, the encoded data would be like +// what we originally receive from the network. See comment +// in |runOneTest| for more detail. +TEST(RiceDeltaDecoder, Empty) { + + // The following structure and testing data is copied from Chromium source code: + // + // https://chromium.googlesource.com/chromium/src.git/+/950f9975599768b6a08c7146cb4befa161be87aa/components/safe_browsing_db/v4_rice_unittest.cc#75 + // + // and will be translated to our own testing format. + + struct RiceDecodingTestInfo { + uint32_t mRiceParameter; + std::vector<uint32_t> mDeltas; + std::string mEncoded; + + RiceDecodingTestInfo(uint32_t aRiceParameter, + const std::vector<uint32_t>& aDeltas, + const std::string& aEncoded) + : mRiceParameter(aRiceParameter) + , mDeltas(aDeltas) + , mEncoded(aEncoded) + { + } + }; + + // Copyright 2016 The Chromium Authors. All rights reserved. + // Use of this source code is governed by a BSD-style license that can be + // found in the media/webrtc/trunk/webrtc/LICENSE. + + // ----- Start of Chromium test code ---- + const std::vector<RiceDecodingTestInfo> TESTING_DATA_CHROMIUM = { + RiceDecodingTestInfo(2, {15, 9}, "\xf7\x2"), + RiceDecodingTestInfo( + 28, {1777762129, 2093280223, 924369848}, + "\xbf\xa8\x3f\xfb\xfc\xfb\x5e\x27\xe6\xc3\x1d\xc6\x38"), + RiceDecodingTestInfo( + 28, {62763050, 1046523781, 192522171, 1800511020, 4442775, 582142548}, + "\x54\x60\x7b\xe7\x0a\x5f\xc1\xdc\xee\x69\xde" + "\xfe\x58\x3c\xa3\xd6\xa5\xf2\x10\x8c\x4a\x59" + "\x56\x00"), + RiceDecodingTestInfo( + 28, {26067715, 344823336, 8420095, 399843890, 95029378, 731622412, + 35811335, 1047558127, 1117722715, 78698892}, + "\x06\x86\x1b\x23\x14\xcb\x46\xf2\xaf\x07\x08\xc9\x88\x54\x1f\x41\x04" + "\xd5\x1a\x03\xeb\xe6\x3a\x80\x13\x91\x7b\xbf\x83\xf3\xb7\x85\xf1\x29" + "\x18\xb3\x61\x09"), + RiceDecodingTestInfo( + 27, {225846818, 328287420, 166748623, 29117720, 552397365, 350353215, + 558267528, 4738273, 567093445, 28563065, 55077698, 73091685, + 339246010, 98242620, 38060941, 63917830, 206319759, 137700744}, + "\x89\x98\xd8\x75\xbc\x44\x91\xeb\x39\x0c\x3e\x30\x9a\x78\xf3\x6a\xd4" + "\xd9\xb1\x9f\xfb\x70\x3e\x44\x3e\xa3\x08\x67\x42\xc2\x2b\x46\x69\x8e" + "\x3c\xeb\xd9\x10\x5a\x43\x9a\x32\xa5\x2d\x4e\x77\x0f\x87\x78\x20\xb6" + "\xab\x71\x98\x48\x0c\x9e\x9e\xd7\x23\x0c\x13\x43\x2c\xa9\x01"), + RiceDecodingTestInfo( + 28, {339784008, 263128563, 63871877, 69723256, 826001074, 797300228, + 671166008, 207712688}, + std::string("\x21\xc5\x02\x91\xf9\x82\xd7\x57\xb8\xe9\x3c\xf0\xc8\x4f" + "\xe8\x64\x8d\x77\x62\x04\xd6\x85\x3f\x1c\x97\x00\x04\x1b" + "\x17\xc6", + 30)), + RiceDecodingTestInfo( + 28, {471820069, 196333855, 855579133, 122737976, 203433838, 85354544, + 1307949392, 165938578, 195134475, 553930435, 49231136}, + "\x95\x9c\x7d\xb0\x8f\xe8\xd9\xbd\xfe\x8c\x7f\x81\x53\x0d\x75\xdc\x4e" + "\x40\x18\x0c\x9a\x45\x3d\xa8\xdc\xfa\x26\x59\x40\x9e\x16\x08\x43\x77" + "\xc3\x4e\x04\x01\xa4\xe6\x5d\x00"), + RiceDecodingTestInfo( + 27, {87336845, 129291033, 30906211, 433549264, 30899891, 53207875, + 11959529, 354827862, 82919275, 489637251, 53561020, 336722992, + 408117728, 204506246, 188216092, 9047110, 479817359, 230317256}, + "\x1a\x4f\x69\x2a\x63\x9a\xf6\xc6\x2e\xaf\x73\xd0\x6f\xd7\x31\xeb\x77" + "\x1d\x43\xe3\x2b\x93\xce\x67\x8b\x59\xf9\x98\xd4\xda\x4f\x3c\x6f\xb0" + "\xe8\xa5\x78\x8d\x62\x36\x18\xfe\x08\x1e\x78\xd8\x14\x32\x24\x84\x61" + "\x1c\xf3\x37\x63\xc4\xa0\x88\x7b\x74\xcb\x64\xc8\x5c\xba\x05"), + RiceDecodingTestInfo( + 28, {297968956, 19709657, 259702329, 76998112, 1023176123, 29296013, + 1602741145, 393745181, 177326295, 55225536, 75194472}, + "\xf1\x94\x0a\x87\x6c\x5f\x96\x90\xe3\xab\xf7\xc0\xcb\x2d\xe9\x76\xdb" + "\xf8\x59\x63\xc1\x6f\x7c\x99\xe3\x87\x5f\xc7\x04\xde\xb9\x46\x8e\x54" + "\xc0\xac\x4a\x03\x0d\x6c\x8f\x00"), + RiceDecodingTestInfo( + 28, {532220688, 780594691, 436816483, 163436269, 573044456, 1069604, + 39629436, 211410997, 227714491, 381562898, 75610008, 196754597, + 40310339, 15204118, 99010842}, + "\x41\x2c\xe4\xfe\x06\xdc\x0d\xbd\x31\xa5\x04\xd5\x6e\xdd\x9b\x43\xb7" + "\x3f\x11\x24\x52\x10\x80\x4f\x96\x4b\xd4\x80\x67\xb2\xdd\x52\xc9\x4e" + "\x02\xc6\xd7\x60\xde\x06\x92\x52\x1e\xdd\x35\x64\x71\x26\x2c\xfe\xcf" + "\x81\x46\xb2\x79\x01"), + RiceDecodingTestInfo( + 28, {219354713, 389598618, 750263679, 554684211, 87381124, 4523497, + 287633354, 801308671, 424169435, 372520475, 277287849}, + "\xb2\x2c\x26\x3a\xcd\x66\x9c\xdb\x5f\x07\x2e\x6f\xe6\xf9\x21\x10\x52" + "\xd5\x94\xf4\x82\x22\x48\xf9\x9d\x24\xf6\xff\x2f\xfc\x6d\x3f\x21\x65" + "\x1b\x36\x34\x56\xea\xc4\x21\x00"), + }; + + // ----- End of Chromium test code ---- + + for (auto tdc : TESTING_DATA_CHROMIUM) { + // Populate chromium testing data to our native testing data struct. + TestingData d; + + d.mRiceParameter = tdc.mRiceParameter; // Populate rice parameter. + + // Populate encoded data from std::string to vector<uint8>. + d.mEncoded.resize(tdc.mEncoded.size()); + memcpy(&d.mEncoded[0], tdc.mEncoded.c_str(), tdc.mEncoded.size()); + + // Populate deltas to expected decoded data. The first value would be just + // set to an arbitrary value, say 7, to avoid any assumption to the + // first value in the implementation. + d.mExpectedDecoded.resize(tdc.mDeltas.size() + 1); + for (size_t i = 0; i < d.mExpectedDecoded.size(); i++) { + if (0 == i) { + d.mExpectedDecoded[i] = 7; // "7" is an arbitrary starting value + } else { + d.mExpectedDecoded[i] = d.mExpectedDecoded[i - 1] + tdc.mDeltas[i - 1]; + } + } + + ASSERT_TRUE(runOneTest(d)); + } +} + +static bool +runOneTest(TestingData& aData) +{ + RiceDeltaDecoder decoder(&aData.mEncoded[0], aData.mEncoded.size()); + + std::vector<uint32_t> decoded(aData.mExpectedDecoded.size()); + + uint32_t firstValue = aData.mExpectedDecoded[0]; + bool rv = decoder.Decode(aData.mRiceParameter, + firstValue, + decoded.size() - 1, // # of entries (first value not included). + &decoded[0]); + + return rv && decoded == aData.mExpectedDecoded; +} diff --git a/toolkit/components/url-classifier/tests/gtest/TestSafeBrowsingProtobuf.cpp b/toolkit/components/url-classifier/tests/gtest/TestSafeBrowsingProtobuf.cpp new file mode 100644 index 000000000..fe6f28960 --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/TestSafeBrowsingProtobuf.cpp @@ -0,0 +1,24 @@ +#include "safebrowsing.pb.h" +#include "gtest/gtest.h" + +TEST(SafeBrowsingProtobuf, Empty) +{ + using namespace mozilla::safebrowsing; + + const std::string CLIENT_ID = "firefox"; + + // Construct a simple update request. + FetchThreatListUpdatesRequest r; + r.set_allocated_client(new ClientInfo()); + r.mutable_client()->set_client_id(CLIENT_ID); + + // Then serialize. + std::string s; + r.SerializeToString(&s); + + // De-serialize. + FetchThreatListUpdatesRequest r2; + r2.ParseFromString(s); + + ASSERT_EQ(r2.client().client_id(), CLIENT_ID); +} diff --git a/toolkit/components/url-classifier/tests/gtest/TestSafebrowsingHash.cpp b/toolkit/components/url-classifier/tests/gtest/TestSafebrowsingHash.cpp new file mode 100644 index 000000000..89ed74be6 --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/TestSafebrowsingHash.cpp @@ -0,0 +1,52 @@ +#include "Entries.h" +#include "mozilla/EndianUtils.h" + +TEST(SafebrowsingHash, ToFromUint32) +{ + using namespace mozilla::safebrowsing; + + // typedef SafebrowsingHash<PREFIX_SIZE, PrefixComparator> Prefix; + // typedef nsTArray<Prefix> PrefixArray; + + const char PREFIX_RAW[4] = { 0x1, 0x2, 0x3, 0x4 }; + uint32_t PREFIX_UINT32; + memcpy(&PREFIX_UINT32, PREFIX_RAW, 4); + + Prefix p; + p.Assign(nsCString(PREFIX_RAW, 4)); + ASSERT_EQ(p.ToUint32(), PREFIX_UINT32); + + p.FromUint32(PREFIX_UINT32); + ASSERT_EQ(memcmp(PREFIX_RAW, p.buf, 4), 0); +} + +TEST(SafebrowsingHash, Compare) +{ + using namespace mozilla; + using namespace mozilla::safebrowsing; + + Prefix p1, p2, p3; + + // The order of p1,p2,p3 is "p1 == p3 < p2" +#if MOZ_LITTLE_ENDIAN + p1.Assign(nsCString("\x01\x00\x00\x00", 4)); + p2.Assign(nsCString("\x00\x00\x00\x01", 4)); + p3.Assign(nsCString("\x01\x00\x00\x00", 4)); +#else + p1.Assign(nsCString("\x00\x00\x00\x01", 4)); + p2.Assign(nsCString("\x01\x00\x00\x00", 4)); + p3.Assign(nsCString("\x00\x00\x00\x01", 4)); +#endif + + // Make sure "p1 == p3 < p2" is true + // on both little and big endian machine. + + ASSERT_EQ(p1.Compare(p2), -1); + ASSERT_EQ(p1.Compare(p1), 0); + ASSERT_EQ(p2.Compare(p1), 1); + ASSERT_EQ(p1.Compare(p3), 0); + + ASSERT_TRUE(p1 < p2); + ASSERT_TRUE(p1 == p1); + ASSERT_TRUE(p1 == p3); +}
\ No newline at end of file diff --git a/toolkit/components/url-classifier/tests/gtest/TestTable.cpp b/toolkit/components/url-classifier/tests/gtest/TestTable.cpp new file mode 100644 index 000000000..307587459 --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/TestTable.cpp @@ -0,0 +1,47 @@ +#include "gtest/gtest.h" +#include "nsUrlClassifierDBService.h" + +using namespace mozilla::safebrowsing; + +void +TestResponseCode(const char* table, nsresult result) +{ + nsCString tableName(table); + ASSERT_EQ(TablesToResponse(tableName), result); +} + +TEST(UrlClassifierTable, ResponseCode) +{ + // malware URIs. + TestResponseCode("goog-malware-shavar", NS_ERROR_MALWARE_URI); + TestResponseCode("test-malware-simple", NS_ERROR_MALWARE_URI); + TestResponseCode("goog-phish-shavar,test-malware-simple", NS_ERROR_MALWARE_URI); + TestResponseCode("test-malware-simple,mozstd-track-digest256,mozplugin-block-digest256", NS_ERROR_MALWARE_URI); + + // phish URIs. + TestResponseCode("goog-phish-shavar", NS_ERROR_PHISHING_URI); + TestResponseCode("test-phish-simple", NS_ERROR_PHISHING_URI); + TestResponseCode("test-phish-simple,mozplugin-block-digest256", NS_ERROR_PHISHING_URI); + TestResponseCode("mozstd-track-digest256,test-phish-simple,goog-unwanted-shavar", NS_ERROR_PHISHING_URI); + + // unwanted URIs. + TestResponseCode("goog-unwanted-shavar", NS_ERROR_UNWANTED_URI); + TestResponseCode("test-unwanted-simple", NS_ERROR_UNWANTED_URI); + TestResponseCode("mozplugin-unwanted-digest256,mozfull-track-digest256", NS_ERROR_UNWANTED_URI); + TestResponseCode("test-block-simple,mozfull-track-digest256,test-unwanted-simple", NS_ERROR_UNWANTED_URI); + + // track URIs. + TestResponseCode("test-track-simple", NS_ERROR_TRACKING_URI); + TestResponseCode("mozstd-track-digest256", NS_ERROR_TRACKING_URI); + TestResponseCode("test-block-simple,mozstd-track-digest256", NS_ERROR_TRACKING_URI); + + // block URIs + TestResponseCode("test-block-simple", NS_ERROR_BLOCKED_URI); + TestResponseCode("mozplugin-block-digest256", NS_ERROR_BLOCKED_URI); + TestResponseCode("mozplugin2-block-digest256", NS_ERROR_BLOCKED_URI); + + TestResponseCode("test-trackwhite-simple", NS_OK); + TestResponseCode("mozstd-trackwhite-digest256", NS_OK); + TestResponseCode("goog-badbinurl-shavar", NS_OK); + TestResponseCode("goog-downloadwhite-digest256", NS_OK); +} diff --git a/toolkit/components/url-classifier/tests/gtest/TestUrlClassifierTableUpdateV4.cpp b/toolkit/components/url-classifier/tests/gtest/TestUrlClassifierTableUpdateV4.cpp new file mode 100644 index 000000000..470a88ba2 --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/TestUrlClassifierTableUpdateV4.cpp @@ -0,0 +1,755 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +#include "Common.h" +#include "Classifier.h" +#include "HashStore.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsIFile.h" +#include "nsIThread.h" +#include "string.h" +#include "gtest/gtest.h" +#include "nsThreadUtils.h" + +using namespace mozilla; +using namespace mozilla::safebrowsing; + +typedef nsCString _Prefix; +typedef nsTArray<_Prefix> _PrefixArray; + +#define GTEST_SAFEBROWSING_DIR NS_LITERAL_CSTRING("safebrowsing") +#define GTEST_TABLE NS_LITERAL_CSTRING("gtest-malware-proto") +#define GTEST_PREFIXFILE NS_LITERAL_CSTRING("gtest-malware-proto.pset") + +// This function removes common elements of inArray and outArray from +// outArray. This is used by partial update testcase to ensure partial update +// data won't contain prefixes we already have. +static void +RemoveIntersection(const _PrefixArray& inArray, _PrefixArray& outArray) +{ + for (uint32_t i = 0; i < inArray.Length(); i++) { + int32_t idx = outArray.BinaryIndexOf(inArray[i]); + if (idx >= 0) { + outArray.RemoveElementAt(idx); + } + } +} + +// This fucntion removes elements from outArray by index specified in +// removal array. +static void +RemoveElements(const nsTArray<uint32_t>& removal, _PrefixArray& outArray) +{ + for (int32_t i = removal.Length() - 1; i >= 0; i--) { + outArray.RemoveElementAt(removal[i]); + } +} + +static void +MergeAndSortArray(const _PrefixArray& array1, + const _PrefixArray& array2, + _PrefixArray& output) +{ + output.Clear(); + output.AppendElements(array1); + output.AppendElements(array2); + output.Sort(); +} + +static void +CalculateCheckSum(_PrefixArray& prefixArray, nsCString& checksum) +{ + prefixArray.Sort(); + + nsresult rv; + nsCOMPtr<nsICryptoHash> cryptoHash = + do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + + cryptoHash->Init(nsICryptoHash::SHA256); + for (uint32_t i = 0; i < prefixArray.Length(); i++) { + const _Prefix& prefix = prefixArray[i]; + cryptoHash->Update(reinterpret_cast<uint8_t*>( + const_cast<char*>(prefix.get())), prefix.Length()); + } + cryptoHash->Finish(false, checksum); +} + +// N: Number of prefixes, MIN/MAX: minimum/maximum prefix size +// This function will append generated prefixes to outArray. +static void +CreateRandomSortedPrefixArray(uint32_t N, + uint32_t MIN, + uint32_t MAX, + _PrefixArray& outArray) +{ + outArray.SetCapacity(outArray.Length() + N); + + const uint32_t range = (MAX - MIN + 1); + + for (uint32_t i = 0; i < N; i++) { + uint32_t prefixSize = (rand() % range) + MIN; + _Prefix prefix; + prefix.SetLength(prefixSize); + + while (true) { + char* dst = prefix.BeginWriting(); + for (uint32_t j = 0; j < prefixSize; j++) { + dst[j] = rand() % 256; + } + + if (!outArray.Contains(prefix)) { + outArray.AppendElement(prefix); + break; + } + } + } + + outArray.Sort(); +} + +// N: Number of removal indices, MAX: maximum index +static void +CreateRandomRemovalIndices(uint32_t N, + uint32_t MAX, + nsTArray<uint32_t>& outArray) +{ + for (uint32_t i = 0; i < N; i++) { + uint32_t idx = rand() % MAX; + if (!outArray.Contains(idx)) { + outArray.InsertElementSorted(idx); + } + } +} + +// Function to generate TableUpdateV4. +static void +GenerateUpdateData(bool fullUpdate, + PrefixStringMap& add, + nsTArray<uint32_t>* removal, + nsCString* checksum, + nsTArray<TableUpdate*>& tableUpdates) +{ + TableUpdateV4* tableUpdate = new TableUpdateV4(GTEST_TABLE); + tableUpdate->SetFullUpdate(fullUpdate); + + for (auto iter = add.ConstIter(); !iter.Done(); iter.Next()) { + nsCString* pstring = iter.Data(); + std::string str(pstring->BeginReading(), pstring->Length()); + + tableUpdate->NewPrefixes(iter.Key(), str); + } + + if (removal) { + tableUpdate->NewRemovalIndices(removal->Elements(), removal->Length()); + } + + if (checksum) { + std::string stdChecksum; + stdChecksum.assign(const_cast<char*>(checksum->BeginReading()), checksum->Length()); + + tableUpdate->NewChecksum(stdChecksum); + } + + tableUpdates.AppendElement(tableUpdate); +} + +static void +VerifyPrefixSet(PrefixStringMap& expected) +{ + // Verify the prefix set is written to disk. + nsCOMPtr<nsIFile> file; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file)); + + file->AppendNative(GTEST_SAFEBROWSING_DIR); + file->AppendNative(GTEST_PREFIXFILE); + + RefPtr<VariableLengthPrefixSet> load = new VariableLengthPrefixSet; + load->Init(GTEST_TABLE); + + PrefixStringMap prefixesInFile; + load->LoadFromFile(file); + load->GetPrefixes(prefixesInFile); + + for (auto iter = expected.ConstIter(); !iter.Done(); iter.Next()) { + nsCString* expectedPrefix = iter.Data(); + nsCString* resultPrefix = prefixesInFile.Get(iter.Key()); + + ASSERT_TRUE(*resultPrefix == *expectedPrefix); + } +} + +static void +Clear() +{ + nsCOMPtr<nsIFile> file; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file)); + + UniquePtr<Classifier> classifier(new Classifier()); + classifier->Open(*file); + classifier->Reset(); +} + +static void +testUpdateFail(nsTArray<TableUpdate*>& tableUpdates) +{ + nsCOMPtr<nsIFile> file; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file)); + + UniquePtr<Classifier> classifier(new Classifier()); + classifier->Open(*file); + + RunTestInNewThread([&] () -> void { + nsresult rv = classifier->ApplyUpdates(&tableUpdates); + ASSERT_TRUE(NS_FAILED(rv)); + }); +} + +static void +testUpdate(nsTArray<TableUpdate*>& tableUpdates, + PrefixStringMap& expected) +{ + nsCOMPtr<nsIFile> file; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file)); + + UniquePtr<Classifier> classifier(new Classifier()); + classifier->Open(*file); + + RunTestInNewThread([&] () -> void { + nsresult rv = classifier->ApplyUpdates(&tableUpdates); + ASSERT_TRUE(rv == NS_OK); + + VerifyPrefixSet(expected); + }); +} + +static void +testFullUpdate(PrefixStringMap& add, nsCString* checksum) +{ + nsTArray<TableUpdate*> tableUpdates; + + GenerateUpdateData(true, add, nullptr, checksum, tableUpdates); + + testUpdate(tableUpdates, add); +} + +static void +testPartialUpdate(PrefixStringMap& add, + nsTArray<uint32_t>* removal, + nsCString* checksum, + PrefixStringMap& expected) +{ + nsTArray<TableUpdate*> tableUpdates; + GenerateUpdateData(false, add, removal, checksum, tableUpdates); + + testUpdate(tableUpdates, expected); +} + +static void +testOpenLookupCache() +{ + nsCOMPtr<nsIFile> file; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file)); + file->AppendNative(GTEST_SAFEBROWSING_DIR); + + RunTestInNewThread([&] () -> void { + LookupCacheV4 cache(nsCString(GTEST_TABLE), EmptyCString(), file); + nsresult rv = cache.Init(); + ASSERT_EQ(rv, NS_OK); + + rv = cache.Open(); + ASSERT_EQ(rv, NS_OK); + }); +} + +// Tests start from here. +TEST(UrlClassifierTableUpdateV4, FixLenghtPSetFullUpdate) +{ + srand(time(NULL)); + + _PrefixArray array; + PrefixStringMap map; + nsCString checksum; + + CreateRandomSortedPrefixArray(5000, 4, 4, array); + PrefixArrayToPrefixStringMap(array, map); + CalculateCheckSum(array, checksum); + + testFullUpdate(map, &checksum); + + Clear(); +} + +TEST(UrlClassifierTableUpdateV4, VariableLenghtPSetFullUpdate) +{ + _PrefixArray array; + PrefixStringMap map; + nsCString checksum; + + CreateRandomSortedPrefixArray(5000, 5, 32, array); + PrefixArrayToPrefixStringMap(array, map); + CalculateCheckSum(array, checksum); + + testFullUpdate(map, &checksum); + + Clear(); +} + +// This test contain both variable length prefix set and fixed-length prefix set +TEST(UrlClassifierTableUpdateV4, MixedPSetFullUpdate) +{ + _PrefixArray array; + PrefixStringMap map; + nsCString checksum; + + CreateRandomSortedPrefixArray(5000, 4, 4, array); + CreateRandomSortedPrefixArray(1000, 5, 32, array); + PrefixArrayToPrefixStringMap(array, map); + CalculateCheckSum(array, checksum); + + testFullUpdate(map, &checksum); + + Clear(); +} + +TEST(UrlClassifierTableUpdateV4, PartialUpdateWithRemoval) +{ + _PrefixArray fArray; + + // Apply a full update first. + { + PrefixStringMap fMap; + nsCString checksum; + + CreateRandomSortedPrefixArray(10000, 4, 4, fArray); + CreateRandomSortedPrefixArray(2000, 5, 32, fArray); + PrefixArrayToPrefixStringMap(fArray, fMap); + CalculateCheckSum(fArray, checksum); + + testFullUpdate(fMap, &checksum); + } + + // Apply a partial update with removal. + { + _PrefixArray pArray, mergedArray; + PrefixStringMap pMap, mergedMap; + nsCString checksum; + + CreateRandomSortedPrefixArray(5000, 4, 4, pArray); + CreateRandomSortedPrefixArray(1000, 5, 32, pArray); + RemoveIntersection(fArray, pArray); + PrefixArrayToPrefixStringMap(pArray, pMap); + + // Remove 1/5 of elements of original prefix set. + nsTArray<uint32_t> removal; + CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal); + RemoveElements(removal, fArray); + + // Calculate the expected prefix map. + MergeAndSortArray(fArray, pArray, mergedArray); + PrefixArrayToPrefixStringMap(mergedArray, mergedMap); + CalculateCheckSum(mergedArray, checksum); + + testPartialUpdate(pMap, &removal, &checksum, mergedMap); + } + + Clear(); +} + +TEST(UrlClassifierTableUpdateV4, PartialUpdateWithoutRemoval) +{ + _PrefixArray fArray; + + // Apply a full update first. + { + PrefixStringMap fMap; + nsCString checksum; + + CreateRandomSortedPrefixArray(10000, 4, 4, fArray); + CreateRandomSortedPrefixArray(2000, 5, 32, fArray); + PrefixArrayToPrefixStringMap(fArray, fMap); + CalculateCheckSum(fArray, checksum); + + testFullUpdate(fMap, &checksum); + } + + // Apply a partial update without removal + { + _PrefixArray pArray, mergedArray; + PrefixStringMap pMap, mergedMap; + nsCString checksum; + + CreateRandomSortedPrefixArray(5000, 4, 4, pArray); + CreateRandomSortedPrefixArray(1000, 5, 32, pArray); + RemoveIntersection(fArray, pArray); + PrefixArrayToPrefixStringMap(pArray, pMap); + + // Calculate the expected prefix map. + MergeAndSortArray(fArray, pArray, mergedArray); + PrefixArrayToPrefixStringMap(mergedArray, mergedMap); + CalculateCheckSum(mergedArray, checksum); + + testPartialUpdate(pMap, nullptr, &checksum, mergedMap); + } + + Clear(); +} + +// Expect failure because partial update contains prefix already +// in old prefix set. +TEST(UrlClassifierTableUpdateV4, PartialUpdatePrefixAlreadyExist) +{ + _PrefixArray fArray; + + // Apply a full update fist. + { + PrefixStringMap fMap; + nsCString checksum; + + CreateRandomSortedPrefixArray(1000, 4, 32, fArray); + PrefixArrayToPrefixStringMap(fArray, fMap); + CalculateCheckSum(fArray, checksum); + + testFullUpdate(fMap, &checksum); + } + + // Apply a partial update which contains a prefix in previous full update. + // This should cause an update error. + { + _PrefixArray pArray; + PrefixStringMap pMap; + nsTArray<TableUpdate*> tableUpdates; + + // Pick one prefix from full update prefix and add it to partial update. + // This should result a failure when call ApplyUpdates. + pArray.AppendElement(fArray[rand() % fArray.Length()]); + CreateRandomSortedPrefixArray(200, 4, 32, pArray); + PrefixArrayToPrefixStringMap(pArray, pMap); + + GenerateUpdateData(false, pMap, nullptr, nullptr, tableUpdates); + testUpdateFail(tableUpdates); + } + + Clear(); +} + +// Test apply partial update directly without applying an full update first. +TEST(UrlClassifierTableUpdateV4, OnlyPartialUpdate) +{ + _PrefixArray pArray; + PrefixStringMap pMap; + nsCString checksum; + + CreateRandomSortedPrefixArray(5000, 4, 4, pArray); + CreateRandomSortedPrefixArray(1000, 5, 32, pArray); + PrefixArrayToPrefixStringMap(pArray, pMap); + CalculateCheckSum(pArray, checksum); + + testPartialUpdate(pMap, nullptr, &checksum, pMap); + + Clear(); +} + +// Test partial update without any ADD prefixes, only removalIndices. +TEST(UrlClassifierTableUpdateV4, PartialUpdateOnlyRemoval) +{ + _PrefixArray fArray; + + // Apply a full update first. + { + PrefixStringMap fMap; + nsCString checksum; + + CreateRandomSortedPrefixArray(5000, 4, 4, fArray); + CreateRandomSortedPrefixArray(1000, 5, 32, fArray); + PrefixArrayToPrefixStringMap(fArray, fMap); + CalculateCheckSum(fArray, checksum); + + testFullUpdate(fMap, &checksum); + } + + // Apply a partial update without add prefix, only contain removal indices. + { + _PrefixArray pArray; + PrefixStringMap pMap, mergedMap; + nsCString checksum; + + // Remove 1/5 of elements of original prefix set. + nsTArray<uint32_t> removal; + CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal); + RemoveElements(removal, fArray); + + PrefixArrayToPrefixStringMap(fArray, mergedMap); + CalculateCheckSum(fArray, checksum); + + testPartialUpdate(pMap, &removal, &checksum, mergedMap); + } + + Clear(); +} + +// Test one tableupdate array contains full update and multiple partial updates. +TEST(UrlClassifierTableUpdateV4, MultipleTableUpdates) +{ + _PrefixArray fArray, pArray, mergedArray; + PrefixStringMap fMap, pMap, mergedMap; + nsCString checksum; + + nsTArray<TableUpdate*> tableUpdates; + + // Generate first full udpate + CreateRandomSortedPrefixArray(10000, 4, 4, fArray); + CreateRandomSortedPrefixArray(2000, 5, 32, fArray); + PrefixArrayToPrefixStringMap(fArray, fMap); + CalculateCheckSum(fArray, checksum); + + GenerateUpdateData(true, fMap, nullptr, &checksum, tableUpdates); + + // Generate second partial update + CreateRandomSortedPrefixArray(3000, 4, 4, pArray); + CreateRandomSortedPrefixArray(1000, 5, 32, pArray); + RemoveIntersection(fArray, pArray); + PrefixArrayToPrefixStringMap(pArray, pMap); + + MergeAndSortArray(fArray, pArray, mergedArray); + CalculateCheckSum(mergedArray, checksum); + + GenerateUpdateData(false, pMap, nullptr, &checksum, tableUpdates); + + // Generate thrid partial update + fArray.AppendElements(pArray); + fArray.Sort(); + pArray.Clear(); + CreateRandomSortedPrefixArray(3000, 4, 4, pArray); + CreateRandomSortedPrefixArray(1000, 5, 32, pArray); + RemoveIntersection(fArray, pArray); + PrefixArrayToPrefixStringMap(pArray, pMap); + + // Remove 1/5 of elements of original prefix set. + nsTArray<uint32_t> removal; + CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal); + RemoveElements(removal, fArray); + + MergeAndSortArray(fArray, pArray, mergedArray); + PrefixArrayToPrefixStringMap(mergedArray, mergedMap); + CalculateCheckSum(mergedArray, checksum); + + GenerateUpdateData(false, pMap, &removal, &checksum, tableUpdates); + + testUpdate(tableUpdates, mergedMap); + + Clear(); +} + +// Test apply full update first, and then apply multiple partial updates +// in one tableupdate array. +TEST(UrlClassifierTableUpdateV4, MultiplePartialUpdateTableUpdates) +{ + _PrefixArray fArray; + + // Apply a full update first + { + PrefixStringMap fMap; + nsCString checksum; + + // Generate first full udpate + CreateRandomSortedPrefixArray(10000, 4, 4, fArray); + CreateRandomSortedPrefixArray(3000, 5, 32, fArray); + PrefixArrayToPrefixStringMap(fArray, fMap); + CalculateCheckSum(fArray, checksum); + + testFullUpdate(fMap, &checksum); + } + + // Apply multiple partial updates in one table update + { + _PrefixArray pArray, mergedArray; + PrefixStringMap pMap, mergedMap; + nsCString checksum; + nsTArray<uint32_t> removal; + nsTArray<TableUpdate*> tableUpdates; + + // Generate first partial update + CreateRandomSortedPrefixArray(3000, 4, 4, pArray); + CreateRandomSortedPrefixArray(1000, 5, 32, pArray); + RemoveIntersection(fArray, pArray); + PrefixArrayToPrefixStringMap(pArray, pMap); + + // Remove 1/5 of elements of original prefix set. + CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal); + RemoveElements(removal, fArray); + + MergeAndSortArray(fArray, pArray, mergedArray); + CalculateCheckSum(mergedArray, checksum); + + GenerateUpdateData(false, pMap, &removal, &checksum, tableUpdates); + + fArray.AppendElements(pArray); + fArray.Sort(); + pArray.Clear(); + removal.Clear(); + + // Generate second partial update. + CreateRandomSortedPrefixArray(2000, 4, 4, pArray); + CreateRandomSortedPrefixArray(1000, 5, 32, pArray); + RemoveIntersection(fArray, pArray); + PrefixArrayToPrefixStringMap(pArray, pMap); + + // Remove 1/5 of elements of original prefix set. + CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal); + RemoveElements(removal, fArray); + + MergeAndSortArray(fArray, pArray, mergedArray); + PrefixArrayToPrefixStringMap(mergedArray, mergedMap); + CalculateCheckSum(mergedArray, checksum); + + GenerateUpdateData(false, pMap, &removal, &checksum, tableUpdates); + + testUpdate(tableUpdates, mergedMap); + } + + Clear(); +} + +// Test removal indices are larger than the original prefix set. +TEST(UrlClassifierTableUpdateV4, RemovalIndexTooLarge) +{ + _PrefixArray fArray; + + // Apply a full update first + { + PrefixStringMap fMap; + nsCString checksum; + + CreateRandomSortedPrefixArray(1000, 4, 32, fArray); + PrefixArrayToPrefixStringMap(fArray, fMap); + CalculateCheckSum(fArray, checksum); + + testFullUpdate(fMap, &checksum); + } + + // Apply a partial update with removal indice array larger than + // old prefix set(fArray). This should cause an error. + { + _PrefixArray pArray; + PrefixStringMap pMap; + nsTArray<uint32_t> removal; + nsTArray<TableUpdate*> tableUpdates; + + CreateRandomSortedPrefixArray(200, 4, 32, pArray); + RemoveIntersection(fArray, pArray); + PrefixArrayToPrefixStringMap(pArray, pMap); + + for (uint32_t i = 0; i < fArray.Length() + 1 ;i++) { + removal.AppendElement(i); + } + + GenerateUpdateData(false, pMap, &removal, nullptr, tableUpdates); + testUpdateFail(tableUpdates); + } + + Clear(); +} + +TEST(UrlClassifierTableUpdateV4, ChecksumMismatch) +{ + // Apply a full update first + { + _PrefixArray fArray; + PrefixStringMap fMap; + nsCString checksum; + + CreateRandomSortedPrefixArray(1000, 4, 32, fArray); + PrefixArrayToPrefixStringMap(fArray, fMap); + CalculateCheckSum(fArray, checksum); + + testFullUpdate(fMap, &checksum); + } + + // Apply a partial update with incorrect checksum + { + _PrefixArray pArray; + PrefixStringMap pMap; + nsCString checksum; + nsTArray<TableUpdate*> tableUpdates; + + CreateRandomSortedPrefixArray(200, 4, 32, pArray); + PrefixArrayToPrefixStringMap(pArray, pMap); + + // Checksum should be calculated with both old prefix set and add prefix set, + // here we only calculate checksum with add prefix set to check if applyUpdate + // will return failure. + CalculateCheckSum(pArray, checksum); + + GenerateUpdateData(false, pMap, nullptr, &checksum, tableUpdates); + testUpdateFail(tableUpdates); + } + + Clear(); +} + +TEST(UrlClassifierTableUpdateV4, ApplyUpdateThenLoad) +{ + // Apply update with checksum + { + _PrefixArray fArray; + PrefixStringMap fMap; + nsCString checksum; + + CreateRandomSortedPrefixArray(1000, 4, 32, fArray); + PrefixArrayToPrefixStringMap(fArray, fMap); + CalculateCheckSum(fArray, checksum); + + testFullUpdate(fMap, &checksum); + + // Open lookup cache will load prefix set and verify the checksum + testOpenLookupCache(); + } + + Clear(); + + // Apply update without checksum + { + _PrefixArray fArray; + PrefixStringMap fMap; + + CreateRandomSortedPrefixArray(1000, 4, 32, fArray); + PrefixArrayToPrefixStringMap(fArray, fMap); + + testFullUpdate(fMap, nullptr); + + testOpenLookupCache(); + } + + Clear(); +} + +// This test is used to avoid an eror from nsICryptoHash +TEST(UrlClassifierTableUpdateV4, ApplyUpdateWithFixedChecksum) +{ + _PrefixArray fArray = { _Prefix("enus"), _Prefix("apollo"), _Prefix("mars"), + _Prefix("Hecatonchires cyclopes"), + _Prefix("vesta"), _Prefix("neptunus"), _Prefix("jupiter"), + _Prefix("diana"), _Prefix("minerva"), _Prefix("ceres"), + _Prefix("Aidos,Adephagia,Adikia,Aletheia"), + _Prefix("hecatonchires"), _Prefix("alcyoneus"), _Prefix("hades"), + _Prefix("vulcanus"), _Prefix("juno"), _Prefix("mercury"), + _Prefix("Stheno, Euryale and Medusa") + }; + fArray.Sort(); + + PrefixStringMap fMap; + PrefixArrayToPrefixStringMap(fArray, fMap); + + nsCString checksum("\xae\x18\x94\xd7\xd0\x83\x5f\xc1" + "\x58\x59\x5c\x2c\x72\xb9\x6e\x5e" + "\xf4\xe8\x0a\x6b\xff\x5e\x6b\x81" + "\x65\x34\x06\x16\x06\x59\xa0\x67"); + + testFullUpdate(fMap, &checksum); + + // Open lookup cache will load prefix set and verify the checksum + testOpenLookupCache(); + + Clear(); +} + diff --git a/toolkit/components/url-classifier/tests/gtest/TestUrlClassifierUtils.cpp b/toolkit/components/url-classifier/tests/gtest/TestUrlClassifierUtils.cpp new file mode 100644 index 000000000..fa5ce4f56 --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/TestUrlClassifierUtils.cpp @@ -0,0 +1,276 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <stdio.h> +#include <ctype.h> + +#include <mozilla/RefPtr.h> +#include "nsString.h" +#include "nsEscape.h" +#include "nsUrlClassifierUtils.h" +#include "stdlib.h" +#include "gtest/gtest.h" + +static char int_to_hex_digit(int32_t i) { + NS_ASSERTION((i >= 0) && (i <= 15), "int too big in int_to_hex_digit"); + return static_cast<char>(((i < 10) ? (i + '0') : ((i - 10) + 'A'))); +} + +static void CheckEquals(nsCString& expected, nsCString& actual) +{ + ASSERT_TRUE((expected).Equals((actual))); +} + +void TestUnescapeHelper(const char* in, const char* expected) +{ + nsCString out, strIn(in), strExp(expected); + + NS_UnescapeURL(strIn.get(), strIn.Length(), esc_AlwaysCopy, out); + CheckEquals(strExp, out); +} + +// Make sure Unescape from nsEncode.h's unescape does what the server does. +TEST(UrlClassifierUtils, Unescape) +{ + // test empty string + TestUnescapeHelper("\0", "\0"); + + // Test docoding of all characters. + nsCString allCharsEncoded, allCharsEncodedLowercase, allCharsAsString; + for (int32_t i = 1; i < 256; ++i) { + allCharsEncoded.Append('%'); + allCharsEncoded.Append(int_to_hex_digit(i / 16)); + allCharsEncoded.Append((int_to_hex_digit(i % 16))); + + allCharsEncodedLowercase.Append('%'); + allCharsEncodedLowercase.Append(tolower(int_to_hex_digit(i / 16))); + allCharsEncodedLowercase.Append(tolower(int_to_hex_digit(i % 16))); + + allCharsAsString.Append(static_cast<char>(i)); + } + + nsCString out; + NS_UnescapeURL(allCharsEncoded.get(), + allCharsEncoded.Length(), + esc_AlwaysCopy, + out); + + CheckEquals(allCharsAsString, out); + + out.Truncate(); + NS_UnescapeURL(allCharsEncodedLowercase.get(), + allCharsEncodedLowercase.Length(), + esc_AlwaysCopy, + out); + CheckEquals(allCharsAsString, out); + + // Test %-related edge cases + TestUnescapeHelper("%", "%"); + TestUnescapeHelper("%xx", "%xx"); + TestUnescapeHelper("%%", "%%"); + TestUnescapeHelper("%%%", "%%%"); + TestUnescapeHelper("%%%%", "%%%%"); + TestUnescapeHelper("%1", "%1"); + TestUnescapeHelper("%1z", "%1z"); + TestUnescapeHelper("a%1z", "a%1z"); + TestUnescapeHelper("abc%d%e%fg%hij%klmno%", "abc%d%e%fg%hij%klmno%"); + + // A few more tests + TestUnescapeHelper("%25", "%"); + TestUnescapeHelper("%25%32%35", "%25"); +} + +void TestEncodeHelper(const char* in, const char* expected) +{ + nsCString out, strIn(in), strExp(expected); + RefPtr<nsUrlClassifierUtils> utils = new nsUrlClassifierUtils; + utils->Init(); + + utils->SpecialEncode(strIn, true, out); + CheckEquals(strExp, out); +} + +TEST(UrlClassifierUtils, Enc) +{ + // Test empty string + TestEncodeHelper("", ""); + + // Test that all characters we shouldn't encode ([33-36],[38,126]) are not. + nsCString noenc; + for (int32_t i = 33; i < 127; i++) { + if (i != 37) { // skip % + noenc.Append(static_cast<char>(i)); + } + } + RefPtr<nsUrlClassifierUtils> utils = new nsUrlClassifierUtils; + utils->Init(); + nsCString out; + utils->SpecialEncode(noenc, false, out); + CheckEquals(noenc, out); + + // Test that all the chars that we should encode [0,32],37,[127,255] are + nsCString yesAsString, yesExpectedString; + for (int32_t i = 1; i < 256; i++) { + if (i < 33 || i == 37 || i > 126) { + yesAsString.Append(static_cast<char>(i)); + yesExpectedString.Append('%'); + yesExpectedString.Append(int_to_hex_digit(i / 16)); + yesExpectedString.Append(int_to_hex_digit(i % 16)); + } + } + + out.Truncate(); + utils->SpecialEncode(yesAsString, false, out); + CheckEquals(yesExpectedString, out); + + TestEncodeHelper("blah//blah", "blah/blah"); +} + +void TestCanonicalizeHelper(const char* in, const char* expected) +{ + nsCString out, strIn(in), strExp(expected); + RefPtr<nsUrlClassifierUtils> utils = new nsUrlClassifierUtils; + utils->Init(); + + utils->CanonicalizePath(strIn, out); + CheckEquals(strExp, out); +} + +TEST(UrlClassifierUtils, Canonicalize) +{ + // Test repeated %-decoding. Note: %25 --> %, %32 --> 2, %35 --> 5 + TestCanonicalizeHelper("%25", "%25"); + TestCanonicalizeHelper("%25%32%35", "%25"); + TestCanonicalizeHelper("asdf%25%32%35asd", "asdf%25asd"); + TestCanonicalizeHelper("%%%25%32%35asd%%", "%25%25%25asd%25%25"); + TestCanonicalizeHelper("%25%32%35%25%32%35%25%32%35", "%25%25%25"); + TestCanonicalizeHelper("%25", "%25"); + TestCanonicalizeHelper("%257Ea%2521b%2540c%2523d%2524e%25f%255E00%252611%252A22%252833%252944_55%252B", + "~a!b@c#d$e%25f^00&11*22(33)44_55+"); + + TestCanonicalizeHelper("", ""); + TestCanonicalizeHelper("%31%36%38%2e%31%38%38%2e%39%39%2e%32%36/%2E%73%65%63%75%72%65/%77%77%77%2E%65%62%61%79%2E%63%6F%6D/", + "168.188.99.26/.secure/www.ebay.com/"); + TestCanonicalizeHelper("195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/", + "195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/"); + // Added in bug 489455. %00 should no longer be changed to %01. + TestCanonicalizeHelper("%00", "%00"); +} + +void TestParseIPAddressHelper(const char *in, const char *expected) +{ + nsCString out, strIn(in), strExp(expected); + RefPtr<nsUrlClassifierUtils> utils = new nsUrlClassifierUtils; + utils->Init(); + + utils->ParseIPAddress(strIn, out); + CheckEquals(strExp, out); +} + +TEST(UrlClassifierUtils, ParseIPAddress) +{ + TestParseIPAddressHelper("123.123.0.0.1", ""); + TestParseIPAddressHelper("255.0.0.1", "255.0.0.1"); + TestParseIPAddressHelper("12.0x12.01234", "12.18.2.156"); + TestParseIPAddressHelper("276.2.3", "20.2.0.3"); + TestParseIPAddressHelper("012.034.01.055", "10.28.1.45"); + TestParseIPAddressHelper("0x12.0x43.0x44.0x01", "18.67.68.1"); + TestParseIPAddressHelper("167838211", "10.1.2.3"); + TestParseIPAddressHelper("3279880203", "195.127.0.11"); + TestParseIPAddressHelper("0x12434401", "18.67.68.1"); + TestParseIPAddressHelper("413960661", "24.172.137.213"); + TestParseIPAddressHelper("03053104725", "24.172.137.213"); + TestParseIPAddressHelper("030.0254.0x89d5", "24.172.137.213"); + TestParseIPAddressHelper("1.234.4.0377", "1.234.4.255"); + TestParseIPAddressHelper("1.2.3.00x0", ""); + TestParseIPAddressHelper("10.192.95.89 xy", "10.192.95.89"); + TestParseIPAddressHelper("10.192.95.89 xyz", ""); + TestParseIPAddressHelper("1.2.3.0x0", "1.2.3.0"); + TestParseIPAddressHelper("1.2.3.4", "1.2.3.4"); +} + +void TestCanonicalNumHelper(const char *in, uint32_t bytes, + bool allowOctal, const char *expected) +{ + nsCString out, strIn(in), strExp(expected); + RefPtr<nsUrlClassifierUtils> utils = new nsUrlClassifierUtils; + utils->Init(); + + utils->CanonicalNum(strIn, bytes, allowOctal, out); + CheckEquals(strExp, out); +} + +TEST(UrlClassifierUtils, CanonicalNum) +{ + TestCanonicalNumHelper("", 1, true, ""); + TestCanonicalNumHelper("10", 0, true, ""); + TestCanonicalNumHelper("45", 1, true, "45"); + TestCanonicalNumHelper("0x10", 1, true, "16"); + TestCanonicalNumHelper("367", 2, true, "1.111"); + TestCanonicalNumHelper("012345", 3, true, "0.20.229"); + TestCanonicalNumHelper("0173", 1, true, "123"); + TestCanonicalNumHelper("09", 1, false, "9"); + TestCanonicalNumHelper("0x120x34", 2, true, ""); + TestCanonicalNumHelper("0x12fc", 2, true, "18.252"); + TestCanonicalNumHelper("3279880203", 4, true, "195.127.0.11"); + TestCanonicalNumHelper("0x0000059", 1, true, "89"); + TestCanonicalNumHelper("0x00000059", 1, true, "89"); + TestCanonicalNumHelper("0x0000067", 1, true, "103"); +} + +void TestHostnameHelper(const char *in, const char *expected) +{ + nsCString out, strIn(in), strExp(expected); + RefPtr<nsUrlClassifierUtils> utils = new nsUrlClassifierUtils; + utils->Init(); + + utils->CanonicalizeHostname(strIn, out); + CheckEquals(strExp, out); +} + +TEST(UrlClassifierUtils, Hostname) +{ + TestHostnameHelper("abcd123;[]", "abcd123;[]"); + TestHostnameHelper("abc.123", "abc.123"); + TestHostnameHelper("abc..123", "abc.123"); + TestHostnameHelper("trailing.", "trailing"); + TestHostnameHelper("i love trailing dots....", "i%20love%20trailing%20dots"); + TestHostnameHelper(".leading", "leading"); + TestHostnameHelper("..leading", "leading"); + TestHostnameHelper(".dots.", "dots"); + TestHostnameHelper(".both.", "both"); + TestHostnameHelper(".both..", "both"); + TestHostnameHelper("..both.", "both"); + TestHostnameHelper("..both..", "both"); + TestHostnameHelper("..a.b.c.d..", "a.b.c.d"); + TestHostnameHelper("..127.0.0.1..", "127.0.0.1"); + TestHostnameHelper("asdf!@#$a", "asdf!@#$a"); + TestHostnameHelper("AB CD 12354", "ab%20cd%2012354"); + TestHostnameHelper("\1\2\3\4\112\177", "%01%02%03%04j%7F"); + TestHostnameHelper("<>.AS/-+", "<>.as/-+"); + // Added in bug 489455. %00 should no longer be changed to %01. + TestHostnameHelper("%00", "%00"); +} + +TEST(UrlClassifierUtils, LongHostname) +{ + static const int kTestSize = 1024 * 150; + char *str = static_cast<char*>(malloc(kTestSize + 1)); + memset(str, 'x', kTestSize); + str[kTestSize] = '\0'; + + RefPtr<nsUrlClassifierUtils> utils = new nsUrlClassifierUtils; + utils->Init(); + + nsAutoCString out; + nsDependentCString in(str); + PRIntervalTime clockStart = PR_IntervalNow(); + utils->CanonicalizeHostname(in, out); + PRIntervalTime clockEnd = PR_IntervalNow(); + + CheckEquals(in, out); + + printf("CanonicalizeHostname on long string (%dms)\n", + PR_IntervalToMilliseconds(clockEnd - clockStart)); +} diff --git a/toolkit/components/url-classifier/tests/gtest/TestVariableLengthPrefixSet.cpp b/toolkit/components/url-classifier/tests/gtest/TestVariableLengthPrefixSet.cpp new file mode 100644 index 000000000..9e380a9d3 --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/TestVariableLengthPrefixSet.cpp @@ -0,0 +1,559 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <mozilla/RefPtr.h> +#include "nsString.h" +#include "nsTArray.h" +#include "nsClassHashtable.h" +#include "VariableLengthPrefixSet.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsIFile.h" +#include "gtest/gtest.h" + +using namespace mozilla::safebrowsing; + +typedef nsCString _Prefix; +typedef nsTArray<_Prefix> _PrefixArray; + +// Create fullhash by appending random characters. +static nsCString* CreateFullHash(const nsACString& in) +{ + nsCString* out = new nsCString(in); + out->SetLength(32); + for (size_t i = in.Length(); i < 32; i++) { + out->SetCharAt(char(rand() % 256), i); + } + + return out; +} + +// This function generate N prefixes with size between MIN and MAX. +// The output array will not be cleared, random result will append to it +static void RandomPrefixes(uint32_t N, uint32_t MIN, uint32_t MAX, _PrefixArray& array) +{ + array.SetCapacity(array.Length() + N); + + uint32_t range = (MAX - MIN + 1); + + for (uint32_t i = 0; i < N; i++) { + uint32_t prefixSize = (rand() % range) + MIN; + _Prefix prefix; + prefix.SetLength(prefixSize); + + bool added = false; + while(!added) { + char* dst = prefix.BeginWriting(); + for (uint32_t j = 0; j < prefixSize; j++) { + dst[j] = rand() % 256; + } + + if (!array.Contains(prefix)) { + array.AppendElement(prefix); + added = true; + } + } + } +} + +static void CheckContent(VariableLengthPrefixSet* pset, + PrefixStringMap& expected) +{ + PrefixStringMap vlPSetMap; + pset->GetPrefixes(vlPSetMap); + + for (auto iter = vlPSetMap.Iter(); !iter.Done(); iter.Next()) { + nsCString* expectedPrefix = expected.Get(iter.Key()); + nsCString* resultPrefix = iter.Data(); + + ASSERT_TRUE(resultPrefix->Equals(*expectedPrefix)); + } +} + +// This test loops through all the prefixes and converts each prefix to +// fullhash by appending random characters, each converted fullhash +// should at least match its original length in the prefixSet. +static void DoExpectedLookup(VariableLengthPrefixSet* pset, + _PrefixArray& array) +{ + uint32_t matchLength = 0; + for (uint32_t i = 0; i < array.Length(); i++) { + const nsCString& prefix = array[i]; + UniquePtr<nsCString> fullhash(CreateFullHash(prefix)); + + // Find match for prefix-generated full hash + pset->Matches(*fullhash, &matchLength); + MOZ_ASSERT(matchLength != 0); + + if (matchLength != prefix.Length()) { + // Return match size is not the same as prefix size. + // In this case it could be because the generated fullhash match other + // prefixes, check if this prefix exist. + bool found = false; + + for (uint32_t j = 0; j < array.Length(); j++) { + if (array[j].Length() != matchLength) { + continue; + } + + if (0 == memcmp(fullhash->BeginReading(), + array[j].BeginReading(), + matchLength)) { + found = true; + break; + } + } + ASSERT_TRUE(found); + } + } +} + +static void DoRandomLookup(VariableLengthPrefixSet* pset, + uint32_t N, + _PrefixArray& array) +{ + for (uint32_t i = 0; i < N; i++) { + // Random 32-bytes test fullhash + char buf[32]; + for (uint32_t j = 0; j < 32; j++) { + buf[j] = (char)(rand() % 256); + } + + // Get the expected result. + nsTArray<uint32_t> expected; + for (uint32_t j = 0; j < array.Length(); j++) { + const nsACString& str = array[j]; + if (0 == memcmp(buf, str.BeginReading(), str.Length())) { + expected.AppendElement(str.Length()); + } + } + + uint32_t matchLength = 0; + pset->Matches(nsDependentCSubstring(buf, 32), &matchLength); + + ASSERT_TRUE(expected.IsEmpty() ? !matchLength : expected.Contains(matchLength)); + } +} + +static void SetupPrefixMap(const _PrefixArray& array, + PrefixStringMap& map) +{ + map.Clear(); + + // Buckets are keyed by prefix length and contain an array of + // all prefixes of that length. + nsClassHashtable<nsUint32HashKey, _PrefixArray> table; + + for (uint32_t i = 0; i < array.Length(); i++) { + _PrefixArray* prefixes = table.Get(array[i].Length()); + if (!prefixes) { + prefixes = new _PrefixArray(); + table.Put(array[i].Length(), prefixes); + } + + prefixes->AppendElement(array[i]); + } + + // The resulting map entries will be a concatenation of all + // prefix data for the prefixes of a given size. + for (auto iter = table.Iter(); !iter.Done(); iter.Next()) { + uint32_t size = iter.Key(); + uint32_t count = iter.Data()->Length(); + + _Prefix* str = new _Prefix(); + str->SetLength(size * count); + + char* dst = str->BeginWriting(); + + iter.Data()->Sort(); + for (uint32_t i = 0; i < count; i++) { + memcpy(dst, iter.Data()->ElementAt(i).get(), size); + dst += size; + } + + map.Put(size, str); + } +} + + +// Test setting prefix set with only 4-bytes prefixes +TEST(VariableLengthPrefixSet, FixedLengthSet) +{ + srand(time(nullptr)); + + RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet; + pset->Init(NS_LITERAL_CSTRING("test")); + + PrefixStringMap map; + _PrefixArray array = { _Prefix("alph"), _Prefix("brav"), _Prefix("char"), + _Prefix("delt"), _Prefix("echo"), _Prefix("foxt"), + }; + + SetupPrefixMap(array, map); + pset->SetPrefixes(map); + + DoExpectedLookup(pset, array); + + DoRandomLookup(pset, 1000, array); + + CheckContent(pset, map); + + // Run random test + array.Clear(); + map.Clear(); + + RandomPrefixes(1500, 4, 4, array); + + SetupPrefixMap(array, map); + pset->SetPrefixes(map); + + DoExpectedLookup(pset, array); + + DoRandomLookup(pset, 1000, array); + + CheckContent(pset, map); +} + +// Test setting prefix set with only 5~32 bytes prefixes +TEST(VariableLengthPrefixSet, VariableLengthSet) +{ + RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet; + pset->Init(NS_LITERAL_CSTRING("test")); + + PrefixStringMap map; + _PrefixArray array = { _Prefix("bravo"), _Prefix("charlie"), _Prefix("delta"), + _Prefix("EchoEchoEchoEchoEcho"), _Prefix("foxtrot"), + _Prefix("GolfGolfGolfGolfGolfGolfGolfGolf"), + _Prefix("hotel"), _Prefix("november"), + _Prefix("oscar"), _Prefix("quebec"), _Prefix("romeo"), + _Prefix("sierrasierrasierrasierrasierra"), + _Prefix("Tango"), _Prefix("whiskey"), _Prefix("yankee"), + _Prefix("ZuluZuluZuluZulu") + }; + + SetupPrefixMap(array, map); + pset->SetPrefixes(map); + + DoExpectedLookup(pset, array); + + DoRandomLookup(pset, 1000, array); + + CheckContent(pset, map); + + // Run random test + array.Clear(); + map.Clear(); + + RandomPrefixes(1500, 5, 32, array); + + SetupPrefixMap(array, map); + pset->SetPrefixes(map); + + DoExpectedLookup(pset, array); + + DoRandomLookup(pset, 1000, array); + + CheckContent(pset, map); + +} + +// Test setting prefix set with both 4-bytes prefixes and 5~32 bytes prefixes +TEST(VariableLengthPrefixSet, MixedPrefixSet) +{ + RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet; + pset->Init(NS_LITERAL_CSTRING("test")); + + PrefixStringMap map; + _PrefixArray array = { _Prefix("enus"), _Prefix("apollo"), _Prefix("mars"), + _Prefix("Hecatonchires cyclopes"), + _Prefix("vesta"), _Prefix("neptunus"), _Prefix("jupiter"), + _Prefix("diana"), _Prefix("minerva"), _Prefix("ceres"), + _Prefix("Aidos,Adephagia,Adikia,Aletheia"), + _Prefix("hecatonchires"), _Prefix("alcyoneus"), _Prefix("hades"), + _Prefix("vulcanus"), _Prefix("juno"), _Prefix("mercury"), + _Prefix("Stheno, Euryale and Medusa") + }; + + SetupPrefixMap(array, map); + pset->SetPrefixes(map); + + DoExpectedLookup(pset, array); + + DoRandomLookup(pset, 1000, array); + + CheckContent(pset, map); + + // Run random test + array.Clear(); + map.Clear(); + + RandomPrefixes(1500, 4, 32, array); + + SetupPrefixMap(array, map); + pset->SetPrefixes(map); + + DoExpectedLookup(pset, array); + + DoRandomLookup(pset, 1000, array); + + CheckContent(pset, map); +} + +// Test resetting prefix set +TEST(VariableLengthPrefixSet, ResetPrefix) +{ + RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet; + pset->Init(NS_LITERAL_CSTRING("test")); + + // First prefix set + _PrefixArray array1 = { _Prefix("Iceland"), _Prefix("Peru"), _Prefix("Mexico"), + _Prefix("Australia"), _Prefix("Japan"), _Prefix("Egypt"), + _Prefix("America"), _Prefix("Finland"), _Prefix("Germany"), + _Prefix("Italy"), _Prefix("France"), _Prefix("Taiwan"), + }; + { + PrefixStringMap map; + + SetupPrefixMap(array1, map); + pset->SetPrefixes(map); + + DoExpectedLookup(pset, array1); + } + + // Second + _PrefixArray array2 = { _Prefix("Pikachu"), _Prefix("Bulbasaur"), _Prefix("Charmander"), + _Prefix("Blastoise"), _Prefix("Pidgey"), _Prefix("Mewtwo"), + _Prefix("Jigglypuff"), _Prefix("Persian"), _Prefix("Tentacool"), + _Prefix("Onix"), _Prefix("Eevee"), _Prefix("Jynx"), + }; + { + PrefixStringMap map; + + SetupPrefixMap(array2, map); + pset->SetPrefixes(map); + + DoExpectedLookup(pset, array2); + } + + // Should not match any of the first prefix set + uint32_t matchLength = 0; + for (uint32_t i = 0; i < array1.Length(); i++) { + UniquePtr<nsACString> fullhash(CreateFullHash(array1[i])); + + pset->Matches(*fullhash, &matchLength); + ASSERT_TRUE(matchLength == 0); + } +} + +// Test only set one 4-bytes prefix and one full-length prefix +TEST(VariableLengthPrefixSet, TinyPrefixSet) +{ + RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet; + pset->Init(NS_LITERAL_CSTRING("test")); + + PrefixStringMap map; + _PrefixArray array = { _Prefix("AAAA"), + _Prefix("11112222333344445555666677778888"), + }; + + SetupPrefixMap(array, map); + pset->SetPrefixes(map); + + DoExpectedLookup(pset, array); + + DoRandomLookup(pset, 1000, array); + + CheckContent(pset, map); +} + +// Test empty prefix set and IsEmpty function +TEST(VariableLengthPrefixSet, EmptyPrefixSet) +{ + RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet; + pset->Init(NS_LITERAL_CSTRING("test")); + + bool empty; + pset->IsEmpty(&empty); + ASSERT_TRUE(empty); + + PrefixStringMap map; + _PrefixArray array1; + + // Lookup an empty array should never match + DoRandomLookup(pset, 100, array1); + + // Insert an 4-bytes prefix, then IsEmpty should return false + _PrefixArray array2 = { _Prefix("test") }; + SetupPrefixMap(array2, map); + pset->SetPrefixes(map); + + pset->IsEmpty(&empty); + ASSERT_TRUE(!empty); + + _PrefixArray array3 = { _Prefix("test variable length") }; + + // Insert an 5~32 bytes prefix, then IsEmpty should return false + SetupPrefixMap(array3, map); + pset->SetPrefixes(map); + + pset->IsEmpty(&empty); + ASSERT_TRUE(!empty); +} + +// Test prefix size should only between 4~32 bytes +TEST(VariableLengthPrefixSet, MinMaxPrefixSet) +{ + RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet; + pset->Init(NS_LITERAL_CSTRING("test")); + + PrefixStringMap map; + { + _PrefixArray array = { _Prefix("1234"), + _Prefix("ABCDEFGHIJKKMNOP"), + _Prefix("1aaa2bbb3ccc4ddd5eee6fff7ggg8hhh") }; + + SetupPrefixMap(array, map); + nsresult rv = pset->SetPrefixes(map); + ASSERT_TRUE(rv == NS_OK); + } + + // Prefix size less than 4-bytes should fail + { + _PrefixArray array = { _Prefix("123") }; + + SetupPrefixMap(array, map); + nsresult rv = pset->SetPrefixes(map); + ASSERT_TRUE(NS_FAILED(rv)); + } + + // Prefix size greater than 32-bytes should fail + { + _PrefixArray array = { _Prefix("1aaa2bbb3ccc4ddd5eee6fff7ggg8hhh9") }; + + SetupPrefixMap(array, map); + nsresult rv = pset->SetPrefixes(map); + ASSERT_TRUE(NS_FAILED(rv)); + } +} + +// Test save then load prefix set with only 4-bytes prefixes +TEST(VariableLengthPrefixSet, LoadSaveFixedLengthPrefixSet) +{ + RefPtr<VariableLengthPrefixSet> save = new VariableLengthPrefixSet; + save->Init(NS_LITERAL_CSTRING("test-save")); + + _PrefixArray array; + RandomPrefixes(10000, 4, 4, array); + + PrefixStringMap map; + SetupPrefixMap(array, map); + save->SetPrefixes(map); + + DoExpectedLookup(save, array); + + DoRandomLookup(save, 1000, array); + + CheckContent(save, map); + + nsCOMPtr<nsIFile> file; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(file)); + file->Append(NS_LITERAL_STRING("test.vlpset")); + + save->StoreToFile(file); + + RefPtr<VariableLengthPrefixSet> load = new VariableLengthPrefixSet; + load->Init(NS_LITERAL_CSTRING("test-load")); + + load->LoadFromFile(file); + + DoExpectedLookup(load, array); + + DoRandomLookup(load, 1000, array); + + CheckContent(load, map); + + file->Remove(false); +} + +// Test save then load prefix set with only 5~32 bytes prefixes +TEST(VariableLengthPrefixSet, LoadSaveVariableLengthPrefixSet) +{ + RefPtr<VariableLengthPrefixSet> save = new VariableLengthPrefixSet; + save->Init(NS_LITERAL_CSTRING("test-save")); + + _PrefixArray array; + RandomPrefixes(10000, 5, 32, array); + + PrefixStringMap map; + SetupPrefixMap(array, map); + save->SetPrefixes(map); + + DoExpectedLookup(save, array); + + DoRandomLookup(save, 1000, array); + + CheckContent(save, map); + + nsCOMPtr<nsIFile> file; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(file)); + file->Append(NS_LITERAL_STRING("test.vlpset")); + + save->StoreToFile(file); + + RefPtr<VariableLengthPrefixSet> load = new VariableLengthPrefixSet; + load->Init(NS_LITERAL_CSTRING("test-load")); + + load->LoadFromFile(file); + + DoExpectedLookup(load, array); + + DoRandomLookup(load, 1000, array); + + CheckContent(load, map); + + file->Remove(false); +} + +// Test save then load prefix with both 4 bytes prefixes and 5~32 bytes prefixes +TEST(VariableLengthPrefixSet, LoadSavePrefixSet) +{ + RefPtr<VariableLengthPrefixSet> save = new VariableLengthPrefixSet; + save->Init(NS_LITERAL_CSTRING("test-save")); + + // Try to simulate the real case that most prefixes are 4bytes + _PrefixArray array; + RandomPrefixes(20000, 4, 4, array); + RandomPrefixes(1000, 5, 32, array); + + PrefixStringMap map; + SetupPrefixMap(array, map); + save->SetPrefixes(map); + + DoExpectedLookup(save, array); + + DoRandomLookup(save, 1000, array); + + CheckContent(save, map); + + nsCOMPtr<nsIFile> file; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(file)); + file->Append(NS_LITERAL_STRING("test.vlpset")); + + save->StoreToFile(file); + + RefPtr<VariableLengthPrefixSet> load = new VariableLengthPrefixSet; + load->Init(NS_LITERAL_CSTRING("test-load")); + + load->LoadFromFile(file); + + DoExpectedLookup(load, array); + + DoRandomLookup(load, 1000, array); + + CheckContent(load, map); + + file->Remove(false); +} diff --git a/toolkit/components/url-classifier/tests/gtest/moz.build b/toolkit/components/url-classifier/tests/gtest/moz.build new file mode 100644 index 000000000..e66af9024 --- /dev/null +++ b/toolkit/components/url-classifier/tests/gtest/moz.build @@ -0,0 +1,27 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +LOCAL_INCLUDES += [ + '../..', +] + +UNIFIED_SOURCES += [ + 'Common.cpp', + 'TestChunkSet.cpp', + 'TestFailUpdate.cpp', + 'TestLookupCacheV4.cpp', + 'TestPerProviderDirectory.cpp', + 'TestProtocolParser.cpp', + 'TestRiceDeltaDecoder.cpp', + 'TestSafebrowsingHash.cpp', + 'TestSafeBrowsingProtobuf.cpp', + 'TestTable.cpp', + 'TestUrlClassifierTableUpdateV4.cpp', + 'TestUrlClassifierUtils.cpp', + 'TestVariableLengthPrefixSet.cpp', +] + +FINAL_LIBRARY = 'xul-gtest' diff --git a/toolkit/components/url-classifier/tests/jar.mn b/toolkit/components/url-classifier/tests/jar.mn new file mode 100644 index 000000000..2264c2896 --- /dev/null +++ b/toolkit/components/url-classifier/tests/jar.mn @@ -0,0 +1,2 @@ +toolkit.jar: + content/global/url-classifier/unittests.xul (unittests.xul) diff --git a/toolkit/components/url-classifier/tests/mochitest/.eslintrc.js b/toolkit/components/url-classifier/tests/mochitest/.eslintrc.js new file mode 100644 index 000000000..58b3df4a7 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/.eslintrc.js @@ -0,0 +1,8 @@ +"use strict"; + +module.exports = { + "extends": [ + "../../../../../testing/mochitest/mochitest.eslintrc.js", + "../../../../../testing/mochitest/chrome.eslintrc.js" + ] +}; diff --git a/toolkit/components/url-classifier/tests/mochitest/allowlistAnnotatedFrame.html b/toolkit/components/url-classifier/tests/mochitest/allowlistAnnotatedFrame.html new file mode 100644 index 000000000..9aae1b841 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/allowlistAnnotatedFrame.html @@ -0,0 +1,144 @@ +<html> +<head> +<title></title> + +<script type="text/javascript"> + +// Modified by evil.js +var scriptItem; + +var scriptItem1 = "untouched"; +var imageItem1 = "untouched"; +var frameItem1 = "untouched"; +var scriptItem2 = "untouched"; +var imageItem2 = "untouched"; +var frameItem2 = "untouched"; +var xhrItem = "untouched"; +var fetchItem = "untouched"; +var mediaItem1 = "untouched"; + +function checkLoads() { + window.parent.is(scriptItem1, "spoiled", "Should not block tracking js 1"); + window.parent.is(scriptItem2, "spoiled", "Should not block tracking js 2"); + window.parent.is(imageItem1, "spoiled", "Should not block tracking img 1"); + window.parent.is(imageItem2, "spoiled", "Should not block tracking img 2"); + window.parent.is(frameItem1, "spoiled", "Should not block tracking iframe 1"); + window.parent.is(frameItem2, "spoiled", "Should not block tracking iframe 2"); + window.parent.is(mediaItem1, "loaded", "Should not block tracking video"); + window.parent.is(xhrItem, "loaded", "Should not block tracking XHR"); + window.parent.is(fetchItem, "loaded", "Should not block fetches from tracking domains"); + window.parent.is(window.document.blockedTrackingNodeCount, 0, + "No elements should be blocked"); + + // End (parent) test. + window.parent.clearPermissions(); + window.parent.SimpleTest.finish(); +} + +var onloadCalled = false; +var xhrFinished = false; +var fetchFinished = false; +var videoLoaded = false; +function loaded(type) { + if (type === "onload") { + onloadCalled = true; + } else if (type === "xhr") { + xhrFinished = true; + } else if (type === "fetch") { + fetchFinished = true; + } else if (type === "video") { + videoLoaded = true; + } + + if (onloadCalled && xhrFinished && fetchFinished && videoLoaded) { + checkLoads(); + } +} +</script> + +</head> + +<body onload="loaded('onload')"> + +<!-- Try loading from a tracking script URI (1) --> +<script id="badscript1" src="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js" onload="scriptItem1 = 'spoiled';"></script> + +<!-- Try loading from a tracking image URI (1) --> +<img id="badimage1" src="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/raptor.jpg" onload="imageItem1 = 'spoiled';"/> + +<!-- Try loading from a tracking frame URI (1) --> +<iframe id="badframe1" src="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/track.html" onload="frameItem1 = 'spoiled';"></iframe> + +<!-- Try loading from a tracking video URI --> +<video id="badmedia1" src="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/vp9.webm"></video> + +<script> +var v = document.getElementById("badmedia1"); +v.addEventListener("loadedmetadata", function() { + mediaItem1 = "loaded"; + loaded("video"); +}, true); +v.addEventListener("error", function() { + mediaItem1 = "error"; + loaded("video"); +}, true); + +// Try loading from a tracking script URI (2) - The loader may follow a +// different path depending on whether the resource is loaded from JS or HTML. +var newScript = document.createElement("script"); +newScript.id = "badscript2"; +newScript.src = "http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js"; +newScript.addEventListener("load", function onload() {scriptItem2 = 'spoiled';}); +document.body.appendChild(newScript); + +/// Try loading from a tracking image URI (2) +var newImage = document.createElement("img"); +newImage.id = "badimage2"; +newImage.src = "http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/raptor.jpg"; +newImage.addEventListener("load", function onload() {imageItem2 = 'spoiled'}); +document.body.appendChild(newImage); + +// Try loading from a tracking iframe URI (2) +var newFrame = document.createElement("iframe"); +newFrame.id = "badframe2"; +newFrame.src = "http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/track.html" +newFrame.addEventListener("load", function onload() {frameItem2 = 'spoiled'}); +document.body.appendChild(newFrame); + +// Try doing an XHR against a tracking domain (bug 1216793) +function reqListener() { + xhrItem = "loaded"; + loaded("xhr"); +} +function transferFailed() { + xhrItem = "failed"; + loaded("xhr"); +} +function transferCanceled() { + xhrItem = "canceled"; + loaded("xhr"); +} +var oReq = new XMLHttpRequest(); +oReq.addEventListener("load", reqListener); +oReq.addEventListener("error", transferFailed); +oReq.addEventListener("abort", transferCanceled); +oReq.open("GET", "http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js"); +oReq.send(); + +// Fetch from a tracking domain +fetch("http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js").then(function(response) { + if(response.ok) { + fetchItem = "loaded"; + loaded("fetch"); + } else { + fetchItem = "badresponse"; + loaded("fetch"); + } + }).catch(function(error) { + fetchItem = "error"; + loaded("fetch"); +}); +</script> +</body> +</html> + diff --git a/toolkit/components/url-classifier/tests/mochitest/bad.css b/toolkit/components/url-classifier/tests/mochitest/bad.css new file mode 100644 index 000000000..f57b36a77 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/bad.css @@ -0,0 +1 @@ +#styleBad { visibility: hidden; } diff --git a/toolkit/components/url-classifier/tests/mochitest/bad.css^headers^ b/toolkit/components/url-classifier/tests/mochitest/bad.css^headers^ new file mode 100644 index 000000000..4030ea1d3 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/bad.css^headers^ @@ -0,0 +1 @@ +Cache-Control: no-store diff --git a/toolkit/components/url-classifier/tests/mochitest/basic.vtt b/toolkit/components/url-classifier/tests/mochitest/basic.vtt new file mode 100644 index 000000000..7781790d0 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/basic.vtt @@ -0,0 +1,27 @@ +WEBVTT +Region: id=testOne lines=2 width=30% +Region: id=testTwo lines=4 width=20% + +1 +00:00.500 --> 00:00.700 region:testOne +This + +2 +00:01.200 --> 00:02.400 region:testTwo +Is + +2.5 +00:02.000 --> 00:03.500 region:testOne +(Over here?!) + +3 +00:02.710 --> 00:02.910 +A + +4 +00:03.217 --> 00:03.989 +Test + +5 +00:03.217 --> 00:03.989 +And more! diff --git a/toolkit/components/url-classifier/tests/mochitest/basic.vtt^headers^ b/toolkit/components/url-classifier/tests/mochitest/basic.vtt^headers^ new file mode 100644 index 000000000..23de552c1 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/basic.vtt^headers^ @@ -0,0 +1 @@ +Access-Control-Allow-Origin: *
\ No newline at end of file diff --git a/toolkit/components/url-classifier/tests/mochitest/bug_1281083.html b/toolkit/components/url-classifier/tests/mochitest/bug_1281083.html new file mode 100644 index 000000000..cd5770177 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/bug_1281083.html @@ -0,0 +1,35 @@ +<html> +<head> +<title></title> + +<script type="text/javascript"> + +var scriptItem = "untouched"; + +function checkLoads() { + // Make sure the javascript did not load. + window.parent.is(scriptItem, "untouched", "Should not load bad javascript"); + + // Call parent.loadTestFrame again to test classification metadata in HTTP + // cache entries. + if (window.parent.firstLoad) { + window.parent.info("Reloading from cache..."); + window.parent.firstLoad = false; + window.parent.loadTestFrame(); + return; + } + + // End (parent) test. + window.parent.SimpleTest.finish(); +} + +</script> + +<!-- Try loading from a malware javascript URI --> +<script type="text/javascript" src="http://bug1281083.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js"></script> + +</head> + +<body onload="checkLoads()"> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/chrome.ini b/toolkit/components/url-classifier/tests/mochitest/chrome.ini new file mode 100644 index 000000000..1652e7421 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/chrome.ini @@ -0,0 +1,23 @@ +[DEFAULT] +skip-if = os == 'android' +support-files = + allowlistAnnotatedFrame.html + classifiedAnnotatedFrame.html + classifiedAnnotatedPBFrame.html + bug_1281083.html + +[test_lookup_system_principal.html] +[test_classified_annotations.html] +tags = trackingprotection +skip-if = os == 'linux' && asan # Bug 1202548 +[test_allowlisted_annotations.html] +tags = trackingprotection +[test_privatebrowsing_trackingprotection.html] +tags = trackingprotection +[test_trackingprotection_bug1157081.html] +tags = trackingprotection +[test_trackingprotection_whitelist.html] +tags = trackingprotection +[test_safebrowsing_bug1272239.html] +[test_donottrack.html] +[test_classifier_changetablepref.html] diff --git a/toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedFrame.html b/toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedFrame.html new file mode 100644 index 000000000..8aab13dd3 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedFrame.html @@ -0,0 +1,213 @@ +<html> +<head> +<title></title> + +<script type="text/javascript"> +"use strict"; + +var scriptItem = "untouched"; +var scriptItem1 = "untouched"; +var scriptItem2 = "untouched"; +var imageItem1 = "untouched"; +var imageItem2 = "untouched"; +var frameItem1 = "untouched"; +var frameItem2 = "untouched"; +var xhrItem = "untouched"; +var fetchItem = "untouched"; +var mediaItem1 = "untouched"; + +var badids = [ + "badscript1", + "badscript2", + "badimage1", + "badimage2", + "badframe1", + "badframe2", + "badmedia1", + "badcss" +]; + +function checkLoads() { + window.parent.is( + scriptItem1, "untouched", "Should not load tracking javascript"); + window.parent.is( + scriptItem2, "untouched", "Should not load tracking javascript (2)"); + + window.parent.is( + imageItem1, "untouched", "Should not load tracking images"); + window.parent.is( + imageItem2, "untouched", "Should not load tracking images (2)"); + + window.parent.is( + frameItem1, "untouched", "Should not load tracking iframes"); + window.parent.is( + frameItem2, "untouched", "Should not load tracking iframes (2)"); + window.parent.is( + mediaItem1, "error", "Should not load tracking videos"); + window.parent.is( + xhrItem, "failed", "Should not load tracking XHRs"); + window.parent.is( + fetchItem, "error", "Should not fetch from tracking URLs"); + + var elt = document.getElementById("styleCheck"); + var style = document.defaultView.getComputedStyle(elt, ""); + window.parent.isnot( + style.visibility, "hidden", "Should not load tracking css"); + + window.parent.is(window.document.blockedTrackingNodeCount, badids.length, + "Should identify all tracking elements"); + + var blockedTrackingNodes = window.document.blockedTrackingNodes; + + // Make sure that every node in blockedTrackingNodes exists in the tree + // (that may not always be the case but do not expect any nodes to disappear + // from the tree here) + var allNodeMatch = true; + for (var i = 0; i < blockedTrackingNodes.length; i++) { + var nodeMatch = false; + for (var j = 0; j < badids.length && !nodeMatch; j++) { + nodeMatch = nodeMatch || + (blockedTrackingNodes[i] == document.getElementById(badids[j])); + } + + allNodeMatch = allNodeMatch && nodeMatch; + } + window.parent.ok(allNodeMatch, + "All annotated nodes are expected in the tree"); + + // Make sure that every node with a badid (see badids) is found in the + // blockedTrackingNodes. This tells us if we are neglecting to annotate + // some nodes + allNodeMatch = true; + for (var j = 0; j < badids.length; j++) { + var nodeMatch = false; + for (var i = 0; i < blockedTrackingNodes.length && !nodeMatch; i++) { + nodeMatch = nodeMatch || + (blockedTrackingNodes[i] == document.getElementById(badids[j])); + } + + if (!nodeMatch) { + console.log(badids[j] + " was not found in blockedTrackingNodes"); + } + allNodeMatch = allNodeMatch && nodeMatch; + } + window.parent.ok(allNodeMatch, + "All tracking nodes are expected to be annotated as such"); + + // Unset prefs, etc. + window.parent.cleanup(); + // End (parent) test. + window.parent.SimpleTest.finish(); +} + +var onloadCalled = false; +var xhrFinished = false; +var fetchFinished = false; +var videoLoaded = false; +function loaded(type) { + if (type === "onload") { + onloadCalled = true; + } else if (type === "xhr") { + xhrFinished = true; + } else if (type === "fetch") { + fetchFinished = true; + } else if (type === "video") { + videoLoaded = true; + } + if (onloadCalled && xhrFinished && fetchFinished && videoLoaded) { + checkLoads(); + } +} +</script> + +<!-- Try loading from a tracking CSS URI --> +<link id="badcss" rel="stylesheet" type="text/css" href="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.css"></link> + +</head> + +<body onload="loaded('onload')"> + +<!-- Try loading from a tracking script URI (1): evil.js onload will have updated the scriptItem variable --> +<script id="badscript1" src="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js" onload="scriptItem1 = scriptItem;"></script> + +<!-- Try loading from a tracking image URI (1) --> +<img id="badimage1" src="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/raptor.jpg?reload=true" onload="imageItem1 = 'spoiled';"/> + +<!-- Try loading from a tracking frame URI (1) --> +<iframe id="badframe1" src="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/track.html" onload="frameItem1 = 'spoiled';"></iframe> + +<!-- Try loading from a tracking video URI --> +<video id="badmedia1" src="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/vp9.webm?reload=true"></video> + +<script> +var v = document.getElementById("badmedia1"); +v.addEventListener("loadedmetadata", function() { + mediaItem1 = "loaded"; + loaded("video"); +}, true); +v.addEventListener("error", function() { + mediaItem1 = "error"; + loaded("video"); +}, true); + +// Try loading from a tracking script URI (2) - The loader may follow a different path depending on whether the resource is loaded from JS or HTML. +var newScript = document.createElement("script"); +newScript.id = "badscript2"; +newScript.src = "http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js"; +newScript.addEventListener("load", function() {scriptItem2 = scriptItem;}); +document.body.appendChild(newScript); + +/// Try loading from a tracking image URI (2) +var newImage = document.createElement("img"); +newImage.id = "badimage2"; +newImage.src = "http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/raptor.jpg?reload=true"; +newImage.addEventListener("load", function() {imageItem2 = 'spoiled'}); +document.body.appendChild(newImage); + +// Try loading from a tracking iframe URI (2) +var newFrame = document.createElement("iframe"); +newFrame.id = "badframe2"; +newFrame.src = "http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/track.html" +newFrame.addEventListener("load", function() {frameItem2 = 'spoiled'}); +document.body.appendChild(newFrame); + +// Try doing an XHR against a tracking domain (bug 1216793) +function reqListener() { + xhrItem = "loaded"; + loaded("xhr"); +} +function transferFailed() { + xhrItem = "failed"; + loaded("xhr"); +} +function transferCanceled() { + xhrItem = "canceled"; + loaded("xhr"); +} +var oReq = new XMLHttpRequest(); +oReq.addEventListener("load", reqListener); +oReq.addEventListener("error", transferFailed); +oReq.addEventListener("abort", transferCanceled); +oReq.open("GET", "http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js"); +oReq.send(); + +// Fetch from a tracking domain +fetch("http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js").then(function(response) { + if(response.ok) { + fetchItem = "loaded"; + loaded("fetch"); + } else { + fetchItem = "badresponse"; + loaded("fetch"); + } + }).catch(function(error) { + fetchItem = "error"; + loaded("fetch"); +}); +</script> + +The following should not be hidden: +<div id="styleCheck">STYLE TEST</div> + +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedPBFrame.html b/toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedPBFrame.html new file mode 100644 index 000000000..f11ec1de3 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedPBFrame.html @@ -0,0 +1,24 @@ +<!DOCTYPE HTML> +<!-- Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ --> +<html> +<head> +<title></title> + +<link id="badcss" rel="stylesheet" type="text/css" href="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.css"></link> + +</head> +<body> + +<script id="badscript" data-touched="not sure" src="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js" onload="this.dataset.touched = 'yes';" onerror="this.dataset.touched = 'no';"></script> + +<script id="goodscript" data-touched="not sure" src="http://itisatracker.org/tests/toolkit/components/url-classifier/tests/mochitest/good.js" onload="this.dataset.touched = 'yes';" onerror="this.dataset.touched = 'no';"></script> + +<!-- The image cache can cache JS handlers, so make sure we use a different URL for raptor.jpg each time --> +<img id="badimage" data-touched="not sure" src="http://tracking.example.com/tests/toolkit/components/url-classifier/tests/mochitest/raptor.jpg?pbmode=test" onload="this.dataset.touched = 'yes';" onerror="this.dataset.touched = 'no';"/> + +The following should not be hidden: +<div id="styleCheck">STYLE TEST</div> + +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/classifierCommon.js b/toolkit/components/url-classifier/tests/mochitest/classifierCommon.js new file mode 100644 index 000000000..49bda38db --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/classifierCommon.js @@ -0,0 +1,112 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { classes: Cc, interfaces: Ci, results: Cr } = Components; + +var dbService = Cc["@mozilla.org/url-classifier/dbservice;1"] + .getService(Ci.nsIUrlClassifierDBService); + +var timer; +function setTimeout(callback, delay) { + timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + timer.initWithCallback({ notify: callback }, + delay, + Ci.nsITimer.TYPE_ONE_SHOT); +} + +function doUpdate(update) { + let listener = { + QueryInterface: function(iid) + { + if (iid.equals(Ci.nsISupports) || + iid.equals(Ci.nsIUrlClassifierUpdateObserver)) + return this; + + throw Cr.NS_ERROR_NO_INTERFACE; + }, + updateUrlRequested: function(url) { }, + streamFinished: function(status) { }, + updateError: function(errorCode) { + sendAsyncMessage("updateError", errorCode); + }, + updateSuccess: function(requestedTimeout) { + sendAsyncMessage("updateSuccess"); + } + }; + + let dbService = Cc["@mozilla.org/url-classifier/dbservice;1"] + .getService(Ci.nsIUrlClassifierDBService); + + try { + dbService.beginUpdate(listener, "test-malware-simple,test-unwanted-simple", ""); + dbService.beginStream("", ""); + dbService.updateStream(update); + dbService.finishStream(); + dbService.finishUpdate(); + } catch(e) { + // beginUpdate may fail if there's an existing update in progress + // retry until success or testcase timeout. + setTimeout(() => { doUpdate(update); }, 1000); + } +} + +function doReload() { + dbService.reloadDatabase(); + + sendAsyncMessage("reloadSuccess"); +} + +// SafeBrowsing.jsm is initialized after mozEntries are added. Add observer +// to receive "finished" event. For the case when this function is called +// after the event had already been notified, we lookup entries to see if +// they are already added to database. +function waitForInit() { + let observerService = Cc["@mozilla.org/observer-service;1"] + .getService(Ci.nsIObserverService); + + observerService.addObserver(function() { + sendAsyncMessage("safeBrowsingInited"); + }, "mozentries-update-finished", false); + + // This url must sync with the table, url in SafeBrowsing.jsm addMozEntries + const table = "test-phish-simple"; + const url = "http://itisatrap.org/firefox/its-a-trap.html"; + + let secMan = Cc["@mozilla.org/scriptsecuritymanager;1"] + .getService(Ci.nsIScriptSecurityManager); + let iosvc = Cc["@mozilla.org/network/io-service;1"] + .getService(Ci.nsIIOService); + + let principal = secMan.createCodebasePrincipal( + iosvc.newURI(url, null, null), {}); + + let listener = { + QueryInterface: function(iid) + { + if (iid.equals(Ci.nsISupports) || + iid.equals(Ci.nsIUrlClassifierUpdateObserver)) + return this; + throw Cr.NS_ERROR_NO_INTERFACE; + }, + + handleEvent: function(value) + { + if (value === table) { + sendAsyncMessage("safeBrowsingInited"); + } + }, + }; + dbService.lookup(principal, table, listener); +} + +addMessageListener("doUpdate", ({ testUpdate }) => { + doUpdate(testUpdate); +}); + +addMessageListener("doReload", () => { + doReload(); +}); + +addMessageListener("waitForInit", () => { + waitForInit(); +}); diff --git a/toolkit/components/url-classifier/tests/mochitest/classifierFrame.html b/toolkit/components/url-classifier/tests/mochitest/classifierFrame.html new file mode 100644 index 000000000..c7923f448 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/classifierFrame.html @@ -0,0 +1,57 @@ +<html> +<head> +<title></title> + +<script type="text/javascript"> + +var scriptItem = "untouched"; + +function checkLoads() { + // Make sure the javascript did not load. + window.parent.is(scriptItem, "untouched", "Should not load bad javascript"); + + // Make sure the css did not load. + var elt = document.getElementById("styleCheck"); + var style = document.defaultView.getComputedStyle(elt, ""); + window.parent.isnot(style.visibility, "hidden", "Should not load bad css"); + + elt = document.getElementById("styleBad"); + style = document.defaultView.getComputedStyle(elt, ""); + window.parent.isnot(style.visibility, "hidden", "Should not load bad css"); + + elt = document.getElementById("styleImport"); + style = document.defaultView.getComputedStyle(elt, ""); + window.parent.isnot(style.visibility, "visible", "Should import clean css"); + + // Call parent.loadTestFrame again to test classification metadata in HTTP + // cache entries. + if (window.parent.firstLoad) { + window.parent.info("Reloading from cache..."); + window.parent.firstLoad = false; + window.parent.loadTestFrame(); + return; + } + + // End (parent) test. + window.parent.SimpleTest.finish(); +} + +</script> + +<!-- Try loading from a malware javascript URI --> +<script type="text/javascript" src="http://malware.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js"></script> + +<!-- Try loading from an uwanted software css URI --> +<link rel="stylesheet" type="text/css" href="http://unwanted.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.css"></link> + +<!-- Try loading a marked-as-malware css through an @import from a clean URI --> +<link rel="stylesheet" type="text/css" href="import.css"></link> +</head> + +<body onload="checkLoads()"> +The following should not be hidden: +<div id="styleCheck">STYLE TEST</div> +<div id="styleBad">STYLE BAD</div> +<div id="styleImport">STYLE IMPORT</div> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/classifierHelper.js b/toolkit/components/url-classifier/tests/mochitest/classifierHelper.js new file mode 100644 index 000000000..973f0c2c4 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/classifierHelper.js @@ -0,0 +1,201 @@ +if (typeof(classifierHelper) == "undefined") { + var classifierHelper = {}; +} + +const CLASSIFIER_COMMON_URL = SimpleTest.getTestFileURL("classifierCommon.js"); +var gScript = SpecialPowers.loadChromeScript(CLASSIFIER_COMMON_URL); + +const ADD_CHUNKNUM = 524; +const SUB_CHUNKNUM = 523; +const HASHLEN = 32; + +const PREFS = { + PROVIDER_LISTS : "browser.safebrowsing.provider.mozilla.lists", + DISALLOW_COMPLETIONS : "urlclassifier.disallow_completions", + PROVIDER_GETHASHURL : "browser.safebrowsing.provider.mozilla.gethashURL" +}; + +// addUrlToDB & removeUrlFromDB are asynchronous, queue the task to ensure +// the callback follow correct order. +classifierHelper._updates = []; + +// Keep urls added to database, those urls should be automatically +// removed after test complete. +classifierHelper._updatesToCleanup = []; + +classifierHelper._initsCB = []; + +// This function return a Promise, promise is resolved when SafeBrowsing.jsm +// is initialized. +classifierHelper.waitForInit = function() { + return new Promise(function(resolve, reject) { + classifierHelper._initsCB.push(resolve); + gScript.sendAsyncMessage("waitForInit"); + }); +} + +// This function is used to allow completion for specific "list", +// some lists like "test-malware-simple" is default disabled to ask for complete. +// "list" is the db we would like to allow it +// "url" is the completion server +classifierHelper.allowCompletion = function(lists, url) { + for (var list of lists) { + // Add test db to provider + var pref = SpecialPowers.getCharPref(PREFS.PROVIDER_LISTS); + pref += "," + list; + SpecialPowers.setCharPref(PREFS.PROVIDER_LISTS, pref); + + // Rename test db so we will not disallow it from completions + pref = SpecialPowers.getCharPref(PREFS.DISALLOW_COMPLETIONS); + pref = pref.replace(list, list + "-backup"); + SpecialPowers.setCharPref(PREFS.DISALLOW_COMPLETIONS, pref); + } + + // Set get hash url + SpecialPowers.setCharPref(PREFS.PROVIDER_GETHASHURL, url); +} + +// Pass { url: ..., db: ... } to add url to database, +// onsuccess/onerror will be called when update complete. +classifierHelper.addUrlToDB = function(updateData) { + return new Promise(function(resolve, reject) { + var testUpdate = ""; + for (var update of updateData) { + var LISTNAME = update.db; + var CHUNKDATA = update.url; + var CHUNKLEN = CHUNKDATA.length; + var HASHLEN = update.len ? update.len : 32; + + classifierHelper._updatesToCleanup.push(update); + testUpdate += + "n:1000\n" + + "i:" + LISTNAME + "\n" + + "ad:1\n" + + "a:" + ADD_CHUNKNUM + ":" + HASHLEN + ":" + CHUNKLEN + "\n" + + CHUNKDATA; + } + + classifierHelper._update(testUpdate, resolve, reject); + }); +} + +// Pass { url: ..., db: ... } to remove url from database, +// onsuccess/onerror will be called when update complete. +classifierHelper.removeUrlFromDB = function(updateData) { + return new Promise(function(resolve, reject) { + var testUpdate = ""; + for (var update of updateData) { + var LISTNAME = update.db; + var CHUNKDATA = ADD_CHUNKNUM + ":" + update.url; + var CHUNKLEN = CHUNKDATA.length; + var HASHLEN = update.len ? update.len : 32; + + testUpdate += + "n:1000\n" + + "i:" + LISTNAME + "\n" + + "s:" + SUB_CHUNKNUM + ":" + HASHLEN + ":" + CHUNKLEN + "\n" + + CHUNKDATA; + } + + classifierHelper._updatesToCleanup = + classifierHelper._updatesToCleanup.filter((v) => { + return updateData.indexOf(v) == -1; + }); + + classifierHelper._update(testUpdate, resolve, reject); + }); +}; + +// This API is used to expire all add/sub chunks we have updated +// by using addUrlToDB and removeUrlFromDB. +classifierHelper.resetDB = function() { + return new Promise(function(resolve, reject) { + var testUpdate = ""; + for (var update of classifierHelper._updatesToCleanup) { + if (testUpdate.includes(update.db)) + continue; + + testUpdate += + "n:1000\n" + + "i:" + update.db + "\n" + + "ad:" + ADD_CHUNKNUM + "\n" + + "sd:" + SUB_CHUNKNUM + "\n" + } + + classifierHelper._update(testUpdate, resolve, reject); + }); +}; + +classifierHelper.reloadDatabase = function() { + return new Promise(function(resolve, reject) { + gScript.addMessageListener("reloadSuccess", function handler() { + gScript.removeMessageListener('reloadSuccess', handler); + resolve(); + }); + + gScript.sendAsyncMessage("doReload"); + }); +} + +classifierHelper._update = function(testUpdate, onsuccess, onerror) { + // Queue the task if there is still an on-going update + classifierHelper._updates.push({"data": testUpdate, + "onsuccess": onsuccess, + "onerror": onerror}); + if (classifierHelper._updates.length != 1) { + return; + } + + gScript.sendAsyncMessage("doUpdate", { testUpdate }); +}; + +classifierHelper._updateSuccess = function() { + var update = classifierHelper._updates.shift(); + update.onsuccess(); + + if (classifierHelper._updates.length) { + var testUpdate = classifierHelper._updates[0].data; + gScript.sendAsyncMessage("doUpdate", { testUpdate }); + } +}; + +classifierHelper._updateError = function(errorCode) { + var update = classifierHelper._updates.shift(); + update.onerror(errorCode); + + if (classifierHelper._updates.length) { + var testUpdate = classifierHelper._updates[0].data; + gScript.sendAsyncMessage("doUpdate", { testUpdate }); + } +}; + +classifierHelper._inited = function() { + classifierHelper._initsCB.forEach(function (cb) { + cb(); + }); + classifierHelper._initsCB = []; +}; + +classifierHelper._setup = function() { + gScript.addMessageListener("updateSuccess", classifierHelper._updateSuccess); + gScript.addMessageListener("updateError", classifierHelper._updateError); + gScript.addMessageListener("safeBrowsingInited", classifierHelper._inited); + + // cleanup will be called at end of each testcase to remove all the urls added to database. + SimpleTest.registerCleanupFunction(classifierHelper._cleanup); +}; + +classifierHelper._cleanup = function() { + // clean all the preferences may touch by helper + for (var pref in PREFS) { + SpecialPowers.clearUserPref(pref); + } + + if (!classifierHelper._updatesToCleanup) { + return Promise.resolve(); + } + + return classifierHelper.resetDB(); +}; + +classifierHelper._setup(); diff --git a/toolkit/components/url-classifier/tests/mochitest/cleanWorker.js b/toolkit/components/url-classifier/tests/mochitest/cleanWorker.js new file mode 100644 index 000000000..685648373 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/cleanWorker.js @@ -0,0 +1,10 @@ +onmessage = function() { + try { + importScripts("evilWorker.js"); + } catch(ex) { + postMessage("success"); + return; + } + + postMessage("failure"); +}; diff --git a/toolkit/components/url-classifier/tests/mochitest/dnt.html b/toolkit/components/url-classifier/tests/mochitest/dnt.html new file mode 100644 index 000000000..effc3a4f8 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/dnt.html @@ -0,0 +1,31 @@ +<html> +<head> +<title></title> + +<script type="text/javascript"> + +function makeXHR(url, callback) { + var xhr = new XMLHttpRequest(); + xhr.open('GET', url, true); + xhr.onload = function() { + callback(xhr.response); + }; + xhr.send(); +} + +function loaded(type) { + window.parent.postMessage("navigator.doNotTrack=" + navigator.doNotTrack, "*"); + + makeXHR("dnt.sjs", (res) => { + window.parent.postMessage("DNT=" + res, "*"); + window.parent.postMessage("finish", "*"); + }); +} + +</script> +</head> + +<body onload="loaded('onload')"> +</body> + +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/dnt.sjs b/toolkit/components/url-classifier/tests/mochitest/dnt.sjs new file mode 100644 index 000000000..bbb836482 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/dnt.sjs @@ -0,0 +1,9 @@ +function handleRequest(request, response) { + var dnt = "unspecified"; + if (request.hasHeader("DNT")) { + dnt = "1"; + } + + response.setHeader("Content-Type", "text/plain", false); + response.write(dnt); +} diff --git a/toolkit/components/url-classifier/tests/mochitest/evil.css b/toolkit/components/url-classifier/tests/mochitest/evil.css new file mode 100644 index 000000000..f6f08d7c5 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/evil.css @@ -0,0 +1 @@ +#styleCheck { visibility: hidden; }
\ No newline at end of file diff --git a/toolkit/components/url-classifier/tests/mochitest/evil.css^headers^ b/toolkit/components/url-classifier/tests/mochitest/evil.css^headers^ new file mode 100644 index 000000000..4030ea1d3 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/evil.css^headers^ @@ -0,0 +1 @@ +Cache-Control: no-store diff --git a/toolkit/components/url-classifier/tests/mochitest/evil.js b/toolkit/components/url-classifier/tests/mochitest/evil.js new file mode 100644 index 000000000..27f2e8c43 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/evil.js @@ -0,0 +1 @@ +scriptItem = "loaded malware javascript!"; diff --git a/toolkit/components/url-classifier/tests/mochitest/evil.js^headers^ b/toolkit/components/url-classifier/tests/mochitest/evil.js^headers^ new file mode 100644 index 000000000..3eced9614 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/evil.js^headers^ @@ -0,0 +1,2 @@ +Access-Control-Allow-Origin: * +Cache-Control: no-store diff --git a/toolkit/components/url-classifier/tests/mochitest/evilWorker.js b/toolkit/components/url-classifier/tests/mochitest/evilWorker.js new file mode 100644 index 000000000..ac34977d7 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/evilWorker.js @@ -0,0 +1,3 @@ +onmessage = function() { + postMessage("loaded bad file"); +} diff --git a/toolkit/components/url-classifier/tests/mochitest/gethash.sjs b/toolkit/components/url-classifier/tests/mochitest/gethash.sjs new file mode 100644 index 000000000..9dcc6e0d5 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/gethash.sjs @@ -0,0 +1,130 @@ +const CC = Components.Constructor; +const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1", + "nsIBinaryInputStream", + "setInputStream"); + +function handleRequest(request, response) +{ + var query = {}; + request.queryString.split('&').forEach(function (val) { + var idx = val.indexOf('='); + query[val.slice(0, idx)] = unescape(val.slice(idx + 1)); + }); + + var responseBody; + + // Store fullhash in the server side. + if ("list" in query && "fullhash" in query) { + // In the server side we will store: + // 1. All the full hashes for a given list + // 2. All the lists we have right now + // data is separate by '\n' + let list = query["list"]; + let hashes = getState(list); + + let hash = base64ToString(query["fullhash"]); + hashes += hash + "\n"; + setState(list, hashes); + + let lists = getState("lists"); + if (lists.indexOf(list) == -1) { + lists += list + "\n"; + setState("lists", lists); + } + + return; + // gethash count return how many gethash request received. + // This is used by client to know if a gethash request is triggered by gecko + } else if ("gethashcount" == request.queryString) { + var counter = getState("counter"); + responseBody = counter == "" ? "0" : counter; + } else { + var body = new BinaryInputStream(request.bodyInputStream); + var avail; + var bytes = []; + + while ((avail = body.available()) > 0) { + Array.prototype.push.apply(bytes, body.readByteArray(avail)); + } + + var counter = getState("counter"); + counter = counter == "" ? "1" : (parseInt(counter) + 1).toString(); + setState("counter", counter); + + responseBody = parseV2Request(bytes); + } + + response.setHeader("Content-Type", "text/plain", false); + response.write(responseBody); + +} + +function parseV2Request(bytes) { + var request = String.fromCharCode.apply(this, bytes); + var [HEADER, PREFIXES] = request.split("\n"); + var [PREFIXSIZE, LENGTH] = HEADER.split(":").map(val => { + return parseInt(val); + }); + + var ret = ""; + for(var start = 0; start < LENGTH; start += PREFIXSIZE) { + getState("lists").split("\n").forEach(function(list) { + var completions = getState(list).split("\n"); + + for (var completion of completions) { + if (completion.indexOf(PREFIXES.substr(start, PREFIXSIZE)) == 0) { + ret += list + ":" + "1" + ":" + "32" + "\n"; + ret += completion; + } + } + }); + } + + return ret; +} + +/* Convert Base64 data to a string */ +const toBinaryTable = [ + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63, + 52,53,54,55, 56,57,58,59, 60,61,-1,-1, -1, 0,-1,-1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, + 15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1, + -1,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, + 41,42,43,44, 45,46,47,48, 49,50,51,-1, -1,-1,-1,-1 +]; +const base64Pad = '='; + +function base64ToString(data) { + var result = ''; + var leftbits = 0; // number of bits decoded, but yet to be appended + var leftdata = 0; // bits decoded, but yet to be appended + + // Convert one by one. + for (var i = 0; i < data.length; i++) { + var c = toBinaryTable[data.charCodeAt(i) & 0x7f]; + var padding = (data[i] == base64Pad); + // Skip illegal characters and whitespace + if (c == -1) continue; + + // Collect data into leftdata, update bitcount + leftdata = (leftdata << 6) | c; + leftbits += 6; + + // If we have 8 or more bits, append 8 bits to the result + if (leftbits >= 8) { + leftbits -= 8; + // Append if not padding. + if (!padding) + result += String.fromCharCode((leftdata >> leftbits) & 0xff); + leftdata &= (1 << leftbits) - 1; + } + } + + // If there are any bits left, the base64 string was corrupted + if (leftbits) + throw Components.Exception('Corrupted base64 string'); + + return result; +} diff --git a/toolkit/components/url-classifier/tests/mochitest/gethashFrame.html b/toolkit/components/url-classifier/tests/mochitest/gethashFrame.html new file mode 100644 index 000000000..560ddcde6 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/gethashFrame.html @@ -0,0 +1,62 @@ +<html> +<head> +<title></title> + +<script type="text/javascript"> + +var scriptItem = "untouched"; + +function checkLoads() { + + var title = document.getElementById("title"); + title.innerHTML = window.parent.shouldLoad ? + "The following should be hidden:" : + "The following should not be hidden:" + + if (window.parent.shouldLoad) { + window.parent.is(scriptItem, "loaded malware javascript!", "Should load bad javascript"); + } else { + window.parent.is(scriptItem, "untouched", "Should not load bad javascript"); + } + + var elt = document.getElementById("styleImport"); + var style = document.defaultView.getComputedStyle(elt, ""); + window.parent.isnot(style.visibility, "visible", "Should load clean css"); + + // Make sure the css did not load. + elt = document.getElementById("styleCheck"); + style = document.defaultView.getComputedStyle(elt, ""); + if (window.parent.shouldLoad) { + window.parent.isnot(style.visibility, "visible", "Should load bad css"); + } else { + window.parent.isnot(style.visibility, "hidden", "Should not load bad css"); + } + + elt = document.getElementById("styleBad"); + style = document.defaultView.getComputedStyle(elt, ""); + if (window.parent.shouldLoad) { + window.parent.isnot(style.visibility, "visible", "Should import bad css"); + } else { + window.parent.isnot(style.visibility, "hidden", "Should not import bad css"); + } +} + +</script> + +<!-- Try loading from a malware javascript URI --> +<script type="text/javascript" src="http://malware.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js"></script> + +<!-- Try loading from an uwanted software css URI --> +<link rel="stylesheet" type="text/css" href="http://unwanted.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.css"></link> + +<!-- Try loading a marked-as-malware css through an @import from a clean URI --> +<link rel="stylesheet" type="text/css" href="import.css"></link> +</head> + +<body onload="checkLoads()"> +<div id="title"></div> +<div id="styleCheck">STYLE EVIL</div> +<div id="styleBad">STYLE BAD</div> +<div id="styleImport">STYLE IMPORT</div> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/good.js b/toolkit/components/url-classifier/tests/mochitest/good.js new file mode 100644 index 000000000..015b9fe52 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/good.js @@ -0,0 +1 @@ +scriptItem = "loaded whitelisted javascript!"; diff --git a/toolkit/components/url-classifier/tests/mochitest/import.css b/toolkit/components/url-classifier/tests/mochitest/import.css new file mode 100644 index 000000000..9b86c8216 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/import.css @@ -0,0 +1,3 @@ +/* malware.example.com is in the malware database. */ +@import url("http://malware.example.com/tests/toolkit/components/url-classifier/tests/mochitest/bad.css"); +#styleImport { visibility: hidden; } diff --git a/toolkit/components/url-classifier/tests/mochitest/mochitest.ini b/toolkit/components/url-classifier/tests/mochitest/mochitest.ini new file mode 100644 index 000000000..c5679e86b --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/mochitest.ini @@ -0,0 +1,39 @@ +[DEFAULT] +support-files = + classifiedAnnotatedPBFrame.html + classifierCommon.js + classifierFrame.html + classifierHelper.js + cleanWorker.js + good.js + evil.css + evil.css^headers^ + evil.js + evil.js^headers^ + evilWorker.js + import.css + raptor.jpg + track.html + unwantedWorker.js + vp9.webm + whitelistFrame.html + workerFrame.html + ping.sjs + basic.vtt + basic.vtt^headers^ + dnt.html + dnt.sjs + update.sjs + bad.css + bad.css^headers^ + gethash.sjs + gethashFrame.html + seek.webm + +[test_classifier.html] +skip-if = (os == 'linux' && debug) #Bug 1199778 +[test_classifier_worker.html] +[test_classify_ping.html] +[test_classify_track.html] +[test_gethash.html] +[test_bug1254766.html] diff --git a/toolkit/components/url-classifier/tests/mochitest/ping.sjs b/toolkit/components/url-classifier/tests/mochitest/ping.sjs new file mode 100644 index 000000000..37a78956e --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/ping.sjs @@ -0,0 +1,16 @@ +function handleRequest(request, response) +{ + var query = {}; + request.queryString.split('&').forEach(function (val) { + var [name, value] = val.split('='); + query[name] = unescape(value); + }); + + if (request.method == "POST") { + setState(query["id"], "ping"); + } else { + var value = getState(query["id"]); + response.setHeader("Content-Type", "text/plain", false); + response.write(value); + } +} diff --git a/toolkit/components/url-classifier/tests/mochitest/raptor.jpg b/toolkit/components/url-classifier/tests/mochitest/raptor.jpg Binary files differnew file mode 100644 index 000000000..243ba9e2d --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/raptor.jpg diff --git a/toolkit/components/url-classifier/tests/mochitest/seek.webm b/toolkit/components/url-classifier/tests/mochitest/seek.webm Binary files differnew file mode 100644 index 000000000..72b029723 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/seek.webm diff --git a/toolkit/components/url-classifier/tests/mochitest/test_allowlisted_annotations.html b/toolkit/components/url-classifier/tests/mochitest/test_allowlisted_annotations.html new file mode 100644 index 000000000..ba9c86f95 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_allowlisted_annotations.html @@ -0,0 +1,56 @@ +<!DOCTYPE HTML> +<html> +<head> + <title>Test the URI Classifier</title> + <script type="text/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"> +</head> + +<body> +<p id="display"></p> +<div id="content" style="display: none"> +</div> +<pre id="test"> + +<script class="testbody" type="text/javascript"> + +var Cc = SpecialPowers.Cc; +var Ci = SpecialPowers.Ci; + +Components.utils.import("resource://testing-common/UrlClassifierTestUtils.jsm"); + +// Add https://allowlisted.example.com to the permissions manager +SpecialPowers.addPermission("trackingprotection", + Ci.nsIPermissionManager.ALLOW_ACTION, + { url: "https://allowlisted.example.com" }); + +function clearPermissions() { + SpecialPowers.removePermission("trackingprotection", + { url: "https://allowlisted.example.com" }); + ok(!SpecialPowers.testPermission("trackingprotection", + Ci.nsIPermissionManager.ALLOW_ACTION, + { url: "https://allowlisted.example.com" })); +} + +SpecialPowers.pushPrefEnv( + {"set" : [["urlclassifier.trackingTable", "test-track-simple"], + ["privacy.trackingprotection.enabled", true], + ["channelclassifier.allowlist_example", true]]}, + test); + +function test() { + SimpleTest.registerCleanupFunction(UrlClassifierTestUtils.cleanupTestTrackers); + UrlClassifierTestUtils.addTestTrackers().then(() => { + document.getElementById("testFrame").src = "allowlistAnnotatedFrame.html"; + }); +} + +// Expected finish() call is in "allowlistedAnnotatedFrame.html". +SimpleTest.waitForExplicitFinish(); + +</script> + +</pre> +<iframe id="testFrame" width="100%" height="100%" onload=""></iframe> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/test_bug1254766.html b/toolkit/components/url-classifier/tests/mochitest/test_bug1254766.html new file mode 100644 index 000000000..1c149406a --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_bug1254766.html @@ -0,0 +1,305 @@ +<!DOCTYPE HTML> +<html> +<head> + <title>Bug 1272239 - Test gethash.</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="classifierHelper.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> + +<body> +<p id="display"></p> +<div id="content" style="display: none"> +</div> +<pre id="test"> + +<script class="testbody" type="text/javascript"> + +const MALWARE_LIST = "test-malware-simple"; +const MALWARE_HOST1 = "malware.example.com/"; +const MALWARE_HOST2 = "test1.example.com/"; + +const UNWANTED_LIST = "test-unwanted-simple"; +const UNWANTED_HOST1 = "unwanted.example.com/"; +const UNWANTED_HOST2 = "test2.example.com/"; + + +const UNUSED_MALWARE_HOST = "unused.malware.com/"; +const UNUSED_UNWANTED_HOST = "unused.unwanted.com/"; + +const GETHASH_URL = + "http://mochi.test:8888/tests/toolkit/components/url-classifier/tests/mochitest/gethash.sjs"; + +var gPreGethashCounter = 0; +var gCurGethashCounter = 0; + +var expectLoad = false; + +function loadTestFrame() { + return new Promise(function(resolve, reject) { + var iframe = document.createElement("iframe"); + iframe.setAttribute("src", "gethashFrame.html"); + document.body.appendChild(iframe); + + iframe.onload = function() { + document.body.removeChild(iframe); + resolve(); + }; + }).then(getGethashCounter); +} + +function getGethashCounter() { + return new Promise(function(resolve, reject) { + var xhr = new XMLHttpRequest; + xhr.open("PUT", GETHASH_URL + "?gethashcount"); + xhr.setRequestHeader("Content-Type", "text/plain"); + xhr.onreadystatechange = function() { + if (this.readyState == this.DONE) { + gPreGethashCounter = gCurGethashCounter; + gCurGethashCounter = parseInt(xhr.response); + resolve(); + } + }; + xhr.send(); + }); +} + +// calculate the fullhash and send it to gethash server +function addCompletionToServer(list, url) { + return new Promise(function(resolve, reject) { + var listParam = "list=" + list; + var fullhashParam = "fullhash=" + hash(url); + + var xhr = new XMLHttpRequest; + xhr.open("PUT", GETHASH_URL + "?" + listParam + "&" + fullhashParam, true); + xhr.setRequestHeader("Content-Type", "text/plain"); + xhr.onreadystatechange = function() { + if (this.readyState == this.DONE) { + resolve(); + } + }; + xhr.send(); + }); +} + +function hash(str) { + function bytesFromString(str) { + var converter = + SpecialPowers.Cc["@mozilla.org/intl/scriptableunicodeconverter"] + .createInstance(SpecialPowers.Ci.nsIScriptableUnicodeConverter); + converter.charset = "UTF-8"; + return converter.convertToByteArray(str); + } + + var hasher = SpecialPowers.Cc["@mozilla.org/security/hash;1"] + .createInstance(SpecialPowers.Ci.nsICryptoHash); + + var data = bytesFromString(str); + hasher.init(hasher.SHA256); + hasher.update(data, data.length); + + return hasher.finish(true); +} + +// setup function allows classifier send gethash request for test database +// also it calculate to fullhash for url and store those hashes in gethash sjs. +function setup() { + classifierHelper.allowCompletion([MALWARE_LIST, UNWANTED_LIST], GETHASH_URL); + + return Promise.all([ + addCompletionToServer(MALWARE_LIST, MALWARE_HOST1), + addCompletionToServer(MALWARE_LIST, MALWARE_HOST2), + addCompletionToServer(UNWANTED_LIST, UNWANTED_HOST1), + addCompletionToServer(UNWANTED_LIST, UNWANTED_HOST2), + ]); +} + +// Reset function in helper try to simulate the behavior we restart firefox +function reset() { + return classifierHelper.resetDB() + .catch(err => { + ok(false, "Couldn't update classifier. Error code: " + errorCode); + // Abort test. + SimpleTest.finish(); + }); +} + +function updateUnusedUrl() { + var testData = [ + { url: UNUSED_MALWARE_HOST, db: MALWARE_LIST }, + { url: UNUSED_UNWANTED_HOST, db: UNWANTED_LIST } + ]; + + return classifierHelper.addUrlToDB(testData) + .catch(err => { + ok(false, "Couldn't update classifier. Error code: " + err); + // Abort test. + SimpleTest.finish(); + }); +} + +function addPrefixToDB() { + return update(true); +} + +function addCompletionToDB() { + return update(false); +} + +function update(prefix = false) { + var length = prefix ? 4 : 32; + var testData = [ + { url: MALWARE_HOST1, db: MALWARE_LIST, len: length }, + { url: MALWARE_HOST2, db: MALWARE_LIST, len: length }, + { url: UNWANTED_HOST1, db: UNWANTED_LIST, len: length }, + { url: UNWANTED_HOST2, db: UNWANTED_LIST, len: length } + ]; + + return classifierHelper.addUrlToDB(testData) + .catch(err => { + ok(false, "Couldn't update classifier. Error code: " + errorCode); + // Abort test. + SimpleTest.finish(); + }); +} + +// This testcase is to make sure gethash works: +// 1. Add prefixes to DB. +// 2. Load test frame contains malware & unwanted url, those urls should be blocked. +// 3. The second step should also trigger a gethash request since completions is not in +// either cache or DB. +// 4. Load test frame again, since completions is stored in cache now, no gethash +// request should be triggered. +function testGethash() { + return Promise.resolve() + .then(addPrefixToDB) + .then(loadTestFrame) + .then(() => { + ok(gCurGethashCounter > gPreGethashCounter, "Gethash request is triggered."); }) + .then(loadTestFrame) + .then(() => { + ok(gCurGethashCounter == gPreGethashCounter, "Gethash request is not triggered."); }) + .then(reset); +} + +// This testcase is to make sure an update request will clear completion cache: +// 1. Add prefixes to DB. +// 2. Load test frame, this should trigger a gethash request +// 3. Trigger an update, completion cache should be cleared now. +// 4. Load test frame again, since cache is cleared now, gethash request should be triggered. +function testUpdateClearCache() { + return Promise.resolve() + .then(addPrefixToDB) + .then(loadTestFrame) + .then(() => { + ok(gCurGethashCounter > gPreGethashCounter, "Gethash request is triggered."); }) + .then(updateUnusedUrl) + .then(loadTestFrame) + .then(() => { + ok(gCurGethashCounter > gPreGethashCounter, "Gethash request is triggered."); }) + .then(reset); +} + +// This testcae is to make sure completions in update works: +// 1. Add completions to DB. +// 2. Load test frame, since completions is stored in DB, gethash request should +// not be triggered. +function testUpdate() { + return Promise.resolve() + .then(addCompletionToDB) + .then(loadTestFrame) + .then(() => { + ok(gCurGethashCounter == gPreGethashCounter, "Gethash request is not triggered."); }) + .then(reset); +} + +// This testcase is to make sure an update request will not clear completions in DB: +// 1. Add completions to DB. +// 2. Load test frame to make sure completions is stored in database, in this case, gethash +// should not be triggered. +// 3. Trigger an update, cache is cleared, but completions in DB should still remain. +// 4. Load test frame again, since completions is in DB, gethash request should not be triggered. +function testUpdateNotClearCompletions() { + return Promise.resolve() + .then(addCompletionToDB) + .then(loadTestFrame) + .then(() => { + ok(gCurGethashCounter == gPreGethashCounter, "Gethash request is not triggered."); }) + .then(updateUnusedUrl) + .then(loadTestFrame) + .then(() => { + ok(gCurGethashCounter == gPreGethashCounter, "Gethash request is not triggered."); }) + .then(reset); +} + +// This testcase is to make sure completion store in DB will properly load after restarting. +// 1. Add completions to DB. +// 2. Simulate firefox restart by calling reloadDatabase. +// 3. Load test frame, since completions should be loaded from DB, no gethash request should +// be triggered. +function testUpdateCompletionsAfterReload() { + return Promise.resolve() + .then(addCompletionToDB) + .then(classifierHelper.reloadDatabase) + .then(loadTestFrame) + .then(() => { + ok(gCurGethashCounter == gPreGethashCounter, "Gethash request is not triggered."); }) + .then(reset); +} + +// This testcase is to make sure cache will be cleared after restarting +// 1. Add prefixes to DB. +// 2. Load test frame, this should trigger a gethash request and completions will be stored in +// cache. +// 3. Load test frame again, no gethash should be triggered because of cache. +// 4. Simulate firefox restart by calling reloadDatabase. +// 5. Load test frame again, since cache is cleared, gethash request should be triggered. +function testGethashCompletionsAfterReload() { + return Promise.resolve() + .then(addPrefixToDB) + .then(loadTestFrame) + .then(() => { + ok(gCurGethashCounter > gPreGethashCounter, "Gethash request is triggered."); }) + .then(loadTestFrame) + .then(() => { + ok(gCurGethashCounter == gPreGethashCounter, "Gethash request is not triggered."); }) + .then(classifierHelper.reloadDatabase) + .then(loadTestFrame) + .then(() => { + ok(gCurGethashCounter > gPreGethashCounter, "Gethash request is triggered."); }) + .then(reset); +} + +function runTest() { + Promise.resolve() + .then(classifierHelper.waitForInit) + .then(setup) + .then(testGethash) + .then(testUpdateClearCache) + .then(testUpdate) + .then(testUpdateNotClearCompletions) + .then(testUpdateCompletionsAfterReload) + .then(testGethashCompletionsAfterReload) + .then(function() { + SimpleTest.finish(); + }).catch(function(e) { + ok(false, "Some test failed with error " + e); + SimpleTest.finish(); + }); +} + +SimpleTest.waitForExplicitFinish(); + +// 'network.predictor.enabled' is disabled because if other testcase load +// evil.js, evil.css ...etc resources, it may cause we load them from cache +// directly and bypass classifier check +SpecialPowers.pushPrefEnv({"set": [ + ["browser.safebrowsing.malware.enabled", true], + ["network.predictor.enabled", false], + ["urlclassifier.gethash.timeout_ms", 30000], +]}, runTest); + +</script> +</pre> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/test_classified_annotations.html b/toolkit/components/url-classifier/tests/mochitest/test_classified_annotations.html new file mode 100644 index 000000000..5814fff00 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_classified_annotations.html @@ -0,0 +1,50 @@ +<!DOCTYPE HTML> +<html> +<head> + <title>Test the URI Classifier</title> + <script type="text/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"> +</head> + +<body> +<p id="display"></p> +<div id="content" style="display: none"> +</div> +<pre id="test"> + +<script class="testbody" type="text/javascript"> + +var Cc = SpecialPowers.Cc; +var Ci = SpecialPowers.Ci; + +Components.utils.import("resource://testing-common/UrlClassifierTestUtils.jsm"); + +function cleanup() { + SpecialPowers.clearUserPref("privacy.trackingprotection.enabled"); + SpecialPowers.clearUserPref("channelclassifier.allowlist_example"); +} + +SpecialPowers.pushPrefEnv( + {"set" : [["urlclassifier.trackingTable", "test-track-simple"]]}, + test); + +function test() { + UrlClassifierTestUtils.addTestTrackers().then(() => { + SpecialPowers.setBoolPref("privacy.trackingprotection.enabled", true); + // Make sure chrome:// URIs are processed. This does not white-list + // any URIs unless 'https://allowlisted.example.com' is added in the + // permission manager (see test_allowlisted_annotations.html) + SpecialPowers.setBoolPref("channelclassifier.allowlist_example", true); + document.getElementById("testFrame").src = "classifiedAnnotatedFrame.html"; + }); +} + +// Expected finish() call is in "classifiedAnnotatedFrame.html". +SimpleTest.waitForExplicitFinish(); + +</script> + +</pre> +<iframe id="testFrame" width="100%" height="100%" onload=""></iframe> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/test_classifier.html b/toolkit/components/url-classifier/tests/mochitest/test_classifier.html new file mode 100644 index 000000000..9533db426 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_classifier.html @@ -0,0 +1,65 @@ +<!DOCTYPE HTML> +<html> +<head> + <title>Test the URI Classifier</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="classifierHelper.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> + +<p id="display"></p> +<div id="content" style="display: none"> +</div> +<pre id="test"> + +<script class="testbody" type="text/javascript"> + +var firstLoad = true; + +// Add some URLs to the malware database. +var testData = [ + { url: "malware.example.com/", + db: "test-malware-simple" + }, + { url: "unwanted.example.com/", + db: "test-unwanted-simple" + } +]; + +function loadTestFrame() { + document.getElementById("testFrame").src = "classifierFrame.html"; +} + +// Expected finish() call is in "classifierFrame.html". +SimpleTest.waitForExplicitFinish(); + +function updateSuccess() { + SpecialPowers.pushPrefEnv( + {"set" : [["browser.safebrowsing.malware.enabled", true]]}, + loadTestFrame); +} + +function updateError(errorCode) { + ok(false, "Couldn't update classifier. Error code: " + errorCode); + // Abort test. + SimpleTest.finish(); +} + +SpecialPowers.pushPrefEnv( + {"set" : [["urlclassifier.malwareTable", "test-malware-simple,test-unwanted-simple"], + ["urlclassifier.phishTable", "test-phish-simple"]]}, + function() { + classifierHelper.waitForInit() + .then(() => classifierHelper.addUrlToDB(testData)) + .then(updateSuccess) + .catch(err => { + updateError(err); + }); + }); + +</script> + +</pre> +<iframe id="testFrame" onload=""></iframe> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/test_classifier_changetablepref.html b/toolkit/components/url-classifier/tests/mochitest/test_classifier_changetablepref.html new file mode 100644 index 000000000..7423d3e8e --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_classifier_changetablepref.html @@ -0,0 +1,149 @@ +<!DOCTYPE HTML> +<html> +<head> + <title>Bug 1281083 - Changing the urlclassifier.*Table prefs doesn't take effect before the next browser restart.</title> + <script type="text/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="classifierHelper.js"></script> + <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"> +</head> + +<body> +<p id="display"></p> +<div id="content" style="display: none"> +</div> +<pre id="test"> + +<script class="testbody" type="text/javascript"> + +const testTable = "moz-track-digest256"; +const UPDATE_URL = "http://mochi.test:8888/tests/toolkit/components/url-classifier/tests/mochitest/update.sjs"; + +var Cc = SpecialPowers.Cc; +var Ci = SpecialPowers.Ci; + +var prefService = Cc["@mozilla.org/preferences-service;1"] + .getService(Ci.nsIPrefService); + +var timer = Cc["@mozilla.org/timer;1"] + .createInstance(Ci.nsITimer); + +// If default preference contain the table we want to test, +// We should change test table to a different one. +var trackingTables = SpecialPowers.getCharPref("urlclassifier.trackingTable").split(","); +ok(!trackingTables.includes(testTable), "test table should not be in the preference"); + +var listmanager = Cc["@mozilla.org/url-classifier/listmanager;1"]. + getService(Ci.nsIUrlListManager); + +is(listmanager.getGethashUrl(testTable), "", + "gethash url for test table should be empty before setting to preference"); + +function loadTestFrame() { + // gethash url of test table "moz-track-digest256" should be updated + // after setting preference. + var url = listmanager.getGethashUrl(testTable); + var expected = SpecialPowers.getCharPref("browser.safebrowsing.provider.mozilla.gethashURL"); + + is(url, expected, testTable + " matches its gethash url"); + + // Trigger update + listmanager.disableUpdate(testTable); + listmanager.enableUpdate(testTable); + listmanager.maybeToggleUpdateChecking(); + + // We wait until "nextupdattime" was set as a signal that update is complete. + waitForUpdateSuccess(function() { + document.getElementById("testFrame").src = "bug_1281083.html"; + }); +} + +function waitForUpdateSuccess(callback) { + let nextupdatetime = + SpecialPowers.getCharPref("browser.safebrowsing.provider.mozilla.nextupdatetime"); + + if (nextupdatetime !== "1") { + callback(); + return; + } + + timer.initWithCallback(function() { + waitForUpdateSuccess(callback); + }, 10, Components.interfaces.nsITimer.TYPE_ONE_SHOT); +} + +function addCompletionToServer(list, url) { + return new Promise(function(resolve, reject) { + var listParam = "list=" + list; + var fullhashParam = "fullhash=" + hash(url); + + var xhr = new XMLHttpRequest; + xhr.open("PUT", UPDATE_URL + "?" + + listParam + "&" + + fullhashParam, true); + xhr.setRequestHeader("Content-Type", "text/plain"); + xhr.onreadystatechange = function() { + if (this.readyState == this.DONE) { + resolve(); + } + }; + xhr.send(); + }); +} + +function hash(str) { + function bytesFromString(str) { + var converter = + SpecialPowers.Cc["@mozilla.org/intl/scriptableunicodeconverter"] + .createInstance(SpecialPowers.Ci.nsIScriptableUnicodeConverter); + converter.charset = "UTF-8"; + return converter.convertToByteArray(str); + } + + var hasher = SpecialPowers.Cc["@mozilla.org/security/hash;1"] + .createInstance(SpecialPowers.Ci.nsICryptoHash); + + var data = bytesFromString(str); + hasher.init(hasher.SHA256); + hasher.update(data, data.length); + + return hasher.finish(true); +} + +function runTest() { + /** + * In this test we try to modify only urlclassifier.*Table preference to see if + * url specified in the table will be blocked after update. + */ + var pushPrefPromise = SpecialPowers.pushPrefEnv( + {"set" : [["urlclassifier.trackingTable", testTable]]}); + + // To make sure url is not blocked by an already blocked url. + // Here we use non-tracking.example.com as a tracked url. + // Since this table is only used in this bug, so it won't affect other testcases. + var addCompletePromise = + addCompletionToServer(testTable, "bug1281083.example.com/"); + + Promise.all([pushPrefPromise, addCompletePromise]) + .then(() => { + loadTestFrame(); + }); +} + +// Set nextupdatetime to 1 to trigger an update +SpecialPowers.pushPrefEnv( + {"set" : [["privacy.trackingprotection.enabled", true], + ["channelclassifier.allowlist_example", true], + ["browser.safebrowsing.provider.mozilla.nextupdatetime", "1"], + ["browser.safebrowsing.provider.mozilla.lists", testTable], + ["browser.safebrowsing.provider.mozilla.updateURL", UPDATE_URL]]}, + runTest); + +// Expected finish() call is in "bug_1281083.html". +SimpleTest.waitForExplicitFinish(); + +</script> +</pre> +<iframe id="testFrame" width="100%" height="100%" onload=""></iframe> +</body> +</html> + diff --git a/toolkit/components/url-classifier/tests/mochitest/test_classifier_worker.html b/toolkit/components/url-classifier/tests/mochitest/test_classifier_worker.html new file mode 100644 index 000000000..1f54d45b0 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_classifier_worker.html @@ -0,0 +1,76 @@ +<!DOCTYPE HTML> +<html> +<head> + <title>Test the URI Classifier</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="classifierHelper.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> + +<p id="display"></p> +<div id="content" style="display: none"> +</div> +<pre id="test"> + +<script class="testbody" type="text/javascript"> + +// Add some URLs to the malware database. +var testData = [ + { url: "example.com/tests/toolkit/components/url-classifier/tests/mochitest/evilWorker.js", + db: "test-malware-simple" + }, + { url: "example.com/tests/toolkit/components/url-classifier/tests/mochitest/unwantedWorker.js", + db: "test-unwanted-simple" + } +]; + +function loadTestFrame() { + document.getElementById("testFrame").src = + "http://example.com/tests/toolkit/components/url-classifier/tests/mochitest/workerFrame.html"; +} + +function onmessage(event) +{ + var pieces = event.data.split(':'); + if (pieces[0] == "finish") { + SimpleTest.finish(); + return; + } + + is(pieces[0], "success", pieces[1]); +} + +function updateSuccess() { + SpecialPowers.pushPrefEnv( + {"set" : [["browser.safebrowsing.malware.enabled", true]]}, + loadTestFrame); +} + +function updateError(errorCode) { + ok(false, "Couldn't update classifier. Error code: " + errorCode); + // Abort test. + SimpleTest.finish(); +}; + +SpecialPowers.pushPrefEnv( + {"set" : [["urlclassifier.malwareTable", "test-malware-simple,test-unwanted-simple"], + ["urlclassifier.phishTable", "test-phish-simple"]]}, + function() { + classifierHelper.waitForInit() + .then(() => classifierHelper.addUrlToDB(testData)) + .then(updateSuccess) + .catch(err => { + updateError(err); + }); + }); + +window.addEventListener("message", onmessage, false); + +SimpleTest.waitForExplicitFinish(); + +</script> + +</pre> +<iframe id="testFrame" onload=""></iframe> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/test_classify_ping.html b/toolkit/components/url-classifier/tests/mochitest/test_classify_ping.html new file mode 100644 index 000000000..96fa2891a --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_classify_ping.html @@ -0,0 +1,121 @@ +<!DOCTYPE HTML> +<html> +<head> + <title>Bug 1233914 - ping doesn't honor the TP list here.</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> + +<body> +<p id="display"></p> +<div id="content" style="display: none"> +</div> +<pre id="test"> + +<script class="testbody" type="text/javascript"> + SimpleTest.requestFlakyTimeout("Delay to make sure ping is made prior than XHR"); + + const timeout = 200; + const host_nottrack = "http://not-tracking.example.com/"; + const host_track = "http://trackertest.org/"; + const path_ping = "tests/toolkit/components/url-classifier/tests/mochitest/ping.sjs"; + const TP_ENABLE_PREF = "privacy.trackingprotection.enabled"; + + function testPingNonBlacklist() { + SpecialPowers.setBoolPref(TP_ENABLE_PREF, true); + + var msg = "ping should reach page not in blacklist"; + var expectPing = true; + var id = "1111"; + ping(id, host_nottrack); + + return new Promise(function(resolve, reject) { + setTimeout(function() { + isPinged(id, expectPing, msg, resolve); + }, timeout); + }); + } + + function testPingBlacklistSafebrowsingOff() { + SpecialPowers.setBoolPref(TP_ENABLE_PREF, false); + + var msg = "ping should reach page in blacklist when tracking protection is off"; + var expectPing = true; + var id = "2222"; + ping(id, host_track); + + return new Promise(function(resolve, reject) { + setTimeout(function() { + isPinged(id, expectPing, msg, resolve); + }, timeout); + }); + } + + function testPingBlacklistSafebrowsingOn() { + SpecialPowers.setBoolPref(TP_ENABLE_PREF, true); + + var msg = "ping should not reach page in blacklist when tracking protection is on"; + var expectPing = false; + var id = "3333"; + ping(id, host_track); + + return new Promise(function(resolve, reject) { + setTimeout(function() { + isPinged(id, expectPing, msg, resolve); + }, timeout); + }); + } + + function ping(id, host) { + var elm = document.createElement("a"); + elm.setAttribute('ping', host + path_ping + "?id=" + id); + elm.setAttribute('href', "#"); + document.body.appendChild(elm); + + // Trigger ping. + elm.click(); + + document.body.removeChild(elm); + } + + function isPinged(id, expected, msg, callback) { + var url = "http://mochi.test:8888/" + path_ping; + var xhr = new XMLHttpRequest(); + xhr.open('GET', url + "?id=" + id); + xhr.onload = function() { + var isPinged = xhr.response === "ping"; + is(expected, isPinged, msg); + + callback(); + }; + xhr.send(); + } + + function cleanup() { + SpecialPowers.clearUserPref(TP_ENABLE_PREF); + } + + function runTest() { + Promise.resolve() + .then(testPingNonBlacklist) + .then(testPingBlacklistSafebrowsingOff) + .then(testPingBlacklistSafebrowsingOn) + .then(function() { + SimpleTest.finish(); + }).catch(function(e) { + ok(false, "Some test failed with error " + e); + SimpleTest.finish(); + }); + } + + SimpleTest.waitForExplicitFinish(); + SimpleTest.registerCleanupFunction(cleanup); + SpecialPowers.pushPrefEnv({"set": [ + ["browser.send_pings", true], + ["urlclassifier.trackingTable", "test-track-simple"], + ]}, runTest); + +</script> +</pre> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/test_classify_track.html b/toolkit/components/url-classifier/tests/mochitest/test_classify_track.html new file mode 100644 index 000000000..a868d7960 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_classify_track.html @@ -0,0 +1,162 @@ +<!DOCTYPE HTML> +<html> +<head> + <title>Bug 1262406 - Track element doesn't use the URL classifier.</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="classifierHelper.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> + +<body> +<p id="display"></p> +<div id="content" style="display: none"> +</div> +<pre id="test"> + +<script class="testbody" type="text/javascript"> + const PREF = "browser.safebrowsing.malware.enabled"; + const track_path = "tests/toolkit/components/url-classifier/tests/mochitest/basic.vtt"; + const malware_url = "http://malware.example.com/" + track_path; + const validtrack_url = "http://mochi.test:8888/" + track_path; + + var video = document.createElement("video"); + video.src = "seek.webm"; + video.crossOrigin = "anonymous"; + + document.body.appendChild(video); + + function testValidTrack() { + SpecialPowers.setBoolPref(PREF, true); + + return new Promise(function(resolve, reject) { + var track = document.createElement("track"); + track.src = validtrack_url; + video.appendChild(track); + + function onload() { + ok(true, "Track should be loaded when url is not in blacklist"); + finish(); + } + + function onerror() { + ok(false, "Error while loading track"); + finish(); + } + + function finish() { + track.removeEventListener("load", onload); + track.removeEventListener("error", onerror) + resolve(); + } + + track.addEventListener("load", onload); + track.addEventListener("error", onerror) + }); + } + + function testBlacklistTrackSafebrowsingOff() { + SpecialPowers.setBoolPref(PREF, false); + + return new Promise(function(resolve, reject) { + var track = document.createElement("track"); + track.src = malware_url; + video.appendChild(track); + + function onload() { + ok(true, "Track should be loaded when url is in blacklist and safebrowsing is off"); + finish(); + } + + function onerror() { + ok(false, "Error while loading track"); + finish(); + } + + function finish() { + track.removeEventListener("load", onload); + track.removeEventListener("error", onerror) + resolve(); + } + + track.addEventListener("load", onload); + track.addEventListener("error", onerror) + }); + } + + function testBlacklistTrackSafebrowsingOn() { + SpecialPowers.setBoolPref(PREF, true); + + return new Promise(function(resolve, reject) { + var track = document.createElement("track"); + + // Add a query string parameter here to avoid url classifier bypass classify + // because of cache. + track.src = malware_url + "?testsbon"; + video.appendChild(track); + + function onload() { + ok(false, "Unexpected result while loading track in blacklist"); + finish(); + } + + function onerror() { + ok(true, "Track should not be loaded when url is in blacklist and safebrowsing is on"); + finish(); + } + + function finish() { + track.removeEventListener("load", onload); + track.removeEventListener("error", onerror) + resolve(); + } + + track.addEventListener("load", onload); + track.addEventListener("error", onerror) + }); + } + + function cleanup() { + SpecialPowers.clearUserPref(PREF); + } + + function setup() { + var testData = [ + { url: "malware.example.com/", + db: "test-malware-simple" + } + ]; + + return classifierHelper.addUrlToDB(testData) + .catch(function(err) { + ok(false, "Couldn't update classifier. Error code: " + err); + // Abort test. + SimpleTest.finish(); + }); + } + + function runTest() { + Promise.resolve() + .then(classifierHelper.waitForInit) + .then(setup) + .then(testValidTrack) + .then(testBlacklistTrackSafebrowsingOff) + .then(testBlacklistTrackSafebrowsingOn) + .then(function() { + SimpleTest.finish(); + }).catch(function(e) { + ok(false, "Some test failed with error " + e); + SimpleTest.finish(); + }); + } + + SimpleTest.waitForExplicitFinish(); + SimpleTest.registerCleanupFunction(cleanup); + SpecialPowers.pushPrefEnv({"set": [ + ["media.webvtt.regions.enabled", true], + ["urlclassifier.malwareTable", "test-malware-simple"], + ]}, runTest); + +</script> +</pre> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/test_donottrack.html b/toolkit/components/url-classifier/tests/mochitest/test_donottrack.html new file mode 100644 index 000000000..56003e7eb --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_donottrack.html @@ -0,0 +1,150 @@ +<!DOCTYPE HTML> +<html> +<head> + <title>Bug 1258033 - Fix the DNT loophole for tracking protection</title> + <script type="text/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"> +</head> + +<body> +<p id="display"></p> +<div id="content" style="display: none"> +</div> +<pre id="test"> + +<script class="testbody" type="text/javascript"> + +var Cc = SpecialPowers.Cc; +var Ci = SpecialPowers.Ci; + +var mainWindow = window.QueryInterface(Ci.nsIInterfaceRequestor) + .getInterface(Ci.nsIWebNavigation) + .QueryInterface(Ci.nsIDocShellTreeItem) + .rootTreeItem + .QueryInterface(Ci.nsIInterfaceRequestor) + .getInterface(Ci.nsIDOMWindow); + +const tests = [ + // DNT turned on and TP turned off, DNT signal sent in both private browsing + // and normal mode. + { + setting: {dntPref:true, tpPref:false, tppbPref:false, pbMode:true}, + expected: {dnt: "1"}, + }, + { + setting: {dntPref:true, tpPref:false, tppbPref:false, pbMode:false}, + expected: {dnt: "1"} + }, + // DNT turned off and TP turned on globally, DNT signal sent in both private + // browsing and normal mode. + { + setting: {dntPref:false, tpPref:true, tppbPref:false, pbMode:true}, + expected: {dnt: "1"} + }, + { + setting: {dntPref:false, tpPref:true, tppbPref:false, pbMode:false}, + expected: {dnt: "1"} + }, + // DNT turned off and TP in Private Browsing only, DNT signal sent in private + // browsing mode only. + { + setting: {dntPref:false, tpPref:false, tppbPref:true, pbMode:true}, + expected: {dnt: "1"} + }, + { + setting: {dntPref:false, tpPref:false, tppbPref:true, pbMode:false}, + expected: {dnt: "unspecified"} + }, + // DNT turned off and TP turned off, DNT signal is never sent. + { + setting: {dntPref:false, tpPref:false, tppbPref:false, pbMode:true}, + expected: {dnt: "unspecified"} + }, + { + setting: {dntPref:false, tpPref:false, tppbPref:false, pbMode:false}, + expected: {dnt: "unspecified"} + }, +] + +const DNT_PREF = 'privacy.donottrackheader.enabled'; +const TP_PREF = 'privacy.trackingprotection.enabled'; +const TP_PB_PREF = 'privacy.trackingprotection.pbmode.enabled'; + +const contentPage = + "http://mochi.test:8888/tests/toolkit/components/url-classifier/tests/mochitest/dnt.html"; + +Components.utils.import("resource://gre/modules/Services.jsm"); + +function whenDelayedStartupFinished(aWindow, aCallback) { + Services.obs.addObserver(function observer(aSubject, aTopic) { + if (aWindow == aSubject) { + Services.obs.removeObserver(observer, aTopic); + setTimeout(aCallback, 0); + } + }, "browser-delayed-startup-finished", false); +} + +function executeTest(test) { + SpecialPowers.pushPrefEnv({"set" : [ + [DNT_PREF, test.setting.dntPref], + [TP_PREF, test.setting.tpPref], + [TP_PB_PREF, test.setting.tppbPref] + ]}); + + var win = mainWindow.OpenBrowserWindow({private: test.setting.pbMode}); + + return new Promise(function(resolve, reject) { + win.addEventListener("load", function onLoad() { + win.removeEventListener("load", onLoad, false); + whenDelayedStartupFinished(win, function() { + win.addEventListener("DOMContentLoaded", function onInnerLoad() { + if (win.content.location.href != contentPage) { + win.gBrowser.loadURI(contentPage); + return; + } + + win.removeEventListener("DOMContentLoaded", onInnerLoad, true); + + win.content.addEventListener('message', function (event) { + let [key, value] = event.data.split("="); + if (key == "finish") { + win.close(); + resolve(); + } else if (key == "navigator.doNotTrack") { + is(value, test.expected.dnt, "navigator.doNotTrack should be " + test.expected.dnt); + } else if (key == "DNT") { + let msg = test.expected.dnt == "1" ? "" : "not "; + is(value, test.expected.dnt, "DNT header should " + msg + "be sent"); + } else { + ok(false, "unexpected message"); + } + }); + }, true); + SimpleTest.executeSoon(function() { win.gBrowser.loadURI(contentPage); }); + }); + }, true); + }); +} + +let loop = function loop(index) { + if (index >= tests.length) { + SimpleTest.finish(); + return; + } + + let test = tests[index]; + let next = function next() { + loop(index + 1); + }; + let result = executeTest(test); + result.then(next, next); +}; + +SimpleTest.waitForExplicitFinish(); +loop(0); + +</script> + +</pre> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/test_gethash.html b/toolkit/components/url-classifier/tests/mochitest/test_gethash.html new file mode 100644 index 000000000..af995e2a5 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_gethash.html @@ -0,0 +1,157 @@ +<!DOCTYPE HTML> +<html> +<head> + <title>Bug 1272239 - Test gethash.</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="classifierHelper.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> + +<body> +<p id="display"></p> +<div id="content" style="display: none"> +</div> +<pre id="test"> +<iframe id="testFrame1" onload=""></iframe> +<iframe id="testFrame2" onload=""></iframe> + +<script class="testbody" type="text/javascript"> + +const MALWARE_LIST = "test-malware-simple"; +const MALWARE_HOST = "malware.example.com/"; + +const UNWANTED_LIST = "test-unwanted-simple"; +const UNWANTED_HOST = "unwanted.example.com/"; + +const GETHASH_URL = "http://mochi.test:8888/tests/toolkit/components/url-classifier/tests/mochitest/gethash.sjs"; +const NOTEXIST_URL = "http://mochi.test:8888/tests/toolkit/components/url-classifier/tests/mochitest/nonexistserver.sjs"; + +var shouldLoad = false; + +// In this testcase we store prefixes to localdb and send the fullhash to gethash server. +// When access the test page gecko should trigger gethash request to server and +// get the completion response. +function loadTestFrame(id) { + return new Promise(function(resolve, reject) { + + var iframe = document.getElementById(id); + iframe.setAttribute("src", "gethashFrame.html"); + + iframe.onload = function() { + resolve(); + }; + }); +} + +// add 4-bytes prefixes to local database, so when we access the url, +// it will trigger gethash request. +function addPrefixToDB(list, url) { + var testData = [{ db: list, url: url, len: 4 }]; + + return classifierHelper.addUrlToDB(testData) + .catch(function(err) { + ok(false, "Couldn't update classifier. Error code: " + err); + // Abort test. + SimpleTest.finish(); + }); +} + +// calculate the fullhash and send it to gethash server +function addCompletionToServer(list, url) { + return new Promise(function(resolve, reject) { + var listParam = "list=" + list; + var fullhashParam = "fullhash=" + hash(url); + + var xhr = new XMLHttpRequest; + xhr.open("PUT", GETHASH_URL + "?" + + listParam + "&" + + fullhashParam, true); + xhr.setRequestHeader("Content-Type", "text/plain"); + xhr.onreadystatechange = function() { + if (this.readyState == this.DONE) { + resolve(); + } + }; + xhr.send(); + }); +} + +function hash(str) { + function bytesFromString(str) { + var converter = + SpecialPowers.Cc["@mozilla.org/intl/scriptableunicodeconverter"] + .createInstance(SpecialPowers.Ci.nsIScriptableUnicodeConverter); + converter.charset = "UTF-8"; + return converter.convertToByteArray(str); + } + + var hasher = SpecialPowers.Cc["@mozilla.org/security/hash;1"] + .createInstance(SpecialPowers.Ci.nsICryptoHash); + + var data = bytesFromString(str); + hasher.init(hasher.SHA256); + hasher.update(data, data.length); + + return hasher.finish(true); +} + +function setup404() { + shouldLoad = true; + + classifierHelper.allowCompletion([MALWARE_LIST, UNWANTED_LIST], NOTEXIST_URL); + + return Promise.all([ + addPrefixToDB(MALWARE_LIST, MALWARE_HOST), + addPrefixToDB(UNWANTED_LIST, UNWANTED_HOST) + ]); +} + +function setup() { + classifierHelper.allowCompletion([MALWARE_LIST, UNWANTED_LIST], GETHASH_URL); + + return Promise.all([ + addPrefixToDB(MALWARE_LIST, MALWARE_HOST), + addPrefixToDB(UNWANTED_LIST, UNWANTED_HOST), + addCompletionToServer(MALWARE_LIST, MALWARE_HOST), + addCompletionToServer(UNWANTED_LIST, UNWANTED_HOST), + ]); +} + +// manually reset DB to make sure next test won't be affected by cache. +function reset() { + return classifierHelper.resetDB; +} + +function runTest() { + Promise.resolve() + // This test resources get blocked when gethash returns successfully + .then(classifierHelper.waitForInit) + .then(setup) + .then(() => loadTestFrame("testFrame1")) + .then(reset) + // This test resources are not blocked when gethash returns an error + .then(setup404) + .then(() => loadTestFrame("testFrame2")) + .then(function() { + SimpleTest.finish(); + }).catch(function(e) { + ok(false, "Some test failed with error " + e); + SimpleTest.finish(); + }); +} + +SimpleTest.waitForExplicitFinish(); + +// 'network.predictor.enabled' is disabled because if other testcase load +// evil.js, evil.css ...etc resources, it may cause we load them from cache +// directly and bypass classifier check +SpecialPowers.pushPrefEnv({"set": [ + ["browser.safebrowsing.malware.enabled", true], + ["network.predictor.enabled", false], + ["urlclassifier.gethash.timeout_ms", 30000], +]}, runTest); + +</script> +</pre> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/test_lookup_system_principal.html b/toolkit/components/url-classifier/tests/mochitest/test_lookup_system_principal.html new file mode 100644 index 000000000..fa61e6a00 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_lookup_system_principal.html @@ -0,0 +1,29 @@ +<!DOCTYPE HTML> +<html> +<head> + <title>Test that lookup() on a system principal doesn't crash</title> + <script type="text/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"> +</head> + +<body> +<div id="content" style="display: none"> +</div> +<pre id="test"> + +<script type="text/javascript"> + +var Cc = Components.classes; +var Ci = Components.interfaces; + +var dbService = Cc["@mozilla.org/url-classifier/dbservice;1"] + .getService(Ci.nsIUrlClassifierDBService); + +dbService.lookup(document.nodePrincipal, "", function(arg) {}); + +ok(true, "lookup() didn't crash"); + +</script> +</pre> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/test_privatebrowsing_trackingprotection.html b/toolkit/components/url-classifier/tests/mochitest/test_privatebrowsing_trackingprotection.html new file mode 100644 index 000000000..02ef57b46 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_privatebrowsing_trackingprotection.html @@ -0,0 +1,154 @@ +<!DOCTYPE HTML> +<!-- Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ --> +<html> +<head> + <title>Test Tracking Protection in Private Browsing mode</title> + <script type="text/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"> +</head> + +<body> +<p id="display"></p> +<div id="content" style="display: none"> +</div> +<pre id="test"> + +<script class="testbody" type="text/javascript"> + +var Cc = SpecialPowers.Cc; +var Ci = SpecialPowers.Ci; + +var mainWindow = window.QueryInterface(Ci.nsIInterfaceRequestor) + .getInterface(Ci.nsIWebNavigation) + .QueryInterface(Ci.nsIDocShellTreeItem) + .rootTreeItem + .QueryInterface(Ci.nsIInterfaceRequestor) + .getInterface(Ci.nsIDOMWindow); +var contentPage = "http://www.itisatrap.org/tests/toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedPBFrame.html"; + +Components.utils.import("resource://gre/modules/Services.jsm"); +Components.utils.import("resource://testing-common/UrlClassifierTestUtils.jsm"); + +function whenDelayedStartupFinished(aWindow, aCallback) { + Services.obs.addObserver(function observer(aSubject, aTopic) { + if (aWindow == aSubject) { + Services.obs.removeObserver(observer, aTopic); + setTimeout(aCallback, 0); + } + }, "browser-delayed-startup-finished", false); +} + +function testOnWindow(aPrivate, aCallback) { + var win = mainWindow.OpenBrowserWindow({private: aPrivate}); + win.addEventListener("load", function onLoad() { + win.removeEventListener("load", onLoad, false); + whenDelayedStartupFinished(win, function() { + win.addEventListener("DOMContentLoaded", function onInnerLoad() { + if (win.content.location.href != contentPage) { + win.gBrowser.loadURI(contentPage); + return; + } + win.removeEventListener("DOMContentLoaded", onInnerLoad, true); + + win.content.addEventListener('load', function innerLoad2() { + win.content.removeEventListener('load', innerLoad2, false); + SimpleTest.executeSoon(function() { aCallback(win); }); + }, false, true); + }, true); + SimpleTest.executeSoon(function() { win.gBrowser.loadURI(contentPage); }); + }); + }, true); +} + +var badids = [ + "badscript", + "badimage", + "badcss" +]; + +function checkLoads(aWindow, aBlocked) { + var win = aWindow.content; + is(win.document.getElementById("badscript").dataset.touched, aBlocked ? "no" : "yes", "Should not load tracking javascript"); + is(win.document.getElementById("badimage").dataset.touched, aBlocked ? "no" : "yes", "Should not load tracking images"); + is(win.document.getElementById("goodscript").dataset.touched, "yes", "Should load whitelisted tracking javascript"); + + var elt = win.document.getElementById("styleCheck"); + var style = win.document.defaultView.getComputedStyle(elt, ""); + isnot(style.visibility, aBlocked ? "hidden" : "", "Should not load tracking css"); + + is(win.document.blockedTrackingNodeCount, aBlocked ? badids.length : 0, "Should identify all tracking elements"); + + var blockedTrackingNodes = win.document.blockedTrackingNodes; + + // Make sure that every node in blockedTrackingNodes exists in the tree + // (that may not always be the case but do not expect any nodes to disappear + // from the tree here) + var allNodeMatch = true; + for (var i = 0; i < blockedTrackingNodes.length; i++) { + var nodeMatch = false; + for (var j = 0; j < badids.length && !nodeMatch; j++) { + nodeMatch = nodeMatch || + (blockedTrackingNodes[i] == win.document.getElementById(badids[j])); + } + + allNodeMatch = allNodeMatch && nodeMatch; + } + is(allNodeMatch, true, "All annotated nodes are expected in the tree"); + + // Make sure that every node with a badid (see badids) is found in the + // blockedTrackingNodes. This tells us if we are neglecting to annotate + // some nodes + allNodeMatch = true; + for (var j = 0; j < badids.length; j++) { + var nodeMatch = false; + for (var i = 0; i < blockedTrackingNodes.length && !nodeMatch; i++) { + nodeMatch = nodeMatch || + (blockedTrackingNodes[i] == win.document.getElementById(badids[j])); + } + + allNodeMatch = allNodeMatch && nodeMatch; + } + is(allNodeMatch, aBlocked, "All tracking nodes are expected to be annotated as such"); +} + +SpecialPowers.pushPrefEnv( + {"set" : [["urlclassifier.trackingTable", "test-track-simple"], + ["privacy.trackingprotection.enabled", false], + ["privacy.trackingprotection.pbmode.enabled", true], + ["channelclassifier.allowlist_example", true]]}, + test); + +function test() { + SimpleTest.registerCleanupFunction(UrlClassifierTestUtils.cleanupTestTrackers); + UrlClassifierTestUtils.addTestTrackers().then(() => { + // Normal mode, with the pref (trackers should be loaded) + testOnWindow(false, function(aWindow) { + checkLoads(aWindow, false); + aWindow.close(); + + // Private Browsing, with the pref (trackers should be blocked) + testOnWindow(true, function(aWindow) { + checkLoads(aWindow, true); + aWindow.close(); + + // Private Browsing, without the pref (trackers should be loaded) + SpecialPowers.setBoolPref("privacy.trackingprotection.pbmode.enabled", false); + testOnWindow(true, function(aWindow) { + checkLoads(aWindow, false); + aWindow.close(); + SimpleTest.finish(); + }); + }); + }); + }); +} + +SimpleTest.waitForExplicitFinish(); + +</script> + +</pre> +<iframe id="testFrame" width="100%" height="100%" onload=""></iframe> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/test_safebrowsing_bug1272239.html b/toolkit/components/url-classifier/tests/mochitest/test_safebrowsing_bug1272239.html new file mode 100644 index 000000000..8066c2a37 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_safebrowsing_bug1272239.html @@ -0,0 +1,87 @@ +<!DOCTYPE HTML> +<html> +<head> + <title>Bug 1272239 - Only tables with provider could register gethash url in listmanager.</title> + <script type="text/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"> +</head> + +<body> +<p id="display"></p> +<div id="content" style="display: none"> +</div> +<pre id="test"> + +<script class="testbody" type="text/javascript"> + +var Cc = SpecialPowers.Cc; +var Ci = SpecialPowers.Ci; + +// List all the tables +const prefs = [ + "urlclassifier.phishTable", + "urlclassifier.malwareTable", + "urlclassifier.downloadBlockTable", + "urlclassifier.downloadAllowTable", + "urlclassifier.trackingTable", + "urlclassifier.trackingWhitelistTable", + "urlclassifier.blockedTable" +]; + +var prefService = Cc["@mozilla.org/preferences-service;1"] + .getService(Ci.nsIPrefService); + +// Get providers +var providers = {}; + +var branch = prefService.getBranch("browser.safebrowsing.provider."); +var children = branch.getChildList("", {}); + +for (var child of children) { + var prefComponents = child.split("."); + var providerName = prefComponents[0]; + providers[providerName] = {}; +} + +// Get lists from |browser.safebrowsing.provider.PROVIDER_NAME.lists| preference. +var listsWithProvider = []; +var listsToProvider = []; +for (var provider in providers) { + var pref = "browser.safebrowsing.provider." + provider + ".lists"; + var list = SpecialPowers.getCharPref(pref).split(","); + + listsToProvider = listsToProvider.concat(list.map( () => { return provider; })); + listsWithProvider = listsWithProvider.concat(list); +} + +// Get all the lists +var lists = []; +for (var pref of prefs) { + lists = lists.concat(SpecialPowers.getCharPref(pref).split(",")); +} + +var listmanager = Cc["@mozilla.org/url-classifier/listmanager;1"]. + getService(Ci.nsIUrlListManager); + +for (var list of lists) { + if (!list) + continue; + + // For lists having a provider, it should have a correct gethash url + // For lists without a provider, for example, test-malware-simple, it should not + // have a gethash url. + var url = listmanager.getGethashUrl(list); + var index = listsWithProvider.indexOf(list); + if (index >= 0) { + var provider = listsToProvider[index]; + var pref = "browser.safebrowsing.provider." + provider + ".gethashURL"; + is(url, SpecialPowers.getCharPref(pref), list + " matches its gethash url"); + } else { + is(url, "", list + " should not have a gethash url"); + } +} + +</script> +</pre> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_bug1157081.html b/toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_bug1157081.html new file mode 100644 index 000000000..7611dd245 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_bug1157081.html @@ -0,0 +1,107 @@ +<!DOCTYPE HTML> +<!-- Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ --> +<html> +<head> + <title>Test Tracking Protection with and without Safe Browsing (Bug #1157081)</title> + <script type="text/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"> +</head> + +<body> +<p id="display"></p> +<div id="content" style="display: none"> +</div> +<pre id="test"> + +<script class="testbody" type="text/javascript"> + +var Cc = SpecialPowers.Cc; +var Ci = SpecialPowers.Ci; + +var mainWindow = window.QueryInterface(Ci.nsIInterfaceRequestor) + .getInterface(Ci.nsIWebNavigation) + .QueryInterface(Ci.nsIDocShellTreeItem) + .rootTreeItem + .QueryInterface(Ci.nsIInterfaceRequestor) + .getInterface(Ci.nsIDOMWindow); +var contentPage = "chrome://mochitests/content/chrome/toolkit/components/url-classifier/tests/mochitest/classifiedAnnotatedPBFrame.html" + +Components.utils.import("resource://gre/modules/Services.jsm"); +Components.utils.import("resource://testing-common/UrlClassifierTestUtils.jsm"); + +function whenDelayedStartupFinished(aWindow, aCallback) { + Services.obs.addObserver(function observer(aSubject, aTopic) { + if (aWindow == aSubject) { + Services.obs.removeObserver(observer, aTopic); + setTimeout(aCallback, 0); + } + }, "browser-delayed-startup-finished", false); +} + +function testOnWindow(aCallback) { + var win = mainWindow.OpenBrowserWindow(); + win.addEventListener("load", function onLoad() { + win.removeEventListener("load", onLoad, false); + whenDelayedStartupFinished(win, function() { + win.addEventListener("DOMContentLoaded", function onInnerLoad() { + if (win.content.location.href != contentPage) { + win.gBrowser.loadURI(contentPage); + return; + } + win.removeEventListener("DOMContentLoaded", onInnerLoad, true); + + win.content.addEventListener('load', function innerLoad2() { + win.content.removeEventListener('load', innerLoad2, false); + SimpleTest.executeSoon(function() { aCallback(win); }); + }, false, true); + }, true); + SimpleTest.executeSoon(function() { win.gBrowser.loadURI(contentPage); }); + }); + }, true); +} + +var badids = [ + "badscript" +]; + +function checkLoads(aWindow, aBlocked) { + var win = aWindow.content; + is(win.document.getElementById("badscript").dataset.touched, aBlocked ? "no" : "yes", "Should not load tracking javascript"); +} + +SpecialPowers.pushPrefEnv( + {"set" : [["urlclassifier.trackingTable", "test-track-simple"], + ["privacy.trackingprotection.enabled", true], + ["browser.safebrowsing.malware.enabled", false], + ["browser.safebrowsing.phishing.enabled", false], + ["channelclassifier.allowlist_example", true]]}, + test); + +function test() { + SimpleTest.registerCleanupFunction(UrlClassifierTestUtils.cleanupTestTrackers); + UrlClassifierTestUtils.addTestTrackers().then(() => { + // Safe Browsing turned OFF, tracking protection should work nevertheless + testOnWindow(function(aWindow) { + checkLoads(aWindow, true); + aWindow.close(); + + // Safe Browsing turned ON, tracking protection should still work + SpecialPowers.setBoolPref("browser.safebrowsing.phishing.enabled", true); + testOnWindow(function(aWindow) { + checkLoads(aWindow, true); + aWindow.close(); + SimpleTest.finish(); + }); + }); + }); +} + +SimpleTest.waitForExplicitFinish(); + +</script> + +</pre> +<iframe id="testFrame" width="100%" height="100%" onload=""></iframe> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_whitelist.html b/toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_whitelist.html new file mode 100644 index 000000000..29de0dfed --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_whitelist.html @@ -0,0 +1,153 @@ +<!DOCTYPE HTML> +<!-- Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ --> +<html> +<head> + <title>Test Tracking Protection in Private Browsing mode</title> + <script type="text/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"> +</head> + +<body> +<p id="display"></p> +<div id="content" style="display: none"> +</div> +<pre id="test"> + +<script class="testbody" type="text/javascript"> + +var Cc = SpecialPowers.Cc; +var Ci = SpecialPowers.Ci; + +var mainWindow = window.QueryInterface(Ci.nsIInterfaceRequestor) + .getInterface(Ci.nsIWebNavigation) + .QueryInterface(Ci.nsIDocShellTreeItem) + .rootTreeItem + .QueryInterface(Ci.nsIInterfaceRequestor) + .getInterface(Ci.nsIDOMWindow); +var contentPage1 = "http://www.itisatrap.org/tests/toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html"; +var contentPage2 = "http://example.com/tests/toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html"; + +Components.utils.import("resource://gre/modules/Services.jsm"); +Components.utils.import("resource://testing-common/UrlClassifierTestUtils.jsm"); + +function whenDelayedStartupFinished(aWindow, aCallback) { + Services.obs.addObserver(function observer(aSubject, aTopic) { + if (aWindow == aSubject) { + Services.obs.removeObserver(observer, aTopic); + setTimeout(aCallback, 0); + } + }, "browser-delayed-startup-finished", false); +} + +function testOnWindow(contentPage, aCallback) { + var win = mainWindow.OpenBrowserWindow(); + win.addEventListener("load", function onLoad() { + win.removeEventListener("load", onLoad, false); + whenDelayedStartupFinished(win, function() { + win.addEventListener("DOMContentLoaded", function onInnerLoad() { + if (win.content.location.href != contentPage) { + win.gBrowser.loadURI(contentPage); + return; + } + win.removeEventListener("DOMContentLoaded", onInnerLoad, true); + + win.content.addEventListener('load', function innerLoad2() { + win.content.removeEventListener('load', innerLoad2, false); + SimpleTest.executeSoon(function() { aCallback(win); }); + }, false, true); + }, true); + SimpleTest.executeSoon(function() { win.gBrowser.loadURI(contentPage); }); + }); + }, true); +} + +var alwaysbadids = [ + "badscript", +]; + +function checkLoads(aWindow, aWhitelisted) { + var win = aWindow.content; + is(win.document.getElementById("badscript").dataset.touched, "no", "Should not load tracking javascript"); + is(win.document.getElementById("goodscript").dataset.touched, aWhitelisted ? "yes" : "no", "Should load whitelisted tracking javascript"); + + var badids = alwaysbadids.slice(); + if (!aWhitelisted) { + badids.push("goodscript"); + } + is(win.document.blockedTrackingNodeCount, badids.length, "Should identify all tracking elements"); + + var blockedTrackingNodes = win.document.blockedTrackingNodes; + + // Make sure that every node in blockedTrackingNodes exists in the tree + // (that may not always be the case but do not expect any nodes to disappear + // from the tree here) + var allNodeMatch = true; + for (var i = 0; i < blockedTrackingNodes.length; i++) { + var nodeMatch = false; + for (var j = 0; j < badids.length && !nodeMatch; j++) { + nodeMatch = nodeMatch || + (blockedTrackingNodes[i] == win.document.getElementById(badids[j])); + } + + allNodeMatch = allNodeMatch && nodeMatch; + } + is(allNodeMatch, true, "All annotated nodes are expected in the tree"); + + // Make sure that every node with a badid (see badids) is found in the + // blockedTrackingNodes. This tells us if we are neglecting to annotate + // some nodes + allNodeMatch = true; + for (var j = 0; j < badids.length; j++) { + var nodeMatch = false; + for (var i = 0; i < blockedTrackingNodes.length && !nodeMatch; i++) { + nodeMatch = nodeMatch || + (blockedTrackingNodes[i] == win.document.getElementById(badids[j])); + } + + allNodeMatch = allNodeMatch && nodeMatch; + } + is(allNodeMatch, true, "All tracking nodes are expected to be annotated as such"); +} + +SpecialPowers.pushPrefEnv( + {"set" : [["privacy.trackingprotection.enabled", true], + ["channelclassifier.allowlist_example", true]]}, + test); + +function test() { + SimpleTest.registerCleanupFunction(UrlClassifierTestUtils.cleanupTestTrackers); + UrlClassifierTestUtils.addTestTrackers().then(() => { + // Load the test from a URL on the whitelist + testOnWindow(contentPage1, function(aWindow) { + checkLoads(aWindow, true); + aWindow.close(); + + // Load the test from a URL that's NOT on the whitelist + testOnWindow(contentPage2, function(aWindow) { + checkLoads(aWindow, false); + aWindow.close(); + + // Load the test from a URL on the whitelist but without the whitelist + SpecialPowers.pushPrefEnv({"set" : [["urlclassifier.trackingWhitelistTable", ""]]}, + function() { + testOnWindow(contentPage1, function(aWindow) { + checkLoads(aWindow, false); + aWindow.close(); + SimpleTest.finish(); + }); + }); + + }); + }); + }); +} + +SimpleTest.waitForExplicitFinish(); + +</script> + +</pre> +<iframe id="testFrame" width="100%" height="100%" onload=""></iframe> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/track.html b/toolkit/components/url-classifier/tests/mochitest/track.html new file mode 100644 index 000000000..8785e7c5b --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/track.html @@ -0,0 +1,7 @@ +<html> + <head> + </head> + <body> + <h1>Tracking Works!</h1> + </body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/unwantedWorker.js b/toolkit/components/url-classifier/tests/mochitest/unwantedWorker.js new file mode 100644 index 000000000..ac34977d7 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/unwantedWorker.js @@ -0,0 +1,3 @@ +onmessage = function() { + postMessage("loaded bad file"); +} diff --git a/toolkit/components/url-classifier/tests/mochitest/update.sjs b/toolkit/components/url-classifier/tests/mochitest/update.sjs new file mode 100644 index 000000000..53efaafdf --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/update.sjs @@ -0,0 +1,114 @@ +const CC = Components.Constructor; +const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1", + "nsIBinaryInputStream", + "setInputStream"); + +function handleRequest(request, response) +{ + var query = {}; + request.queryString.split('&').forEach(function (val) { + var idx = val.indexOf('='); + query[val.slice(0, idx)] = unescape(val.slice(idx + 1)); + }); + + // Store fullhash in the server side. + if ("list" in query && "fullhash" in query) { + // In the server side we will store: + // 1. All the full hashes for a given list + // 2. All the lists we have right now + // data is separate by '\n' + let list = query["list"]; + let hashes = getState(list); + + let hash = base64ToString(query["fullhash"]); + hashes += hash + "\n"; + setState(list, hashes); + + let lists = getState("lists"); + if (lists.indexOf(list) == -1) { + lists += list + "\n"; + setState("lists", lists); + } + + return; + } + + var body = new BinaryInputStream(request.bodyInputStream); + var avail; + var bytes = []; + + while ((avail = body.available()) > 0) { + Array.prototype.push.apply(bytes, body.readByteArray(avail)); + } + + var responseBody = parseV2Request(bytes); + + response.setHeader("Content-Type", "text/plain", false); + response.write(responseBody); +} + +function parseV2Request(bytes) { + var table = String.fromCharCode.apply(this, bytes).slice(0,-2); + + var ret = ""; + getState("lists").split("\n").forEach(function(list) { + if (list == table) { + var completions = getState(list).split("\n"); + ret += "n:1000\n" + ret += "i:" + list + "\n"; + ret += "a:1:32:" + 32*(completions.length - 1) + "\n"; + + for (var completion of completions) { + ret += completion; + } + } + }); + + return ret; +} + +/* Convert Base64 data to a string */ +const toBinaryTable = [ + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63, + 52,53,54,55, 56,57,58,59, 60,61,-1,-1, -1, 0,-1,-1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, + 15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1, + -1,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, + 41,42,43,44, 45,46,47,48, 49,50,51,-1, -1,-1,-1,-1 +]; +const base64Pad = '='; + +function base64ToString(data) { + var result = ''; + var leftbits = 0; // number of bits decoded, but yet to be appended + var leftdata = 0; // bits decoded, but yet to be appended + + // Convert one by one. + for (var i = 0; i < data.length; i++) { + var c = toBinaryTable[data.charCodeAt(i) & 0x7f]; + var padding = (data[i] == base64Pad); + // Skip illegal characters and whitespace + if (c == -1) continue; + + // Collect data into leftdata, update bitcount + leftdata = (leftdata << 6) | c; + leftbits += 6; + + // If we have 8 or more bits, append 8 bits to the result + if (leftbits >= 8) { + leftbits -= 8; + // Append if not padding. + if (!padding) + result += String.fromCharCode((leftdata >> leftbits) & 0xff); + leftdata &= (1 << leftbits) - 1; + } + } + + // If there are any bits left, the base64 string was corrupted + if (leftbits) + throw Components.Exception('Corrupted base64 string'); + + return result; +} diff --git a/toolkit/components/url-classifier/tests/mochitest/vp9.webm b/toolkit/components/url-classifier/tests/mochitest/vp9.webm Binary files differnew file mode 100644 index 000000000..221877e30 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/vp9.webm diff --git a/toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html b/toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html new file mode 100644 index 000000000..620416fc7 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html @@ -0,0 +1,15 @@ +<!DOCTYPE HTML> +<!-- Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ --> +<html> +<head> +<title></title> +</head> +<body> + +<script id="badscript" data-touched="not sure" src="http://trackertest.org/tests/toolkit/components/url-classifier/tests/mochitest/evil.js" onload="this.dataset.touched = 'yes';" onerror="this.dataset.touched = 'no';"></script> + +<script id="goodscript" data-touched="not sure" src="http://itisatracker.org/tests/toolkit/components/url-classifier/tests/mochitest/good.js" onload="this.dataset.touched = 'yes';" onerror="this.dataset.touched = 'no';"></script> + +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/mochitest/workerFrame.html b/toolkit/components/url-classifier/tests/mochitest/workerFrame.html new file mode 100644 index 000000000..69e8dd007 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/workerFrame.html @@ -0,0 +1,65 @@ +<html> +<head> +<title></title> + +<script type="text/javascript"> + +function startCleanWorker() { + var worker = new Worker("cleanWorker.js"); + + worker.onmessage = function(event) { + if (event.data == "success") { + window.parent.postMessage("success:blocked importScripts('evilWorker.js')", "*"); + } else { + window.parent.postMessage("failure:failed to block importScripts('evilWorker.js')", "*"); + } + window.parent.postMessage("finish", "*"); + }; + + worker.onerror = function(event) { + window.parent.postmessage("failure:failed to load cleanWorker.js", "*"); + window.parent.postMessage("finish", "*"); + }; + + worker.postMessage(""); +} + +function startEvilWorker() { + var worker = new Worker("evilWorker.js"); + + worker.onmessage = function(event) { + window.parent.postMessage("failure:failed to block evilWorker.js", "*"); + startUnwantedWorker(); + }; + + worker.onerror = function(event) { + window.parent.postMessage("success:blocked evilWorker.js", "*"); + startUnwantedWorker(); + }; + + worker.postMessage(""); +} + +function startUnwantedWorker() { + var worker = new Worker("unwantedWorker.js"); + + worker.onmessage = function(event) { + window.parent.postMessage("failure:failed to block unwantedWorker.js", "*"); + startCleanWorker(); + }; + + worker.onerror = function(event) { + window.parent.postMessage("success:blocked unwantedWorker.js", "*"); + startCleanWorker(); + }; + + worker.postMessage(""); +} + +</script> + +</head> + +<body onload="startEvilWorker()"> +</body> +</html> diff --git a/toolkit/components/url-classifier/tests/moz.build b/toolkit/components/url-classifier/tests/moz.build new file mode 100644 index 000000000..599727ab9 --- /dev/null +++ b/toolkit/components/url-classifier/tests/moz.build @@ -0,0 +1,18 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +MOCHITEST_MANIFESTS += ['mochitest/mochitest.ini'] +MOCHITEST_CHROME_MANIFESTS += ['mochitest/chrome.ini'] +XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini'] + +JAR_MANIFESTS += ['jar.mn'] + +TESTING_JS_MODULES += [ + 'UrlClassifierTestUtils.jsm', +] + +if CONFIG['ENABLE_TESTS']: + DIRS += ['gtest'] diff --git a/toolkit/components/url-classifier/tests/unit/.eslintrc.js b/toolkit/components/url-classifier/tests/unit/.eslintrc.js new file mode 100644 index 000000000..d35787cd2 --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/.eslintrc.js @@ -0,0 +1,7 @@ +"use strict"; + +module.exports = { + "extends": [ + "../../../../../testing/xpcshell/xpcshell.eslintrc.js" + ] +}; diff --git a/toolkit/components/url-classifier/tests/unit/data/digest1.chunk b/toolkit/components/url-classifier/tests/unit/data/digest1.chunk Binary files differnew file mode 100644 index 000000000..3850373c1 --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/data/digest1.chunk diff --git a/toolkit/components/url-classifier/tests/unit/data/digest2.chunk b/toolkit/components/url-classifier/tests/unit/data/digest2.chunk new file mode 100644 index 000000000..738c96f6b --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/data/digest2.chunk @@ -0,0 +1,2 @@ +a:5:32:32 +“Ê_Há^˜aÍ7ÂÙ]´=#ÌnmåÃøún‹æo—ÌQ‰
\ No newline at end of file diff --git a/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js b/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js new file mode 100644 index 000000000..21849ced7 --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js @@ -0,0 +1,429 @@ +//* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- * +function dumpn(s) { + dump(s + "\n"); +} + +const NS_APP_USER_PROFILE_50_DIR = "ProfD"; +const NS_APP_USER_PROFILE_LOCAL_50_DIR = "ProfLD"; + +var Cc = Components.classes; +var Ci = Components.interfaces; +var Cu = Components.utils; +var Cr = Components.results; + +Cu.import("resource://testing-common/httpd.js"); + +do_get_profile(); + +var dirSvc = Cc["@mozilla.org/file/directory_service;1"].getService(Ci.nsIProperties); + +var iosvc = Cc["@mozilla.org/network/io-service;1"].getService(Ci.nsIIOService); + +var secMan = Cc["@mozilla.org/scriptsecuritymanager;1"] + .getService(Ci.nsIScriptSecurityManager); + +// Disable hashcompleter noise for tests +var prefBranch = Cc["@mozilla.org/preferences-service;1"]. + getService(Ci.nsIPrefBranch); +prefBranch.setIntPref("urlclassifier.gethashnoise", 0); + +// Enable malware/phishing checking for tests +prefBranch.setBoolPref("browser.safebrowsing.malware.enabled", true); +prefBranch.setBoolPref("browser.safebrowsing.blockedURIs.enabled", true); +prefBranch.setBoolPref("browser.safebrowsing.phishing.enabled", true); + +// Enable all completions for tests +prefBranch.setCharPref("urlclassifier.disallow_completions", ""); + +// Hash completion timeout +prefBranch.setIntPref("urlclassifier.gethash.timeout_ms", 5000); + +function delFile(name) { + try { + // Delete a previously created sqlite file + var file = dirSvc.get('ProfLD', Ci.nsIFile); + file.append(name); + if (file.exists()) + file.remove(false); + } catch(e) { + } +} + +function cleanUp() { + delFile("urlclassifier3.sqlite"); + delFile("safebrowsing/classifier.hashkey"); + delFile("safebrowsing/test-phish-simple.sbstore"); + delFile("safebrowsing/test-malware-simple.sbstore"); + delFile("safebrowsing/test-unwanted-simple.sbstore"); + delFile("safebrowsing/test-block-simple.sbstore"); + delFile("safebrowsing/test-track-simple.sbstore"); + delFile("safebrowsing/test-trackwhite-simple.sbstore"); + delFile("safebrowsing/test-phish-simple.pset"); + delFile("safebrowsing/test-malware-simple.pset"); + delFile("safebrowsing/test-unwanted-simple.pset"); + delFile("safebrowsing/test-block-simple.pset"); + delFile("safebrowsing/test-track-simple.pset"); + delFile("safebrowsing/test-trackwhite-simple.pset"); + delFile("safebrowsing/moz-phish-simple.sbstore"); + delFile("safebrowsing/moz-phish-simple.pset"); + delFile("testLarge.pset"); + delFile("testNoDelta.pset"); +} + +// Update uses allTables by default +var allTables = "test-phish-simple,test-malware-simple,test-unwanted-simple,test-track-simple,test-trackwhite-simple,test-block-simple"; +var mozTables = "moz-phish-simple"; + +var dbservice = Cc["@mozilla.org/url-classifier/dbservice;1"].getService(Ci.nsIUrlClassifierDBService); +var streamUpdater = Cc["@mozilla.org/url-classifier/streamupdater;1"] + .getService(Ci.nsIUrlClassifierStreamUpdater); + + +/* + * Builds an update from an object that looks like: + *{ "test-phish-simple" : [{ + * "chunkType" : "a", // 'a' is assumed if not specified + * "chunkNum" : 1, // numerically-increasing chunk numbers are assumed + * // if not specified + * "urls" : [ "foo.com/a", "foo.com/b", "bar.com/" ] + * } + */ + +function buildUpdate(update, hashSize) { + if (!hashSize) { + hashSize = 32; + } + var updateStr = "n:1000\n"; + + for (var tableName in update) { + if (tableName != "") + updateStr += "i:" + tableName + "\n"; + var chunks = update[tableName]; + for (var j = 0; j < chunks.length; j++) { + var chunk = chunks[j]; + var chunkType = chunk.chunkType ? chunk.chunkType : 'a'; + var chunkNum = chunk.chunkNum ? chunk.chunkNum : j; + updateStr += chunkType + ':' + chunkNum + ':' + hashSize; + + if (chunk.urls) { + var chunkData = chunk.urls.join("\n"); + updateStr += ":" + chunkData.length + "\n" + chunkData; + } + + updateStr += "\n"; + } + } + + return updateStr; +} + +function buildPhishingUpdate(chunks, hashSize) { + return buildUpdate({"test-phish-simple" : chunks}, hashSize); +} + +function buildMalwareUpdate(chunks, hashSize) { + return buildUpdate({"test-malware-simple" : chunks}, hashSize); +} + +function buildUnwantedUpdate(chunks, hashSize) { + return buildUpdate({"test-unwanted-simple" : chunks}, hashSize); +} + +function buildBlockedUpdate(chunks, hashSize) { + return buildUpdate({"test-block-simple" : chunks}, hashSize); +} + +function buildMozPhishingUpdate(chunks, hashSize) { + return buildUpdate({"moz-phish-simple" : chunks}, hashSize); +} + +function buildBareUpdate(chunks, hashSize) { + return buildUpdate({"" : chunks}, hashSize); +} + +/** + * Performs an update of the dbservice manually, bypassing the stream updater + */ +function doSimpleUpdate(updateText, success, failure) { + var listener = { + QueryInterface: function(iid) + { + if (iid.equals(Ci.nsISupports) || + iid.equals(Ci.nsIUrlClassifierUpdateObserver)) + return this; + throw Cr.NS_ERROR_NO_INTERFACE; + }, + + updateUrlRequested: function(url) { }, + streamFinished: function(status) { }, + updateError: function(errorCode) { failure(errorCode); }, + updateSuccess: function(requestedTimeout) { success(requestedTimeout); } + }; + + dbservice.beginUpdate(listener, allTables); + dbservice.beginStream("", ""); + dbservice.updateStream(updateText); + dbservice.finishStream(); + dbservice.finishUpdate(); +} + +/** + * Simulates a failed database update. + */ +function doErrorUpdate(tables, success, failure) { + var listener = { + QueryInterface: function(iid) + { + if (iid.equals(Ci.nsISupports) || + iid.equals(Ci.nsIUrlClassifierUpdateObserver)) + return this; + throw Cr.NS_ERROR_NO_INTERFACE; + }, + + updateUrlRequested: function(url) { }, + streamFinished: function(status) { }, + updateError: function(errorCode) { success(errorCode); }, + updateSuccess: function(requestedTimeout) { failure(requestedTimeout); } + }; + + dbservice.beginUpdate(listener, tables, null); + dbservice.beginStream("", ""); + dbservice.cancelUpdate(); +} + +/** + * Performs an update of the dbservice using the stream updater and a + * data: uri + */ +function doStreamUpdate(updateText, success, failure, downloadFailure) { + var dataUpdate = "data:," + encodeURIComponent(updateText); + + if (!downloadFailure) { + downloadFailure = failure; + } + + streamUpdater.downloadUpdates(allTables, "", true, + dataUpdate, success, failure, downloadFailure); +} + +var gAssertions = { + +tableData : function(expectedTables, cb) +{ + dbservice.getTables(function(tables) { + // rebuild the tables in a predictable order. + var parts = tables.split("\n"); + while (parts[parts.length - 1] == '') { + parts.pop(); + } + parts.sort(); + tables = parts.join("\n"); + + do_check_eq(tables, expectedTables); + cb(); + }); +}, + +checkUrls: function(urls, expected, cb, useMoz = false) +{ + // work with a copy of the list. + urls = urls.slice(0); + var doLookup = function() { + if (urls.length > 0) { + var tables = useMoz ? mozTables : allTables; + var fragment = urls.shift(); + var principal = secMan.createCodebasePrincipal(iosvc.newURI("http://" + fragment, null, null), {}); + dbservice.lookup(principal, tables, + function(arg) { + do_check_eq(expected, arg); + doLookup(); + }, true); + } else { + cb(); + } + }; + doLookup(); +}, + +checkTables: function(url, expected, cb) +{ + var principal = secMan.createCodebasePrincipal(iosvc.newURI("http://" + url, null, null), {}); + dbservice.lookup(principal, allTables, function(tables) { + // Rebuild tables in a predictable order. + var parts = tables.split(","); + while (parts[parts.length - 1] == '') { + parts.pop(); + } + parts.sort(); + tables = parts.join(","); + do_check_eq(tables, expected); + cb(); + }, true); +}, + +urlsDontExist: function(urls, cb) +{ + this.checkUrls(urls, '', cb); +}, + +urlsExist: function(urls, cb) +{ + this.checkUrls(urls, 'test-phish-simple', cb); +}, + +malwareUrlsExist: function(urls, cb) +{ + this.checkUrls(urls, 'test-malware-simple', cb); +}, + +unwantedUrlsExist: function(urls, cb) +{ + this.checkUrls(urls, 'test-unwanted-simple', cb); +}, + +blockedUrlsExist: function(urls, cb) +{ + this.checkUrls(urls, 'test-block-simple', cb); +}, + +mozPhishingUrlsExist: function(urls, cb) +{ + this.checkUrls(urls, 'moz-phish-simple', cb, true); +}, + +subsDontExist: function(urls, cb) +{ + // XXX: there's no interface for checking items in the subs table + cb(); +}, + +subsExist: function(urls, cb) +{ + // XXX: there's no interface for checking items in the subs table + cb(); +}, + +urlExistInMultipleTables: function(data, cb) +{ + this.checkTables(data["url"], data["tables"], cb); +} + +}; + +/** + * Check a set of assertions against the gAssertions table. + */ +function checkAssertions(assertions, doneCallback) +{ + var checkAssertion = function() { + for (var i in assertions) { + var data = assertions[i]; + delete assertions[i]; + gAssertions[i](data, checkAssertion); + return; + } + + doneCallback(); + } + + checkAssertion(); +} + +function updateError(arg) +{ + do_throw(arg); +} + +// Runs a set of updates, and then checks a set of assertions. +function doUpdateTest(updates, assertions, successCallback, errorCallback) { + var errorUpdate = function() { + checkAssertions(assertions, errorCallback); + } + + var runUpdate = function() { + if (updates.length > 0) { + var update = updates.shift(); + doStreamUpdate(update, runUpdate, errorUpdate, null); + } else { + checkAssertions(assertions, successCallback); + } + } + + runUpdate(); +} + +var gTests; +var gNextTest = 0; + +function runNextTest() +{ + if (gNextTest >= gTests.length) { + do_test_finished(); + return; + } + + dbservice.resetDatabase(); + dbservice.setHashCompleter('test-phish-simple', null); + + let test = gTests[gNextTest++]; + dump("running " + test.name + "\n"); + test(); +} + +function runTests(tests) +{ + gTests = tests; + runNextTest(); +} + +var timerArray = []; + +function Timer(delay, cb) { + this.cb = cb; + var timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + timer.initWithCallback(this, delay, timer.TYPE_ONE_SHOT); + timerArray.push(timer); +} + +Timer.prototype = { +QueryInterface: function(iid) { + if (!iid.equals(Ci.nsISupports) && !iid.equals(Ci.nsITimerCallback)) { + throw Cr.NS_ERROR_NO_INTERFACE; + } + return this; + }, +notify: function(timer) { + this.cb(); + } +} + +// LFSRgenerator is a 32-bit linear feedback shift register random number +// generator. It is highly predictable and is not intended to be used for +// cryptography but rather to allow easier debugging than a test that uses +// Math.random(). +function LFSRgenerator(seed) { + // Force |seed| to be a number. + seed = +seed; + // LFSR generators do not work with a value of 0. + if (seed == 0) + seed = 1; + + this._value = seed; +} +LFSRgenerator.prototype = { + // nextNum returns a random unsigned integer of in the range [0,2^|bits|]. + nextNum: function(bits) { + if (!bits) + bits = 32; + + let val = this._value; + // Taps are 32, 22, 2 and 1. + let bit = ((val >>> 0) ^ (val >>> 10) ^ (val >>> 30) ^ (val >>> 31)) & 1; + val = (val >>> 1) | (bit << 31); + this._value = val; + + return (val >>> (32 - bits)); + }, +}; + +cleanUp(); diff --git a/toolkit/components/url-classifier/tests/unit/tail_urlclassifier.js b/toolkit/components/url-classifier/tests/unit/tail_urlclassifier.js new file mode 100644 index 000000000..37f39d1a8 --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/tail_urlclassifier.js @@ -0,0 +1 @@ +cleanUp(); diff --git a/toolkit/components/url-classifier/tests/unit/test_addsub.js b/toolkit/components/url-classifier/tests/unit/test_addsub.js new file mode 100644 index 000000000..1ed65c7ba --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/test_addsub.js @@ -0,0 +1,488 @@ + +function doTest(updates, assertions) +{ + doUpdateTest(updates, assertions, runNextTest, updateError); +} + +// Test an add of two urls to a fresh database +function testSimpleAdds() { + var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ]; + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }]); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + "urlsExist" : addUrls + }; + + doTest([update], assertions); +} + +// Same as testSimpleAdds, but make the same-domain URLs come from different +// chunks. +function testMultipleAdds() { + var add1Urls = [ "foo.com/a", "bar.com/c" ]; + var add2Urls = [ "foo.com/b" ]; + + var update = buildPhishingUpdate( + [{ "chunkNum" : 1, + "urls" : add1Urls }, + { "chunkNum" : 2, + "urls" : add2Urls }]); + var assertions = { + "tableData" : "test-phish-simple;a:1-2", + "urlsExist" : add1Urls.concat(add2Urls) + }; + + doTest([update], assertions); +} + +// Test that a sub will remove an existing add +function testSimpleSub() +{ + var addUrls = ["foo.com/a", "bar.com/b"]; + var subUrls = ["1:foo.com/a"]; + + var addUpdate = buildPhishingUpdate( + [{ "chunkNum" : 1, // adds and subtracts don't share a chunk numbering space + "urls": addUrls }]); + + var subUpdate = buildPhishingUpdate( + [{ "chunkNum" : 50, + "chunkType" : "s", + "urls": subUrls }]); + + + var assertions = { + "tableData" : "test-phish-simple;a:1:s:50", + "urlsExist" : [ "bar.com/b" ], + "urlsDontExist": ["foo.com/a" ], + "subsDontExist" : [ "foo.com/a" ] + } + + doTest([addUpdate, subUpdate], assertions); + +} + +// Same as testSimpleSub(), but the sub comes in before the add. +function testSubEmptiesAdd() +{ + var subUrls = ["1:foo.com/a"]; + var addUrls = ["foo.com/a", "bar.com/b"]; + + var subUpdate = buildPhishingUpdate( + [{ "chunkNum" : 50, + "chunkType" : "s", + "urls": subUrls }]); + + var addUpdate = buildPhishingUpdate( + [{ "chunkNum" : 1, + "urls": addUrls }]); + + var assertions = { + "tableData" : "test-phish-simple;a:1:s:50", + "urlsExist" : [ "bar.com/b" ], + "urlsDontExist": ["foo.com/a" ], + "subsDontExist" : [ "foo.com/a" ] // this sub was found, it shouldn't exist anymore + } + + doTest([subUpdate, addUpdate], assertions); +} + +// Very similar to testSubEmptiesAdd, except that the domain entry will +// still have an item left over that needs to be synced. +function testSubPartiallyEmptiesAdd() +{ + var subUrls = ["1:foo.com/a"]; + var addUrls = ["foo.com/a", "foo.com/b", "bar.com/b"]; + + var subUpdate = buildPhishingUpdate( + [{ "chunkNum" : 1, + "chunkType" : "s", + "urls": subUrls }]); + + var addUpdate = buildPhishingUpdate( + [{ "chunkNum" : 1, // adds and subtracts don't share a chunk numbering space + "urls": addUrls }]); + + var assertions = { + "tableData" : "test-phish-simple;a:1:s:1", + "urlsExist" : [ "foo.com/b", "bar.com/b" ], + "urlsDontExist" : ["foo.com/a" ], + "subsDontExist" : [ "foo.com/a" ] // this sub was found, it shouldn't exist anymore + } + + doTest([subUpdate, addUpdate], assertions); +} + +// We SHOULD be testing that pending subs are removed using +// subsDontExist assertions. Since we don't have a good interface for getting +// at sub entries, we'll verify it by side-effect. Subbing a url once +// then adding it twice should leave the url intact. +function testPendingSubRemoved() +{ + var subUrls = ["1:foo.com/a", "2:foo.com/b"]; + var addUrls = ["foo.com/a", "foo.com/b"]; + + var subUpdate = buildPhishingUpdate( + [{ "chunkNum" : 1, + "chunkType" : "s", + "urls": subUrls }]); + + var addUpdate1 = buildPhishingUpdate( + [{ "chunkNum" : 1, // adds and subtracts don't share a chunk numbering space + "urls": addUrls }]); + + var addUpdate2 = buildPhishingUpdate( + [{ "chunkNum" : 2, + "urls": addUrls }]); + + var assertions = { + "tableData" : "test-phish-simple;a:1-2:s:1", + "urlsExist" : [ "foo.com/a", "foo.com/b" ], + "subsDontExist" : [ "foo.com/a", "foo.com/b" ] // this sub was found, it shouldn't exist anymore + } + + doTest([subUpdate, addUpdate1, addUpdate2], assertions); +} + +// Make sure that a saved sub is removed when the sub chunk is expired. +function testPendingSubExpire() +{ + var subUrls = ["1:foo.com/a", "1:foo.com/b"]; + var addUrls = ["foo.com/a", "foo.com/b"]; + + var subUpdate = buildPhishingUpdate( + [{ "chunkNum" : 1, + "chunkType" : "s", + "urls": subUrls }]); + + var expireUpdate = buildPhishingUpdate( + [{ "chunkNum" : 1, + "chunkType" : "sd" }]); + + var addUpdate = buildPhishingUpdate( + [{ "chunkNum" : 1, // adds and subtracts don't share a chunk numbering space + "urls": addUrls }]); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + "urlsExist" : [ "foo.com/a", "foo.com/b" ], + "subsDontExist" : [ "foo.com/a", "foo.com/b" ] // this sub was expired + } + + doTest([subUpdate, expireUpdate, addUpdate], assertions); +} + +// Make sure that the sub url removes from only the chunk that it specifies +function testDuplicateAdds() +{ + var urls = ["foo.com/a"]; + + var addUpdate1 = buildPhishingUpdate( + [{ "chunkNum" : 1, + "urls": urls }]); + var addUpdate2 = buildPhishingUpdate( + [{ "chunkNum" : 2, + "urls": urls }]); + var subUpdate = buildPhishingUpdate( + [{ "chunkNum" : 3, + "chunkType" : "s", + "urls": ["2:foo.com/a"]}]); + + var assertions = { + "tableData" : "test-phish-simple;a:1-2:s:3", + "urlsExist" : [ "foo.com/a"], + "subsDontExist" : [ "foo.com/a"] + } + + doTest([addUpdate1, addUpdate2, subUpdate], assertions); +} + +// Tests a sub which matches some existing adds but leaves others. +function testSubPartiallyMatches() +{ + var subUrls = ["foo.com/a"]; + var addUrls = ["1:foo.com/a", "2:foo.com/b"]; + + var addUpdate = buildPhishingUpdate( + [{ "chunkNum" : 1, + "urls" : addUrls }]); + + var subUpdate = buildPhishingUpdate( + [{ "chunkNum" : 1, + "chunkType" : "s", + "urls" : addUrls }]); + + var assertions = { + "tableData" : "test-phish-simple;a:1:s:1", + "urlsDontExist" : ["foo.com/a"], + "subsDontExist" : ["foo.com/a"], + "subsExist" : ["foo.com/b"] + }; + + doTest([addUpdate, subUpdate], assertions); +} + +// XXX: because subsExist isn't actually implemented, this is the same +// test as above but with a second add chunk that should fail to be added +// because of a pending sub chunk. +function testSubPartiallyMatches2() +{ + var addUrls = ["foo.com/a"]; + var subUrls = ["1:foo.com/a", "2:foo.com/b"]; + var addUrls2 = ["foo.com/b"]; + + var addUpdate = buildPhishingUpdate( + [{ "chunkNum" : 1, + "urls" : addUrls }]); + + var subUpdate = buildPhishingUpdate( + [{ "chunkNum" : 1, + "chunkType" : "s", + "urls" : subUrls }]); + + var addUpdate2 = buildPhishingUpdate( + [{ "chunkNum" : 2, + "urls" : addUrls2 }]); + + var assertions = { + "tableData" : "test-phish-simple;a:1-2:s:1", + "urlsDontExist" : ["foo.com/a", "foo.com/b"], + "subsDontExist" : ["foo.com/a", "foo.com/b"] + }; + + doTest([addUpdate, subUpdate, addUpdate2], assertions); +} + +// Verify that two subs for the same domain but from different chunks +// match (tests that existing sub entries are properly updated) +function testSubsDifferentChunks() { + var subUrls1 = [ "3:foo.com/a" ]; + var subUrls2 = [ "3:foo.com/b" ]; + + var addUrls = [ "foo.com/a", "foo.com/b", "foo.com/c" ]; + + var subUpdate1 = buildPhishingUpdate( + [{ "chunkNum" : 1, + "chunkType" : "s", + "urls": subUrls1 }]); + var subUpdate2 = buildPhishingUpdate( + [{ "chunkNum" : 2, + "chunkType" : "s", + "urls" : subUrls2 }]); + var addUpdate = buildPhishingUpdate( + [{ "chunkNum" : 3, + "urls" : addUrls }]); + + var assertions = { + "tableData" : "test-phish-simple;a:3:s:1-2", + "urlsExist" : [ "foo.com/c" ], + "urlsDontExist" : [ "foo.com/a", "foo.com/b" ], + "subsDontExist" : [ "foo.com/a", "foo.com/b" ] + }; + + doTest([subUpdate1, subUpdate2, addUpdate], assertions); +} + +// for bug 534079 +function testSubsDifferentChunksSameHostId() { + var subUrls1 = [ "1:foo.com/a" ]; + var subUrls2 = [ "1:foo.com/b", "2:foo.com/c" ]; + + var addUrls = [ "foo.com/a", "foo.com/b" ]; + var addUrls2 = [ "foo.com/c" ]; + + var subUpdate1 = buildPhishingUpdate( + [{ "chunkNum" : 1, + "chunkType" : "s", + "urls": subUrls1 }]); + var subUpdate2 = buildPhishingUpdate( + [{ "chunkNum" : 2, + "chunkType" : "s", + "urls" : subUrls2 }]); + + var addUpdate = buildPhishingUpdate( + [{ "chunkNum" : 1, + "urls" : addUrls }]); + var addUpdate2 = buildPhishingUpdate( + [{ "chunkNum" : 2, + "urls" : addUrls2 }]); + + var assertions = { + "tableData" : "test-phish-simple;a:1-2:s:1-2", + "urlsDontExist" : [ "foo.com/c", "foo.com/b", "foo.com/a", ], + }; + + doTest([addUpdate, addUpdate2, subUpdate1, subUpdate2], assertions); +} + +// Test lists of expired chunks +function testExpireLists() { + var addUpdate = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : [ "foo.com/a" ] + }, + { "chunkNum" : 3, + "urls" : [ "bar.com/a" ] + }, + { "chunkNum" : 4, + "urls" : [ "baz.com/a" ] + }, + { "chunkNum" : 5, + "urls" : [ "blah.com/a" ] + }, + ]); + var subUpdate = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "chunkType" : "s", + "urls" : [ "50:foo.com/1" ] + }, + { "chunkNum" : 2, + "chunkType" : "s", + "urls" : [ "50:bar.com/1" ] + }, + { "chunkNum" : 3, + "chunkType" : "s", + "urls" : [ "50:baz.com/1" ] + }, + { "chunkNum" : 5, + "chunkType" : "s", + "urls" : [ "50:blah.com/1" ] + }, + ]); + + var expireUpdate = buildPhishingUpdate( + [ { "chunkType" : "ad:1,3-5" }, + { "chunkType" : "sd:1-3,5" }]); + + var assertions = { + // "tableData" : "test-phish-simple;" + "tableData": "" + }; + + doTest([addUpdate, subUpdate, expireUpdate], assertions); +} + +// Test a duplicate add chunk. +function testDuplicateAddChunks() { + var addUrls1 = [ "foo.com/a" ]; + var addUrls2 = [ "bar.com/b" ]; + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls1 + }, + { "chunkNum" : 1, + "urls" : addUrls2 + }]); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + "urlsExist" : addUrls1, + "urlsDontExist" : addUrls2 + }; + + doTest([update], assertions); +} + +// This test is a bit tricky. We want to test that an add removes all +// subs with the same add chunk id, even if there is no match. To do +// that we need to add the same add chunk twice, with an expiration +// in the middle. This would be easier if subsDontExist actually +// worked... +function testExpireWholeSub() +{ + var subUrls = ["1:foo.com/a"]; + + var update = buildPhishingUpdate( + [{ "chunkNum" : 5, + "chunkType" : "s", + "urls" : subUrls + }, + // empty add chunk should still cause foo.com/a to go away. + { "chunkNum" : 1, + "urls" : [] + }, + // and now adding chunk 1 again with foo.com/a should succeed, + // because the sub should have been expired with the empty + // add chunk. + + // we need to expire this chunk to let us add chunk 1 again. + { + "chunkType" : "ad:1" + }, + { "chunkNum" : 1, + "urls" : [ "foo.com/a" ] + }]); + + var assertions = { + "tableData" : "test-phish-simple;a:1:s:5", + "urlsExist" : ["foo.com/a"] + }; + + doTest([update], assertions); +} + + +// This test is roughly the opposite of testExpireWholeSub(). We add +// the empty add first, and make sure that it prevents a sub for that +// add from being applied. +function testPreventWholeSub() +{ + var subUrls = ["1:foo.com/a"]; + + var update = buildPhishingUpdate( + [ // empty add chunk should cause foo.com/a to not be saved + { "chunkNum" : 1, + "urls" : [] + }, + { "chunkNum" : 5, + "chunkType" : "s", + "urls" : subUrls + }, + // and now adding chunk 1 again with foo.com/a should succeed, + // because the sub should have been expired with the empty + // add chunk. + + // we need to expire this chunk to let us add chunk 1 again. + { + "chunkType" : "ad:1" + }, + { "chunkNum" : 1, + "urls" : [ "foo.com/a" ] + }]); + + var assertions = { + "tableData" : "test-phish-simple;a:1:s:5", + "urlsExist" : ["foo.com/a"] + }; + + doTest([update], assertions); +} + +function run_test() +{ + runTests([ + testSimpleAdds, + testMultipleAdds, + testSimpleSub, + testSubEmptiesAdd, + testSubPartiallyEmptiesAdd, + testPendingSubRemoved, + testPendingSubExpire, + testDuplicateAdds, + testSubPartiallyMatches, + testSubPartiallyMatches2, + testSubsDifferentChunks, + testSubsDifferentChunksSameHostId, + testExpireLists + ]); +} + +do_test_pending(); diff --git a/toolkit/components/url-classifier/tests/unit/test_backoff.js b/toolkit/components/url-classifier/tests/unit/test_backoff.js new file mode 100644 index 000000000..365568c47 --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/test_backoff.js @@ -0,0 +1,89 @@ +// Some unittests (e.g., paste into JS shell) +var jslib = Cc["@mozilla.org/url-classifier/jslib;1"]. + getService().wrappedJSObject; +var _Datenow = jslib.Date.now; +function setNow(time) { + jslib.Date.now = function() { + return time; + } +} + +function run_test() { + // 3 errors, 1ms retry period, max 3 requests per ten milliseconds, + // 5ms backoff interval, 19ms max delay + var rb = new jslib.RequestBackoff(3, 1, 3, 10, 5, 19); + setNow(1); + rb.noteServerResponse(200); + do_check_true(rb.canMakeRequest()); + setNow(2); + do_check_true(rb.canMakeRequest()); + + // First error should trigger a 1ms delay + rb.noteServerResponse(500); + do_check_false(rb.canMakeRequest()); + do_check_eq(rb.nextRequestTime_, 3); + setNow(3); + do_check_true(rb.canMakeRequest()); + + // Second error should also trigger a 1ms delay + rb.noteServerResponse(500); + do_check_false(rb.canMakeRequest()); + do_check_eq(rb.nextRequestTime_, 4); + setNow(4); + do_check_true(rb.canMakeRequest()); + + // Third error should trigger a 5ms backoff + rb.noteServerResponse(500); + do_check_false(rb.canMakeRequest()); + do_check_eq(rb.nextRequestTime_, 9); + setNow(9); + do_check_true(rb.canMakeRequest()); + + // Trigger backoff again + rb.noteServerResponse(503); + do_check_false(rb.canMakeRequest()); + do_check_eq(rb.nextRequestTime_, 19); + setNow(19); + do_check_true(rb.canMakeRequest()); + + // Trigger backoff a third time and hit max timeout + rb.noteServerResponse(302); + do_check_false(rb.canMakeRequest()); + do_check_eq(rb.nextRequestTime_, 38); + setNow(38); + do_check_true(rb.canMakeRequest()); + + // One more backoff, should still be at the max timeout + rb.noteServerResponse(400); + do_check_false(rb.canMakeRequest()); + do_check_eq(rb.nextRequestTime_, 57); + setNow(57); + do_check_true(rb.canMakeRequest()); + + // Request goes through + rb.noteServerResponse(200); + do_check_true(rb.canMakeRequest()); + do_check_eq(rb.nextRequestTime_, 0); + setNow(58); + rb.noteServerResponse(500); + + // Another error, should trigger a 1ms backoff + do_check_false(rb.canMakeRequest()); + do_check_eq(rb.nextRequestTime_, 59); + + setNow(59); + do_check_true(rb.canMakeRequest()); + + setNow(200); + rb.noteRequest(); + setNow(201); + rb.noteRequest(); + setNow(202); + do_check_true(rb.canMakeRequest()); + rb.noteRequest(); + do_check_false(rb.canMakeRequest()); + setNow(211); + do_check_true(rb.canMakeRequest()); + + jslib.Date.now = _Datenow; +} diff --git a/toolkit/components/url-classifier/tests/unit/test_bug1274685_unowned_list.js b/toolkit/components/url-classifier/tests/unit/test_bug1274685_unowned_list.js new file mode 100644 index 000000000..037bc7b88 --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/test_bug1274685_unowned_list.js @@ -0,0 +1,32 @@ +Cu.import("resource://gre/modules/SafeBrowsing.jsm"); +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://testing-common/AppInfo.jsm"); + +// 'Cc["@mozilla.org/xre/app-info;1"]' for xpcshell has no nsIXULAppInfo +// so that we have to update it to make nsURLFormatter.js happy. +// (SafeBrowsing.init() will indirectly use nsURLFormatter.js) +updateAppInfo(); + +function run_test() { + SafeBrowsing.init(); + + let origList = Services.prefs.getCharPref("browser.safebrowsing.provider.google.lists"); + + // Remove 'goog-malware-shavar' from the original. + let trimmedList = origList.replace('goog-malware-shavar,', ''); + Services.prefs.setCharPref("browser.safebrowsing.provider.google.lists", trimmedList); + + try { + // Bug 1274685 - Unowned Safe Browsing tables break list updates + // + // If SafeBrowsing.registerTableWithURLs() doesn't check if + // a provider is found before registering table, an exception + // will be thrown while accessing a null object. + // + SafeBrowsing.registerTables(); + } catch (e) { + ok(false, 'Exception thrown due to ' + e.toString()); + } + + Services.prefs.setCharPref("browser.safebrowsing.provider.google.lists", origList); +} diff --git a/toolkit/components/url-classifier/tests/unit/test_dbservice.js b/toolkit/components/url-classifier/tests/unit/test_dbservice.js new file mode 100644 index 000000000..4b01e7016 --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/test_dbservice.js @@ -0,0 +1,314 @@ +var checkUrls = []; +var checkExpect; + +var chunk1Urls = [ + "test.com/aba", + "test.com/foo/bar", + "foo.bar.com/a/b/c" +]; +var chunk1 = chunk1Urls.join("\n"); + +var chunk2Urls = [ + "blah.com/a", + "baz.com/", + "255.255.0.1/", + "www.foo.com/test2?param=1" +]; +var chunk2 = chunk2Urls.join("\n"); + +var chunk3Urls = [ + "test.com/a", + "foo.bar.com/a", + "blah.com/a", + ]; +var chunk3 = chunk3Urls.join("\n"); + +var chunk3SubUrls = [ + "1:test.com/a", + "1:foo.bar.com/a", + "2:blah.com/a" ]; +var chunk3Sub = chunk3SubUrls.join("\n"); + +var chunk4Urls = [ + "a.com/b", + "b.com/c", + ]; +var chunk4 = chunk4Urls.join("\n"); + +var chunk5Urls = [ + "d.com/e", + "f.com/g", + ]; +var chunk5 = chunk5Urls.join("\n"); + +var chunk6Urls = [ + "h.com/i", + "j.com/k", + ]; +var chunk6 = chunk6Urls.join("\n"); + +var chunk7Urls = [ + "l.com/m", + "n.com/o", + ]; +var chunk7 = chunk7Urls.join("\n"); + +// we are going to add chunks 1, 2, 4, 5, and 6 to phish-simple, +// chunk 2 to malware-simple, and chunk 3 to unwanted-simple, +// and chunk 7 to block-simple. +// Then we'll remove the urls in chunk3 from phish-simple, then +// expire chunk 1 and chunks 4-7 from phish-simple. +var phishExpected = {}; +var phishUnexpected = {}; +var malwareExpected = {}; +var unwantedExpected = {}; +var blockedExpected = {}; +for (var i = 0; i < chunk2Urls.length; i++) { + phishExpected[chunk2Urls[i]] = true; + malwareExpected[chunk2Urls[i]] = true; +} +for (var i = 0; i < chunk3Urls.length; i++) { + unwantedExpected[chunk3Urls[i]] = true; + delete phishExpected[chunk3Urls[i]]; + phishUnexpected[chunk3Urls[i]] = true; +} +for (var i = 0; i < chunk1Urls.length; i++) { + // chunk1 urls are expired + phishUnexpected[chunk1Urls[i]] = true; +} +for (var i = 0; i < chunk4Urls.length; i++) { + // chunk4 urls are expired + phishUnexpected[chunk4Urls[i]] = true; +} +for (var i = 0; i < chunk5Urls.length; i++) { + // chunk5 urls are expired + phishUnexpected[chunk5Urls[i]] = true; +} +for (var i = 0; i < chunk6Urls.length; i++) { + // chunk6 urls are expired + phishUnexpected[chunk6Urls[i]] = true; +} +for (var i = 0; i < chunk7Urls.length; i++) { + blockedExpected[chunk7Urls[i]] = true; + // chunk7 urls are expired + phishUnexpected[chunk7Urls[i]] = true; +} + +// Check that the entries hit based on sub-parts +phishExpected["baz.com/foo/bar"] = true; +phishExpected["foo.bar.baz.com/foo"] = true; +phishExpected["bar.baz.com/"] = true; + +var numExpecting; + +function testFailure(arg) { + do_throw(arg); +} + +function checkNoHost() +{ + // Looking up a no-host uri such as a data: uri should throw an exception. + var exception; + try { + var principal = secMan.createCodebasePrincipal(iosvc.newURI("data:text/html,<b>test</b>", null, null), {}); + dbservice.lookup(principal, allTables); + + exception = false; + } catch(e) { + exception = true; + } + do_check_true(exception); + + do_test_finished(); +} + +function tablesCallbackWithoutSub(tables) +{ + var parts = tables.split("\n"); + parts.sort(); + + // there's a leading \n here because splitting left an empty string + // after the trailing newline, which will sort first + do_check_eq(parts.join("\n"), + "\ntest-block-simple;a:1\ntest-malware-simple;a:1\ntest-phish-simple;a:2\ntest-unwanted-simple;a:1"); + + checkNoHost(); +} + + +function expireSubSuccess(result) { + dbservice.getTables(tablesCallbackWithoutSub); +} + +function tablesCallbackWithSub(tables) +{ + var parts = tables.split("\n"); + parts.sort(); + + // there's a leading \n here because splitting left an empty string + // after the trailing newline, which will sort first + do_check_eq(parts.join("\n"), + "\ntest-block-simple;a:1\ntest-malware-simple;a:1\ntest-phish-simple;a:2:s:3\ntest-unwanted-simple;a:1"); + + // verify that expiring a sub chunk removes its name from the list + var data = + "n:1000\n" + + "i:test-phish-simple\n" + + "sd:3\n"; + + doSimpleUpdate(data, expireSubSuccess, testFailure); +} + +function checkChunksWithSub() +{ + dbservice.getTables(tablesCallbackWithSub); +} + +function checkDone() { + if (--numExpecting == 0) + checkChunksWithSub(); +} + +function phishExists(result) { + dumpn("phishExists: " + result); + try { + do_check_true(result.indexOf("test-phish-simple") != -1); + } finally { + checkDone(); + } +} + +function phishDoesntExist(result) { + dumpn("phishDoesntExist: " + result); + try { + do_check_true(result.indexOf("test-phish-simple") == -1); + } finally { + checkDone(); + } +} + +function malwareExists(result) { + dumpn("malwareExists: " + result); + + try { + do_check_true(result.indexOf("test-malware-simple") != -1); + } finally { + checkDone(); + } +} + +function unwantedExists(result) { + dumpn("unwantedExists: " + result); + + try { + do_check_true(result.indexOf("test-unwanted-simple") != -1); + } finally { + checkDone(); + } +} + +function blockedExists(result) { + dumpn("blockedExists: " + result); + + try { + do_check_true(result.indexOf("test-block-simple") != -1); + } finally { + checkDone(); + } +} + +function checkState() +{ + numExpecting = 0; + + + for (var key in phishExpected) { + var principal = secMan.createCodebasePrincipal(iosvc.newURI("http://" + key, null, null), {}); + dbservice.lookup(principal, allTables, phishExists, true); + numExpecting++; + } + + for (var key in phishUnexpected) { + var principal = secMan.createCodebasePrincipal(iosvc.newURI("http://" + key, null, null), {}); + dbservice.lookup(principal, allTables, phishDoesntExist, true); + numExpecting++; + } + + for (var key in malwareExpected) { + var principal = secMan.createCodebasePrincipal(iosvc.newURI("http://" + key, null, null), {}); + dbservice.lookup(principal, allTables, malwareExists, true); + numExpecting++; + } + + for (var key in unwantedExpected) { + var principal = secMan.createCodebasePrincipal(iosvc.newURI("http://" + key, null, null), {}); + dbservice.lookup(principal, allTables, unwantedExists, true); + numExpecting++; + } + + for (var key in blockedExpected) { + var principal = secMan.createCodebasePrincipal(iosvc.newURI("http://" + key, null, null), {}); + dbservice.lookup(principal, allTables, blockedExists, true); + numExpecting++; + } +} + +function testSubSuccess(result) +{ + do_check_eq(result, "1000"); + checkState(); +} + +function do_subs() { + var data = + "n:1000\n" + + "i:test-phish-simple\n" + + "s:3:32:" + chunk3Sub.length + "\n" + + chunk3Sub + "\n" + + "ad:1\n" + + "ad:4-6\n"; + + doSimpleUpdate(data, testSubSuccess, testFailure); +} + +function testAddSuccess(arg) { + do_check_eq(arg, "1000"); + + do_subs(); +} + +function do_adds() { + // This test relies on the fact that only -regexp tables are ungzipped, + // and only -hash tables are assumed to be pre-md5'd. So we use + // a 'simple' table type to get simple hostname-per-line semantics. + + var data = + "n:1000\n" + + "i:test-phish-simple\n" + + "a:1:32:" + chunk1.length + "\n" + + chunk1 + "\n" + + "a:2:32:" + chunk2.length + "\n" + + chunk2 + "\n" + + "a:4:32:" + chunk4.length + "\n" + + chunk4 + "\n" + + "a:5:32:" + chunk5.length + "\n" + + chunk5 + "\n" + + "a:6:32:" + chunk6.length + "\n" + + chunk6 + "\n" + + "i:test-malware-simple\n" + + "a:1:32:" + chunk2.length + "\n" + + chunk2 + "\n" + + "i:test-unwanted-simple\n" + + "a:1:32:" + chunk3.length + "\n" + + chunk3 + "\n" + + "i:test-block-simple\n" + + "a:1:32:" + chunk7.length + "\n" + + chunk7 + "\n"; + + doSimpleUpdate(data, testAddSuccess, testFailure); +} + +function run_test() { + do_adds(); + do_test_pending(); +} diff --git a/toolkit/components/url-classifier/tests/unit/test_digest256.js b/toolkit/components/url-classifier/tests/unit/test_digest256.js new file mode 100644 index 000000000..6ae652915 --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/test_digest256.js @@ -0,0 +1,147 @@ +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); + +XPCOMUtils.defineLazyModuleGetter(this, "NetUtil", + "resource://gre/modules/NetUtil.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "Promise", + "resource://gre/modules/Promise.jsm"); +// Global test server for serving safebrowsing updates. +var gHttpServ = null; +// Global nsIUrlClassifierDBService +var gDbService = Cc["@mozilla.org/url-classifier/dbservice;1"] + .getService(Ci.nsIUrlClassifierDBService); +// Security manager for creating nsIPrincipals from URIs +var gSecMan = Cc["@mozilla.org/scriptsecuritymanager;1"] + .getService(Ci.nsIScriptSecurityManager); + +// A map of tables to arrays of update redirect urls. +var gTables = {}; + +// Construct an update from a file. +function readFileToString(aFilename) { + let f = do_get_file(aFilename); + let stream = Cc["@mozilla.org/network/file-input-stream;1"] + .createInstance(Ci.nsIFileInputStream); + stream.init(f, -1, 0, 0); + let buf = NetUtil.readInputStreamToString(stream, stream.available()); + return buf; +} + +// Registers a table for which to serve update chunks. Returns a promise that +// resolves when that chunk has been downloaded. +function registerTableUpdate(aTable, aFilename) { + let deferred = Promise.defer(); + // If we haven't been given an update for this table yet, add it to the map + if (!(aTable in gTables)) { + gTables[aTable] = []; + } + + // The number of chunks associated with this table. + let numChunks = gTables[aTable].length + 1; + let redirectPath = "/" + aTable + "-" + numChunks; + let redirectUrl = "localhost:4444" + redirectPath; + + // Store redirect url for that table so we can return it later when we + // process an update request. + gTables[aTable].push(redirectUrl); + + gHttpServ.registerPathHandler(redirectPath, function(request, response) { + do_print("Mock safebrowsing server handling request for " + redirectPath); + let contents = readFileToString(aFilename); + response.setHeader("Content-Type", + "application/vnd.google.safebrowsing-update", false); + response.setStatusLine(request.httpVersion, 200, "OK"); + response.bodyOutputStream.write(contents, contents.length); + deferred.resolve(contents); + }); + return deferred.promise; +} + +// Construct a response with redirect urls. +function processUpdateRequest() { + let response = "n:1000\n"; + for (let table in gTables) { + response += "i:" + table + "\n"; + for (let i = 0; i < gTables[table].length; ++i) { + response += "u:" + gTables[table][i] + "\n"; + } + } + do_print("Returning update response: " + response); + return response; +} + +// Set up our test server to handle update requests. +function run_test() { + gHttpServ = new HttpServer(); + gHttpServ.registerDirectory("/", do_get_cwd()); + + gHttpServ.registerPathHandler("/downloads", function(request, response) { + let buf = NetUtil.readInputStreamToString(request.bodyInputStream, + request.bodyInputStream.available()); + let blob = processUpdateRequest(); + response.setHeader("Content-Type", + "application/vnd.google.safebrowsing-update", false); + response.setStatusLine(request.httpVersion, 200, "OK"); + response.bodyOutputStream.write(blob, blob.length); + }); + + gHttpServ.start(4444); + run_next_test(); +} + +function createURI(s) { + let service = Cc["@mozilla.org/network/io-service;1"] + .getService(Ci.nsIIOService); + return service.newURI(s, null, null); +} + +// Just throw if we ever get an update or download error. +function handleError(aEvent) { + do_throw("We didn't download or update correctly: " + aEvent); +} + +add_test(function test_update() { + let streamUpdater = Cc["@mozilla.org/url-classifier/streamupdater;1"] + .getService(Ci.nsIUrlClassifierStreamUpdater); + + // Load up some update chunks for the safebrowsing server to serve. + registerTableUpdate("goog-downloadwhite-digest256", "data/digest1.chunk"); + registerTableUpdate("goog-downloadwhite-digest256", "data/digest2.chunk"); + + // Download some updates, and don't continue until the downloads are done. + function updateSuccess(aEvent) { + // Timeout of n:1000 is constructed in processUpdateRequest above and + // passed back in the callback in nsIUrlClassifierStreamUpdater on success. + do_check_eq("1000", aEvent); + do_print("All data processed"); + run_next_test(); + } + streamUpdater.downloadUpdates( + "goog-downloadwhite-digest256", + "goog-downloadwhite-digest256;\n", + true, + "http://localhost:4444/downloads", + updateSuccess, handleError, handleError); +}); + +add_test(function test_url_not_whitelisted() { + let uri = createURI("http://example.com"); + let principal = gSecMan.createCodebasePrincipal(uri, {}); + gDbService.lookup(principal, "goog-downloadwhite-digest256", + function handleEvent(aEvent) { + // This URI is not on any lists. + do_check_eq("", aEvent); + run_next_test(); + }); +}); + +add_test(function test_url_whitelisted() { + // Hash of "whitelisted.com/" (canonicalized URL) is: + // 93CA5F48E15E9861CD37C2D95DB43D23CC6E6DE5C3F8FA6E8BE66F97CC518907 + let uri = createURI("http://whitelisted.com"); + let principal = gSecMan.createCodebasePrincipal(uri, {}); + gDbService.lookup(principal, "goog-downloadwhite-digest256", + function handleEvent(aEvent) { + do_check_eq("goog-downloadwhite-digest256", aEvent); + run_next_test(); + }); +}); diff --git a/toolkit/components/url-classifier/tests/unit/test_hashcompleter.js b/toolkit/components/url-classifier/tests/unit/test_hashcompleter.js new file mode 100644 index 000000000..40fafd923 --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/test_hashcompleter.js @@ -0,0 +1,403 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// This test ensures that the nsIUrlClassifierHashCompleter works as expected +// and simulates an HTTP server to provide completions. +// +// In order to test completions, each group of completions sent as one request +// to the HTTP server is called a completion set. There is currently not +// support for multiple requests being sent to the server at once, in this test. +// This tests makes a request for each element of |completionSets|, waits for +// a response and then moves to the next element. +// Each element of |completionSets| is an array of completions, and each +// completion is an object with the properties: +// hash: complete hash for the completion. Automatically right-padded +// to be COMPLETE_LENGTH. +// expectCompletion: boolean indicating whether the server should respond +// with a full hash. +// forceServerError: boolean indicating whether the server should respond +// with a 503. +// table: name of the table that the hash corresponds to. Only needs to be set +// if a completion is expected. +// chunkId: positive integer corresponding to the chunk that the hash belongs +// to. Only needs to be set if a completion is expected. +// multipleCompletions: boolean indicating whether the server should respond +// with more than one full hash. If this is set to true +// then |expectCompletion| must also be set to true and +// |hash| must have the same prefix as all |completions|. +// completions: an array of completions (objects with a hash, table and +// chunkId property as described above). This property is only +// used when |multipleCompletions| is set to true. + +// Basic prefixes with 2/3 completions. +var basicCompletionSet = [ + { + hash: "abcdefgh", + expectCompletion: true, + table: "test", + chunkId: 1234, + }, + { + hash: "1234", + expectCompletion: false, + }, + { + hash: "\u0000\u0000\u000012312", + expectCompletion: true, + table: "test", + chunkId: 1234, + } +]; + +// 3 prefixes with 0 completions to test HashCompleter handling a 204 status. +var falseCompletionSet = [ + { + hash: "1234", + expectCompletion: false, + }, + { + hash: "", + expectCompletion: false, + }, + { + hash: "abc", + expectCompletion: false, + } +]; + +// The current implementation (as of Mar 2011) sometimes sends duplicate +// entries to HashCompleter and even expects responses for duplicated entries. +var dupedCompletionSet = [ + { + hash: "1234", + expectCompletion: true, + table: "test", + chunkId: 1, + }, + { + hash: "5678", + expectCompletion: false, + table: "test2", + chunkId: 2, + }, + { + hash: "1234", + expectCompletion: true, + table: "test", + chunkId: 1, + }, + { + hash: "5678", + expectCompletion: false, + table: "test2", + chunkId: 2 + } +]; + +// It is possible for a hash completion request to return with multiple +// completions, the HashCompleter should return all of these. +var multipleResponsesCompletionSet = [ + { + hash: "1234", + expectCompletion: true, + multipleCompletions: true, + completions: [ + { + hash: "123456", + table: "test1", + chunkId: 3, + }, + { + hash: "123478", + table: "test2", + chunkId: 4, + } + ], + } +]; + +function buildCompletionRequest(aCompletionSet) { + let prefixes = []; + let prefixSet = new Set(); + aCompletionSet.forEach(s => { + let prefix = s.hash.substring(0, 4); + if (prefixSet.has(prefix)) { + return; + } + prefixSet.add(prefix); + prefixes.push(prefix); + }); + return 4 + ":" + (4 * prefixes.length) + "\n" + prefixes.join(""); +} + +function parseCompletionRequest(aRequest) { + // Format: [partial_length]:[num_of_prefix * partial_length]\n[prefixes_data] + + let tokens = /(\d):(\d+)/.exec(aRequest); + if (tokens.length < 3) { + dump("Request format error."); + return null; + } + + let partialLength = parseInt(tokens[1]); + let payloadLength = parseInt(tokens[2]); + + let payloadStart = tokens[1].length + // partial length + 1 + // ':' + tokens[2].length + // payload length + 1; // '\n' + + let prefixSet = []; + for (let i = payloadStart; i < aRequest.length; i += partialLength) { + let prefix = aRequest.substr(i, partialLength); + if (prefix.length !== partialLength) { + dump("Header info not correct: " + aRequest.substr(0, payloadStart)); + return null; + } + prefixSet.push(prefix); + } + prefixSet.sort(); + + return prefixSet; +} + +// Compare the requests in string format. +function compareCompletionRequest(aRequest1, aRequest2) { + let prefixSet1 = parseCompletionRequest(aRequest1); + let prefixSet2 = parseCompletionRequest(aRequest2); + + return equal(JSON.stringify(prefixSet1), JSON.stringify(prefixSet2)); +} + +// The fifth completion set is added at runtime by getRandomCompletionSet. +// Each completion in the set only has one response and its purpose is to +// provide an easy way to test the HashCompleter handling an arbitrarily large +// completion set (determined by SIZE_OF_RANDOM_SET). +const SIZE_OF_RANDOM_SET = 16; +function getRandomCompletionSet(forceServerError) { + let completionSet = []; + let hashPrefixes = []; + + let seed = Math.floor(Math.random() * Math.pow(2, 32)); + dump("Using seed of " + seed + " for random completion set.\n"); + let rand = new LFSRgenerator(seed); + + for (let i = 0; i < SIZE_OF_RANDOM_SET; i++) { + let completion = { expectCompletion: false, forceServerError: false, _finished: false }; + + // Generate a random 256 bit hash. First we get a random number and then + // convert it to a string. + let hash; + let prefix; + do { + hash = ""; + let length = 1 + rand.nextNum(5); + for (let i = 0; i < length; i++) + hash += String.fromCharCode(rand.nextNum(8)); + prefix = hash.substring(0,4); + } while (hashPrefixes.indexOf(prefix) != -1); + + hashPrefixes.push(prefix); + completion.hash = hash; + + if (!forceServerError) { + completion.expectCompletion = rand.nextNum(1) == 1; + } else { + completion.forceServerError = true; + } + if (completion.expectCompletion) { + // Generate a random alpha-numeric string of length at most 6 for the + // table name. + completion.table = (rand.nextNum(31)).toString(36); + + completion.chunkId = rand.nextNum(16); + } + completionSet.push(completion); + } + + return completionSet; +} + +var completionSets = [basicCompletionSet, falseCompletionSet, + dupedCompletionSet, multipleResponsesCompletionSet]; +var currentCompletionSet = -1; +var finishedCompletions = 0; + +const SERVER_PORT = 8080; +const SERVER_PATH = "/hash-completer"; +var server; + +// Completion hashes are automatically right-padded with null chars to have a +// length of COMPLETE_LENGTH. +// Taken from nsUrlClassifierDBService.h +const COMPLETE_LENGTH = 32; + +var completer = Cc["@mozilla.org/url-classifier/hashcompleter;1"]. + getService(Ci.nsIUrlClassifierHashCompleter); + +var gethashUrl; + +// Expected highest completion set for which the server sends a response. +var expectedMaxServerCompletionSet = 0; +var maxServerCompletionSet = 0; + +function run_test() { + // Generate a random completion set that return successful responses. + completionSets.push(getRandomCompletionSet(false)); + // We backoff after receiving an error, so requests shouldn't reach the + // server after that. + expectedMaxServerCompletionSet = completionSets.length; + // Generate some completion sets that return 503s. + for (let j = 0; j < 10; ++j) { + completionSets.push(getRandomCompletionSet(true)); + } + + // Fix up the completions before running the test. + for (let completionSet of completionSets) { + for (let completion of completionSet) { + // Pad the right of each |hash| so that the length is COMPLETE_LENGTH. + if (completion.multipleCompletions) { + for (let responseCompletion of completion.completions) { + let numChars = COMPLETE_LENGTH - responseCompletion.hash.length; + responseCompletion.hash += (new Array(numChars + 1)).join("\u0000"); + } + } + else { + let numChars = COMPLETE_LENGTH - completion.hash.length; + completion.hash += (new Array(numChars + 1)).join("\u0000"); + } + } + } + do_test_pending(); + + server = new HttpServer(); + server.registerPathHandler(SERVER_PATH, hashCompleterServer); + + server.start(-1); + const SERVER_PORT = server.identity.primaryPort; + + gethashUrl = "http://localhost:" + SERVER_PORT + SERVER_PATH; + + runNextCompletion(); +} + +function runNextCompletion() { + // The server relies on currentCompletionSet to send the correct response, so + // don't increment it until we start the new set of callbacks. + currentCompletionSet++; + if (currentCompletionSet >= completionSets.length) { + finish(); + return; + } + + dump("Now on completion set index " + currentCompletionSet + ", length " + + completionSets[currentCompletionSet].length + "\n"); + // Number of finished completions for this set. + finishedCompletions = 0; + for (let completion of completionSets[currentCompletionSet]) { + completer.complete(completion.hash.substring(0,4), gethashUrl, + (new callback(completion))); + } +} + +function hashCompleterServer(aRequest, aResponse) { + let stream = aRequest.bodyInputStream; + let wrapperStream = Cc["@mozilla.org/binaryinputstream;1"]. + createInstance(Ci.nsIBinaryInputStream); + wrapperStream.setInputStream(stream); + + let len = stream.available(); + let data = wrapperStream.readBytes(len); + + // Check if we got the expected completion request. + let expectedRequest = buildCompletionRequest(completionSets[currentCompletionSet]); + compareCompletionRequest(data, expectedRequest); + + // To avoid a response with duplicate hash completions, we keep track of all + // completed hash prefixes so far. + let completedHashes = []; + let responseText = ""; + + function responseForCompletion(x) { + return x.table + ":" + x.chunkId + ":" + x.hash.length + "\n" + x.hash; + } + // As per the spec, a server should response with a 204 if there are no + // full-length hashes that match the prefixes. + let httpStatus = 204; + for (let completion of completionSets[currentCompletionSet]) { + if (completion.expectCompletion && + (completedHashes.indexOf(completion.hash) == -1)) { + completedHashes.push(completion.hash); + + if (completion.multipleCompletions) + responseText += completion.completions.map(responseForCompletion).join(""); + else + responseText += responseForCompletion(completion); + } + if (completion.forceServerError) { + httpStatus = 503; + } + } + + dump("Server sending response for " + currentCompletionSet + "\n"); + maxServerCompletionSet = currentCompletionSet; + if (responseText && httpStatus != 503) { + aResponse.write(responseText); + } else { + aResponse.setStatusLine(null, httpStatus, null); + } +} + + +function callback(completion) { + this._completion = completion; +} + +callback.prototype = { + completion: function completion(hash, table, chunkId, trusted) { + do_check_true(this._completion.expectCompletion); + if (this._completion.multipleCompletions) { + for (let completion of this._completion.completions) { + if (completion.hash == hash) { + do_check_eq(JSON.stringify(hash), JSON.stringify(completion.hash)); + do_check_eq(table, completion.table); + do_check_eq(chunkId, completion.chunkId); + + completion._completed = true; + + if (this._completion.completions.every(x => x._completed)) + this._completed = true; + + break; + } + } + } + else { + // Hashes are not actually strings and can contain arbitrary data. + do_check_eq(JSON.stringify(hash), JSON.stringify(this._completion.hash)); + do_check_eq(table, this._completion.table); + do_check_eq(chunkId, this._completion.chunkId); + + this._completed = true; + } + }, + + completionFinished: function completionFinished(status) { + finishedCompletions++; + do_check_eq(!!this._completion.expectCompletion, !!this._completed); + this._completion._finished = true; + + // currentCompletionSet can mutate before all of the callbacks are complete. + if (currentCompletionSet < completionSets.length && + finishedCompletions == completionSets[currentCompletionSet].length) { + runNextCompletion(); + } + }, +}; + +function finish() { + do_check_eq(expectedMaxServerCompletionSet, maxServerCompletionSet); + server.stop(function() { + do_test_finished(); + }); +} diff --git a/toolkit/components/url-classifier/tests/unit/test_listmanager.js b/toolkit/components/url-classifier/tests/unit/test_listmanager.js new file mode 100644 index 000000000..ba11d930e --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/test_listmanager.js @@ -0,0 +1,376 @@ +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); +Cu.import("resource://gre/modules/Services.jsm"); + +XPCOMUtils.defineLazyModuleGetter(this, "NetUtil", + "resource://gre/modules/NetUtil.jsm"); + +// These tables share the same updateURL. +const TEST_TABLE_DATA_LIST = [ + // 0: + { + tableName: "test-listmanager0-digest256", + providerName: "google", + updateUrl: "http://localhost:4444/safebrowsing/update", + gethashUrl: "http://localhost:4444/safebrowsing/gethash0", + }, + + // 1: + { + tableName: "test-listmanager1-digest256", + providerName: "google", + updateUrl: "http://localhost:4444/safebrowsing/update", + gethashUrl: "http://localhost:4444/safebrowsing/gethash1", + }, + + // 2. + { + tableName: "test-listmanager2-digest256", + providerName: "google", + updateUrl: "http://localhost:4444/safebrowsing/update", + gethashUrl: "http://localhost:4444/safebrowsing/gethash2", + } +]; + +// These tables have a different update URL (for v4). +const TEST_TABLE_DATA_V4 = { + tableName: "test-phish-proto", + providerName: "google4", + updateUrl: "http://localhost:5555/safebrowsing/update?", + gethashUrl: "http://localhost:5555/safebrowsing/gethash-v4", +}; +const TEST_TABLE_DATA_V4_DISABLED = { + tableName: "test-unwanted-proto", + providerName: "google4", + updateUrl: "http://localhost:5555/safebrowsing/update?", + gethashUrl: "http://localhost:5555/safebrowsing/gethash-v4", +}; + +const PREF_NEXTUPDATETIME = "browser.safebrowsing.provider.google.nextupdatetime"; +const PREF_NEXTUPDATETIME_V4 = "browser.safebrowsing.provider.google4.nextupdatetime"; + +let gListManager = Cc["@mozilla.org/url-classifier/listmanager;1"] + .getService(Ci.nsIUrlListManager); + +let gUrlUtils = Cc["@mozilla.org/url-classifier/utils;1"] + .getService(Ci.nsIUrlClassifierUtils); + +// Global test server for serving safebrowsing updates. +let gHttpServ = null; +let gUpdateResponse = ""; +let gExpectedUpdateRequest = ""; +let gExpectedQueryV4 = ""; + +// Handles request for TEST_TABLE_DATA_V4. +let gHttpServV4 = null; + +// These two variables are used to synchronize the last two racing updates +// (in terms of "update URL") in test_update_all_tables(). +let gUpdatedCntForTableData = 0; // For TEST_TABLE_DATA_LIST. +let gIsV4Updated = false; // For TEST_TABLE_DATA_V4. + +const NEW_CLIENT_STATE = 'sta\0te'; +const CHECKSUM = '\x30\x67\xc7\x2c\x5e\x50\x1c\x31\xe3\xfe\xca\x73\xf0\x47\xdc\x34\x1a\x95\x63\x99\xec\x70\x5e\x0a\xee\x9e\xfb\x17\xa1\x55\x35\x78'; + +prefBranch.setBoolPref("browser.safebrowsing.debug", true); + +// The "\xFF\xFF" is to generate a base64 string with "/". +prefBranch.setCharPref("browser.safebrowsing.id", "Firefox\xFF\xFF"); + +// Register tables. +TEST_TABLE_DATA_LIST.forEach(function(t) { + gListManager.registerTable(t.tableName, + t.providerName, + t.updateUrl, + t.gethashUrl); +}); + +gListManager.registerTable(TEST_TABLE_DATA_V4.tableName, + TEST_TABLE_DATA_V4.providerName, + TEST_TABLE_DATA_V4.updateUrl, + TEST_TABLE_DATA_V4.gethashUrl); + +// To test Bug 1302044. +gListManager.registerTable(TEST_TABLE_DATA_V4_DISABLED.tableName, + TEST_TABLE_DATA_V4_DISABLED.providerName, + TEST_TABLE_DATA_V4_DISABLED.updateUrl, + TEST_TABLE_DATA_V4_DISABLED.gethashUrl); + +const SERVER_INVOLVED_TEST_CASE_LIST = [ + // - Do table0 update. + // - Server would respond "a:5:32:32\n[DATA]". + function test_update_table0() { + disableAllUpdates(); + + gListManager.enableUpdate(TEST_TABLE_DATA_LIST[0].tableName); + gExpectedUpdateRequest = TEST_TABLE_DATA_LIST[0].tableName + ";\n"; + + gUpdateResponse = "n:1000\ni:" + TEST_TABLE_DATA_LIST[0].tableName + "\n"; + gUpdateResponse += readFileToString("data/digest2.chunk"); + + forceTableUpdate(); + }, + + // - Do table0 update again. Since chunk 5 was added to table0 in the last + // update, the expected request contains "a:5". + // - Server would respond "s;2-12\n[DATA]". + function test_update_table0_with_existing_chunks() { + disableAllUpdates(); + + gListManager.enableUpdate(TEST_TABLE_DATA_LIST[0].tableName); + gExpectedUpdateRequest = TEST_TABLE_DATA_LIST[0].tableName + ";a:5\n"; + + gUpdateResponse = "n:1000\ni:" + TEST_TABLE_DATA_LIST[0].tableName + "\n"; + gUpdateResponse += readFileToString("data/digest1.chunk"); + + forceTableUpdate(); + }, + + // - Do all-table update. + // - Server would respond no chunk control. + // + // Note that this test MUST be the last one in the array since we rely on + // the number of sever-involved test case to synchronize the racing last + // two udpates for different URL. + function test_update_all_tables() { + disableAllUpdates(); + + // Enable all tables including TEST_TABLE_DATA_V4! + TEST_TABLE_DATA_LIST.forEach(function(t) { + gListManager.enableUpdate(t.tableName); + }); + + // We register two v4 tables but only enable one of them + // to verify that the disabled tables are not updated. + // See Bug 1302044. + gListManager.enableUpdate(TEST_TABLE_DATA_V4.tableName); + gListManager.disableUpdate(TEST_TABLE_DATA_V4_DISABLED.tableName); + + // Expected results for v2. + gExpectedUpdateRequest = TEST_TABLE_DATA_LIST[0].tableName + ";a:5:s:2-12\n" + + TEST_TABLE_DATA_LIST[1].tableName + ";\n" + + TEST_TABLE_DATA_LIST[2].tableName + ";\n"; + gUpdateResponse = "n:1000\n"; + + // We test the request against the query string since v4 request + // would be appened to the query string. The request is generated + // by protobuf API (binary) then encoded to base64 format. + let requestV4 = gUrlUtils.makeUpdateRequestV4([TEST_TABLE_DATA_V4.tableName], + [""], + 1); + gExpectedQueryV4 = "&$req=" + requestV4; + + forceTableUpdate(); + }, + +]; + +SERVER_INVOLVED_TEST_CASE_LIST.forEach(t => add_test(t)); + +add_test(function test_partialUpdateV4() { + disableAllUpdates(); + + gListManager.enableUpdate(TEST_TABLE_DATA_V4.tableName); + + // Since the new client state has been responded and saved in + // test_update_all_tables, this update request should send + // a partial update to the server. + let requestV4 = gUrlUtils.makeUpdateRequestV4([TEST_TABLE_DATA_V4.tableName], + [btoa(NEW_CLIENT_STATE)], + 1); + gExpectedQueryV4 = "&$req=" + requestV4; + + forceTableUpdate(); +}); + +// Tests nsIUrlListManager.getGethashUrl. +add_test(function test_getGethashUrl() { + TEST_TABLE_DATA_LIST.forEach(function (t) { + equal(gListManager.getGethashUrl(t.tableName), t.gethashUrl); + }); + equal(gListManager.getGethashUrl(TEST_TABLE_DATA_V4.tableName), + TEST_TABLE_DATA_V4.gethashUrl); + run_next_test(); +}); + +function run_test() { + // Setup primary testing server. + gHttpServ = new HttpServer(); + gHttpServ.registerDirectory("/", do_get_cwd()); + + gHttpServ.registerPathHandler("/safebrowsing/update", function(request, response) { + let body = NetUtil.readInputStreamToString(request.bodyInputStream, + request.bodyInputStream.available()); + + // Verify if the request is as expected. + equal(body, gExpectedUpdateRequest); + + // Respond the update which is controlled by the test case. + response.setHeader("Content-Type", + "application/vnd.google.safebrowsing-update", false); + response.setStatusLine(request.httpVersion, 200, "OK"); + response.bodyOutputStream.write(gUpdateResponse, gUpdateResponse.length); + + gUpdatedCntForTableData++; + + if (gUpdatedCntForTableData !== SERVER_INVOLVED_TEST_CASE_LIST.length) { + // This is not the last test case so run the next once upon the + // the update success. + waitForUpdateSuccess(run_next_test); + return; + } + + if (gIsV4Updated) { + run_next_test(); // All tests are done. Just finish. + return; + } + + do_print("Waiting for TEST_TABLE_DATA_V4 to be tested ..."); + }); + + gHttpServ.start(4444); + + // Setup v4 testing server for the different update URL. + gHttpServV4 = new HttpServer(); + gHttpServV4.registerDirectory("/", do_get_cwd()); + + gHttpServV4.registerPathHandler("/safebrowsing/update", function(request, response) { + // V4 update request body should be empty. + equal(request.bodyInputStream.available(), 0); + + // Not on the spec. Found in Chromium source code... + equal(request.getHeader("X-HTTP-Method-Override"), "POST"); + + // V4 update request uses GET. + equal(request.method, "GET"); + + // V4 append the base64 encoded request to the query string. + equal(request.queryString, gExpectedQueryV4); + equal(request.queryString.indexOf('+'), -1); + equal(request.queryString.indexOf('/'), -1); + + // Respond a V2 compatible content for now. In the future we can + // send a meaningful response to test Bug 1284178 to see if the + // update is successfully stored to database. + response.setHeader("Content-Type", + "application/vnd.google.safebrowsing-update", false); + response.setStatusLine(request.httpVersion, 200, "OK"); + + // The protobuf binary represention of response: + // + // [ + // { + // 'threat_type': 2, // SOCIAL_ENGINEERING_PUBLIC + // 'response_type': 2, // FULL_UPDATE + // 'new_client_state': 'sta\x00te', // NEW_CLIENT_STATE + // 'checksum': { "sha256": CHECKSUM }, // CHECKSUM + // 'additions': { 'compression_type': RAW, + // 'prefix_size': 4, + // 'raw_hashes': "00000001000000020000000300000004"} + // } + // ] + // + let content = "\x0A\x4A\x08\x02\x20\x02\x2A\x18\x08\x01\x12\x14\x08\x04\x12\x10\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x3A\x06\x73\x74\x61\x00\x74\x65\x42\x22\x0A\x20\x30\x67\xC7\x2C\x5E\x50\x1C\x31\xE3\xFE\xCA\x73\xF0\x47\xDC\x34\x1A\x95\x63\x99\xEC\x70\x5E\x0A\xEE\x9E\xFB\x17\xA1\x55\x35\x78\x12\x08\x08\x08\x10\x80\x94\xEB\xDC\x03"; + + response.bodyOutputStream.write(content, content.length); + + if (gIsV4Updated) { + // This falls to the case where test_partialUpdateV4 is running. + // We are supposed to have verified the update request contains + // the state we set in the previous request. + run_next_test(); + return; + } + + waitUntilMetaDataSaved(NEW_CLIENT_STATE, CHECKSUM, () => { + gIsV4Updated = true; + + if (gUpdatedCntForTableData === SERVER_INVOLVED_TEST_CASE_LIST.length) { + // All tests are done! + run_next_test(); + return; + } + + do_print("Wait for all sever-involved tests to be done ..."); + }); + + }); + + gHttpServV4.start(5555); + + run_next_test(); +} + +// A trick to force updating tables. However, before calling this, we have to +// call disableAllUpdates() first to clean up the updateCheckers in listmanager. +function forceTableUpdate() { + prefBranch.setCharPref(PREF_NEXTUPDATETIME, "1"); + prefBranch.setCharPref(PREF_NEXTUPDATETIME_V4, "1"); + gListManager.maybeToggleUpdateChecking(); +} + +function disableAllUpdates() { + TEST_TABLE_DATA_LIST.forEach(t => gListManager.disableUpdate(t.tableName)); + gListManager.disableUpdate(TEST_TABLE_DATA_V4.tableName); +} + +// Since there's no public interface on listmanager to know the update success, +// we could only rely on the refresh of "nextupdatetime". +function waitForUpdateSuccess(callback) { + let nextupdatetime = parseInt(prefBranch.getCharPref(PREF_NEXTUPDATETIME)); + do_print("nextupdatetime: " + nextupdatetime); + if (nextupdatetime !== 1) { + callback(); + return; + } + do_timeout(1000, waitForUpdateSuccess.bind(null, callback)); +} + +// Construct an update from a file. +function readFileToString(aFilename) { + let f = do_get_file(aFilename); + let stream = Cc["@mozilla.org/network/file-input-stream;1"] + .createInstance(Ci.nsIFileInputStream); + stream.init(f, -1, 0, 0); + let buf = NetUtil.readInputStreamToString(stream, stream.available()); + return buf; +} + +function waitUntilMetaDataSaved(expectedState, expectedChecksum, callback) { + let dbService = Cc["@mozilla.org/url-classifier/dbservice;1"] + .getService(Ci.nsIUrlClassifierDBService); + + dbService.getTables(metaData => { + do_print("metadata: " + metaData); + let didCallback = false; + metaData.split("\n").some(line => { + // Parse [tableName];[stateBase64] + let p = line.indexOf(";"); + if (-1 === p) { + return false; // continue. + } + let tableName = line.substring(0, p); + let metadata = line.substring(p + 1).split(":"); + let stateBase64 = metadata[0]; + let checksumBase64 = metadata[1]; + + if (tableName !== 'test-phish-proto') { + return false; // continue. + } + + if (stateBase64 === btoa(expectedState) && + checksumBase64 === btoa(expectedChecksum)) { + do_print('State has been saved to disk!'); + callback(); + didCallback = true; + } + + return true; // break no matter whether the state is matching. + }); + + if (!didCallback) { + do_timeout(1000, waitUntilMetaDataSaved.bind(null, expectedState, + expectedChecksum, + callback)); + } + }); +} diff --git a/toolkit/components/url-classifier/tests/unit/test_partial.js b/toolkit/components/url-classifier/tests/unit/test_partial.js new file mode 100644 index 000000000..83243fb4e --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/test_partial.js @@ -0,0 +1,825 @@ + +/** + * DummyCompleter() lets tests easily specify the results of a partial + * hash completion request. + */ +function DummyCompleter() { + this.fragments = {}; + this.queries = []; + this.tableName = "test-phish-simple"; +} + +DummyCompleter.prototype = +{ +QueryInterface: function(iid) +{ + if (!iid.equals(Ci.nsISupports) && + !iid.equals(Ci.nsIUrlClassifierHashCompleter)) { + throw Cr.NS_ERROR_NO_INTERFACE; + } + return this; +}, + +complete: function(partialHash, gethashUrl, cb) +{ + this.queries.push(partialHash); + var fragments = this.fragments; + var self = this; + var doCallback = function() { + if (self.alwaysFail) { + cb.completionFinished(1); + return; + } + var results; + if (fragments[partialHash]) { + for (var i = 0; i < fragments[partialHash].length; i++) { + var chunkId = fragments[partialHash][i][0]; + var hash = fragments[partialHash][i][1]; + cb.completion(hash, self.tableName, chunkId); + } + } + cb.completionFinished(0); + } + var timer = new Timer(0, doCallback); +}, + +getHash: function(fragment) +{ + var converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]. + createInstance(Ci.nsIScriptableUnicodeConverter); + converter.charset = "UTF-8"; + var data = converter.convertToByteArray(fragment); + var ch = Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); + ch.init(ch.SHA256); + ch.update(data, data.length); + var hash = ch.finish(false); + return hash.slice(0, 32); +}, + +addFragment: function(chunkId, fragment) +{ + this.addHash(chunkId, this.getHash(fragment)); +}, + +// This method allows the caller to generate complete hashes that match the +// prefix of a real fragment, but have different complete hashes. +addConflict: function(chunkId, fragment) +{ + var realHash = this.getHash(fragment); + var invalidHash = this.getHash("blah blah blah blah blah"); + this.addHash(chunkId, realHash.slice(0, 4) + invalidHash.slice(4, 32)); +}, + +addHash: function(chunkId, hash) +{ + var partial = hash.slice(0, 4); + if (this.fragments[partial]) { + this.fragments[partial].push([chunkId, hash]); + } else { + this.fragments[partial] = [[chunkId, hash]]; + } +}, + +compareQueries: function(fragments) +{ + var expectedQueries = []; + for (var i = 0; i < fragments.length; i++) { + expectedQueries.push(this.getHash(fragments[i]).slice(0, 4)); + } + do_check_eq(this.queries.length, expectedQueries.length); + expectedQueries.sort(); + this.queries.sort(); + for (var i = 0; i < this.queries.length; i++) { + do_check_eq(this.queries[i], expectedQueries[i]); + } +} +}; + +function setupCompleter(table, hits, conflicts) +{ + var completer = new DummyCompleter(); + completer.tableName = table; + for (var i = 0; i < hits.length; i++) { + var chunkId = hits[i][0]; + var fragments = hits[i][1]; + for (var j = 0; j < fragments.length; j++) { + completer.addFragment(chunkId, fragments[j]); + } + } + for (var i = 0; i < conflicts.length; i++) { + var chunkId = conflicts[i][0]; + var fragments = conflicts[i][1]; + for (var j = 0; j < fragments.length; j++) { + completer.addConflict(chunkId, fragments[j]); + } + } + + dbservice.setHashCompleter(table, completer); + + return completer; +} + +function installCompleter(table, fragments, conflictFragments) +{ + return setupCompleter(table, fragments, conflictFragments); +} + +function installFailingCompleter(table) { + var completer = setupCompleter(table, [], []); + completer.alwaysFail = true; + return completer; +} + +// Helper assertion for checking dummy completer queries +gAssertions.completerQueried = function(data, cb) +{ + var completer = data[0]; + completer.compareQueries(data[1]); + cb(); +} + +function doTest(updates, assertions) +{ + doUpdateTest(updates, assertions, runNextTest, updateError); +} + +// Test an add of two partial urls to a fresh database +function testPartialAdds() { + var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ]; + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }], + 4); + + + var completer = installCompleter('test-phish-simple', [[1, addUrls]], []); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + "urlsExist" : addUrls, + "completerQueried" : [completer, addUrls] + }; + + + doTest([update], assertions); +} + +function testPartialAddsWithConflicts() { + var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ]; + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }], + 4); + + // Each result will have both a real match and a conflict + var completer = installCompleter('test-phish-simple', + [[1, addUrls]], + [[1, addUrls]]); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + "urlsExist" : addUrls, + "completerQueried" : [completer, addUrls] + }; + + doTest([update], assertions); +} + +// Test whether the fragmenting code does not cause duplicated completions +function testFragments() { + var addUrls = [ "foo.com/a/b/c", "foo.net/", "foo.com/c/" ]; + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }], + 4); + + + var completer = installCompleter('test-phish-simple', [[1, addUrls]], []); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + "urlsExist" : addUrls, + "completerQueried" : [completer, addUrls] + }; + + + doTest([update], assertions); +} + +// Test http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec +// section 6.2 example 1 +function testSpecFragments() { + var probeUrls = [ "a.b.c/1/2.html?param=1" ]; + + var addUrls = [ "a.b.c/1/2.html", + "a.b.c/", + "a.b.c/1/", + "b.c/1/2.html?param=1", + "b.c/1/2.html", + "b.c/", + "b.c/1/", + "a.b.c/1/2.html?param=1" ]; + + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }], + 4); + + + var completer = installCompleter('test-phish-simple', [[1, addUrls]], []); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + "urlsExist" : probeUrls, + "completerQueried" : [completer, addUrls] + }; + + doTest([update], assertions); + +} + +// Test http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec +// section 6.2 example 2 +function testMoreSpecFragments() { + var probeUrls = [ "a.b.c.d.e.f.g/1.html" ]; + + var addUrls = [ "a.b.c.d.e.f.g/1.html", + "a.b.c.d.e.f.g/", + "c.d.e.f.g/1.html", + "c.d.e.f.g/", + "d.e.f.g/1.html", + "d.e.f.g/", + "e.f.g/1.html", + "e.f.g/", + "f.g/1.html", + "f.g/" ]; + + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }], + 4); + + var completer = installCompleter('test-phish-simple', [[1, addUrls]], []); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + "urlsExist" : probeUrls, + "completerQueried" : [completer, addUrls] + }; + + doTest([update], assertions); + +} + +function testFalsePositives() { + var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ]; + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }], + 4); + + // Each result will have no matching complete hashes and a non-matching + // conflict + var completer = installCompleter('test-phish-simple', [], [[1, addUrls]]); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + "urlsDontExist" : addUrls, + "completerQueried" : [completer, addUrls] + }; + + doTest([update], assertions); +} + +function testEmptyCompleter() { + var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ]; + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }], + 4); + + // Completer will never return full hashes + var completer = installCompleter('test-phish-simple', [], []); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + "urlsDontExist" : addUrls, + "completerQueried" : [completer, addUrls] + }; + + doTest([update], assertions); +} + +function testCompleterFailure() { + var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ]; + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }], + 4); + + // Completer will never return full hashes + var completer = installFailingCompleter('test-phish-simple'); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + "urlsDontExist" : addUrls, + "completerQueried" : [completer, addUrls] + }; + + doTest([update], assertions); +} + +function testMixedSizesSameDomain() { + var add1Urls = [ "foo.com/a" ]; + var add2Urls = [ "foo.com/b" ]; + + var update1 = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : add1Urls }], + 4); + var update2 = buildPhishingUpdate( + [ + { "chunkNum" : 2, + "urls" : add2Urls }], + 32); + + // We should only need to complete the partial hashes + var completer = installCompleter('test-phish-simple', [[1, add1Urls]], []); + + var assertions = { + "tableData" : "test-phish-simple;a:1-2", + // both urls should match... + "urlsExist" : add1Urls.concat(add2Urls), + // ... but the completer should only be queried for the partial entry + "completerQueried" : [completer, add1Urls] + }; + + doTest([update1, update2], assertions); +} + +function testMixedSizesDifferentDomains() { + var add1Urls = [ "foo.com/a" ]; + var add2Urls = [ "bar.com/b" ]; + + var update1 = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : add1Urls }], + 4); + var update2 = buildPhishingUpdate( + [ + { "chunkNum" : 2, + "urls" : add2Urls }], + 32); + + // We should only need to complete the partial hashes + var completer = installCompleter('test-phish-simple', [[1, add1Urls]], []); + + var assertions = { + "tableData" : "test-phish-simple;a:1-2", + // both urls should match... + "urlsExist" : add1Urls.concat(add2Urls), + // ... but the completer should only be queried for the partial entry + "completerQueried" : [completer, add1Urls] + }; + + doTest([update1, update2], assertions); +} + +function testInvalidHashSize() +{ + var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ]; + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }], + 12); // only 4 and 32 are legal hash sizes + + var addUrls2 = [ "zaz.com/a", "xyz.com/b" ]; + var update2 = buildPhishingUpdate( + [ + { "chunkNum" : 2, + "urls" : addUrls2 + }], + 4); + + var completer = installCompleter('test-phish-simple', [[1, addUrls]], []); + + var assertions = { + "tableData" : "test-phish-simple;a:2", + "urlsDontExist" : addUrls + }; + + // A successful update will trigger an error + doUpdateTest([update2, update], assertions, updateError, runNextTest); +} + +function testWrongTable() +{ + var addUrls = [ "foo.com/a" ]; + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }], + 4); + var completer = installCompleter('test-malware-simple', // wrong table + [[1, addUrls]], []); + + // The above installCompleter installs the completer for test-malware-simple, + // we want it to be used for test-phish-simple too. + dbservice.setHashCompleter("test-phish-simple", completer); + + + var assertions = { + "tableData" : "test-phish-simple;a:1", + // The urls were added as phishing urls, but the completer is claiming + // that they are malware urls, and we trust the completer in this case. + // The result will be discarded, so we can only check for non-existence. + "urlsDontExist" : addUrls, + // Make sure the completer was actually queried. + "completerQueried" : [completer, addUrls] + }; + + doUpdateTest([update], assertions, + function() { + // Give the dbservice a chance to (not) cache the result. + var timer = new Timer(3000, function() { + // The miss earlier will have caused a miss to be cached. + // Resetting the completer does not count as an update, + // so we will not be probed again. + var newCompleter = installCompleter('test-malware-simple', [[1, addUrls]], []); dbservice.setHashCompleter("test-phish-simple", + newCompleter); + + var assertions = { + "urlsDontExist" : addUrls + }; + checkAssertions(assertions, runNextTest); + }); + }, updateError); +} + +function setupCachedResults(addUrls, part2) +{ + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }], + 4); + + var completer = installCompleter('test-phish-simple', [[1, addUrls]], []); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + // Request the add url. This should cause the completion to be cached. + "urlsExist" : addUrls, + // Make sure the completer was actually queried. + "completerQueried" : [completer, addUrls] + }; + + doUpdateTest([update], assertions, + function() { + // Give the dbservice a chance to cache the result. + var timer = new Timer(3000, part2); + }, updateError); +} + +function testCachedResults() +{ + setupCachedResults(["foo.com/a"], function(add) { + // This is called after setupCachedResults(). Verify that + // checking the url again does not cause a completer request. + + // install a new completer, this one should never be queried. + var newCompleter = installCompleter('test-phish-simple', [[1, []]], []); + + var assertions = { + "urlsExist" : ["foo.com/a"], + "completerQueried" : [newCompleter, []] + }; + checkAssertions(assertions, runNextTest); + }); +} + +function testCachedResultsWithSub() { + setupCachedResults(["foo.com/a"], function() { + // install a new completer, this one should never be queried. + var newCompleter = installCompleter('test-phish-simple', [[1, []]], []); + + var removeUpdate = buildPhishingUpdate( + [ { "chunkNum" : 2, + "chunkType" : "s", + "urls": ["1:foo.com/a"] }], + 4); + + var assertions = { + "urlsDontExist" : ["foo.com/a"], + "completerQueried" : [newCompleter, []] + } + + doTest([removeUpdate], assertions); + }); +} + +function testCachedResultsWithExpire() { + setupCachedResults(["foo.com/a"], function() { + // install a new completer, this one should never be queried. + var newCompleter = installCompleter('test-phish-simple', [[1, []]], []); + + var expireUpdate = + "n:1000\n" + + "i:test-phish-simple\n" + + "ad:1\n"; + + var assertions = { + "urlsDontExist" : ["foo.com/a"], + "completerQueried" : [newCompleter, []] + } + doTest([expireUpdate], assertions); + }); +} + +function testCachedResultsUpdate() +{ + var existUrls = ["foo.com/a"]; + setupCachedResults(existUrls, function() { + // This is called after setupCachedResults(). Verify that + // checking the url again does not cause a completer request. + + // install a new completer, this one should never be queried. + var newCompleter = installCompleter('test-phish-simple', [[1, []]], []); + + var assertions = { + "urlsExist" : existUrls, + "completerQueried" : [newCompleter, []] + }; + + var addUrls = ["foobar.org/a"]; + + var update2 = buildPhishingUpdate( + [ + { "chunkNum" : 2, + "urls" : addUrls + }], + 4); + + checkAssertions(assertions, function () { + // Apply the update. The cached completes should be gone. + doStreamUpdate(update2, function() { + // Now the completer gets queried again. + var newCompleter2 = installCompleter('test-phish-simple', [[1, existUrls]], []); + var assertions2 = { + "tableData" : "test-phish-simple;a:1-2", + "urlsExist" : existUrls, + "completerQueried" : [newCompleter2, existUrls] + }; + checkAssertions(assertions2, runNextTest); + }, updateError); + }); + }); +} + +function testCachedResultsFailure() +{ + var existUrls = ["foo.com/a"]; + setupCachedResults(existUrls, function() { + // This is called after setupCachedResults(). Verify that + // checking the url again does not cause a completer request. + + // install a new completer, this one should never be queried. + var newCompleter = installCompleter('test-phish-simple', [[1, []]], []); + + var assertions = { + "urlsExist" : existUrls, + "completerQueried" : [newCompleter, []] + }; + + var addUrls = ["foobar.org/a"]; + + var update2 = buildPhishingUpdate( + [ + { "chunkNum" : 2, + "urls" : addUrls + }], + 4); + + checkAssertions(assertions, function() { + // Apply the update. The cached completes should be gone. + doErrorUpdate("test-phish-simple,test-malware-simple", function() { + // Now the completer gets queried again. + var newCompleter2 = installCompleter('test-phish-simple', [[1, existUrls]], []); + var assertions2 = { + "tableData" : "test-phish-simple;a:1", + "urlsExist" : existUrls, + "completerQueried" : [newCompleter2, existUrls] + }; + checkAssertions(assertions2, runNextTest); + }, updateError); + }); + }); +} + +function testErrorList() +{ + var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ]; + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }], + 4); + // The update failure should will kill the completes, so the above + // must be a prefix to get any hit at all past the update failure. + + var completer = installCompleter('test-phish-simple', [[1, addUrls]], []); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + "urlsExist" : addUrls, + // These are complete urls, and will only be completed if the + // list is stale. + "completerQueried" : [completer, addUrls] + }; + + // Apply the update. + doStreamUpdate(update, function() { + // Now the test-phish-simple and test-malware-simple tables are marked + // as fresh. Fake an update failure to mark them stale. + doErrorUpdate("test-phish-simple,test-malware-simple", function() { + // Now the lists should be marked stale. Check assertions. + checkAssertions(assertions, runNextTest); + }, updateError); + }, updateError); +} + + +function testStaleList() +{ + var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ]; + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }], + 32); + + var completer = installCompleter('test-phish-simple', [[1, addUrls]], []); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + "urlsExist" : addUrls, + // These are complete urls, and will only be completed if the + // list is stale. + "completerQueried" : [completer, addUrls] + }; + + // Consider a match stale after one second. + prefBranch.setIntPref("urlclassifier.max-complete-age", 1); + + // Apply the update. + doStreamUpdate(update, function() { + // Now the test-phish-simple and test-malware-simple tables are marked + // as fresh. Wait three seconds to make sure the list is marked stale. + new Timer(3000, function() { + // Now the lists should be marked stale. Check assertions. + checkAssertions(assertions, function() { + prefBranch.setIntPref("urlclassifier.max-complete-age", 2700); + runNextTest(); + }); + }, updateError); + }, updateError); +} + +// Same as testStaleList, but verifies that an empty response still +// unconfirms the entry. +function testStaleListEmpty() +{ + var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ]; + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls + }], + 32); + + var completer = installCompleter('test-phish-simple', [], []); + + var assertions = { + "tableData" : "test-phish-simple;a:1", + // None of these should match, because they won't be completed + "urlsDontExist" : addUrls, + // These are complete urls, and will only be completed if the + // list is stale. + "completerQueried" : [completer, addUrls] + }; + + // Consider a match stale after one second. + prefBranch.setIntPref("urlclassifier.max-complete-age", 1); + + // Apply the update. + doStreamUpdate(update, function() { + // Now the test-phish-simple and test-malware-simple tables are marked + // as fresh. Wait three seconds to make sure the list is marked stale. + new Timer(3000, function() { + // Now the lists should be marked stale. Check assertions. + checkAssertions(assertions, function() { + prefBranch.setIntPref("urlclassifier.max-complete-age", 2700); + runNextTest(); + }); + }, updateError); + }, updateError); +} + + +// Verify that different lists (test-phish-simple, +// test-malware-simple) maintain their freshness separately. +function testErrorListIndependent() +{ + var phishUrls = [ "phish.com/a" ]; + var malwareUrls = [ "attack.com/a" ]; + var update = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : phishUrls + }], + 4); + // These have to persist past the update failure, so they must be prefixes, + // not completes. + + update += buildMalwareUpdate( + [ + { "chunkNum" : 2, + "urls" : malwareUrls + }], + 32); + + var completer = installCompleter('test-phish-simple', [[1, phishUrls]], []); + + var assertions = { + "tableData" : "test-malware-simple;a:2\ntest-phish-simple;a:1", + "urlsExist" : phishUrls, + "malwareUrlsExist" : malwareUrls, + // Only this phishing urls should be completed, because only the phishing + // urls will be stale. + "completerQueried" : [completer, phishUrls] + }; + + // Apply the update. + doStreamUpdate(update, function() { + // Now the test-phish-simple and test-malware-simple tables are + // marked as fresh. Fake an update failure to mark *just* + // phishing data as stale. + doErrorUpdate("test-phish-simple", function() { + // Now the lists should be marked stale. Check assertions. + checkAssertions(assertions, runNextTest); + }, updateError); + }, updateError); +} + +function run_test() +{ + runTests([ + testPartialAdds, + testPartialAddsWithConflicts, + testFragments, + testSpecFragments, + testMoreSpecFragments, + testFalsePositives, + testEmptyCompleter, + testCompleterFailure, + testMixedSizesSameDomain, + testMixedSizesDifferentDomains, + testInvalidHashSize, + testWrongTable, + testCachedResults, + testCachedResultsWithSub, + testCachedResultsWithExpire, + testCachedResultsUpdate, + testCachedResultsFailure, + testStaleList, + testStaleListEmpty, + testErrorList, + testErrorListIndependent + ]); +} + +do_test_pending(); diff --git a/toolkit/components/url-classifier/tests/unit/test_pref.js b/toolkit/components/url-classifier/tests/unit/test_pref.js new file mode 100644 index 000000000..68030a246 --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/test_pref.js @@ -0,0 +1,14 @@ +function run_test() { + let urlUtils = Cc["@mozilla.org/url-classifier/utils;1"] + .getService(Ci.nsIUrlClassifierUtils); + + // The google protocol version should be "2.2" until we enable SB v4 + // by default. + equal(urlUtils.getProtocolVersion("google"), "2.2"); + + // Mozilla protocol version will stick to "2.2". + equal(urlUtils.getProtocolVersion("mozilla"), "2.2"); + + // Unknown provider version will be "2.2". + equal(urlUtils.getProtocolVersion("unknown-provider"), "2.2"); +}
\ No newline at end of file diff --git a/toolkit/components/url-classifier/tests/unit/test_prefixset.js b/toolkit/components/url-classifier/tests/unit/test_prefixset.js new file mode 100644 index 000000000..f2ecc9c2b --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/test_prefixset.js @@ -0,0 +1,232 @@ +// newPset: returns an empty nsIUrlClassifierPrefixSet. +function newPset() { + let pset = Cc["@mozilla.org/url-classifier/prefixset;1"] + .createInstance(Ci.nsIUrlClassifierPrefixSet); + pset.init("all"); + return pset; +} + +// arrContains: returns true if |arr| contains the element |target|. Uses binary +// search and requires |arr| to be sorted. +function arrContains(arr, target) { + let start = 0; + let end = arr.length - 1; + let i = 0; + + while (end > start) { + i = start + (end - start >> 1); + let value = arr[i]; + + if (value < target) + start = i+1; + else if (value > target) + end = i-1; + else + break; + } + if (start == end) + i = start; + + return (!(i < 0 || i >= arr.length) && arr[i] == target); +} + +// checkContents: Check whether the PrefixSet pset contains +// the prefixes in the passed array. +function checkContents(pset, prefixes) { + var outcount = {}, outset = {}; + outset = pset.getPrefixes(outcount); + let inset = prefixes; + do_check_eq(inset.length, outset.length); + inset.sort((x,y) => x - y); + for (let i = 0; i < inset.length; i++) { + do_check_eq(inset[i], outset[i]); + } +} + +function wrappedProbe(pset, prefix) { + return pset.contains(prefix); +}; + +// doRandomLookups: we use this to test for false membership with random input +// over the range of prefixes (unsigned 32-bits integers). +// pset: a nsIUrlClassifierPrefixSet to test. +// prefixes: an array of prefixes supposed to make up the prefix set. +// N: number of random lookups to make. +function doRandomLookups(pset, prefixes, N) { + for (let i = 0; i < N; i++) { + let randInt = prefixes[0]; + while (arrContains(prefixes, randInt)) + randInt = Math.floor(Math.random() * Math.pow(2, 32)); + + do_check_false(wrappedProbe(pset, randInt)); + } +} + +// doExpectedLookups: we use this to test expected membership. +// pset: a nsIUrlClassifierPrefixSet to test. +// prefixes: +function doExpectedLookups(pset, prefixes, N) { + for (let i = 0; i < N; i++) { + prefixes.forEach(function (x) { + dump("Checking " + x + "\n"); + do_check_true(wrappedProbe(pset, x)); + }); + } +} + +// testBasicPset: A very basic test of the prefix set to make sure that it +// exists and to give a basic example of its use. +function testBasicPset() { + let pset = Cc["@mozilla.org/url-classifier/prefixset;1"] + .createInstance(Ci.nsIUrlClassifierPrefixSet); + let prefixes = [2,50,100,2000,78000,1593203]; + pset.setPrefixes(prefixes, prefixes.length); + + do_check_true(wrappedProbe(pset, 100)); + do_check_false(wrappedProbe(pset, 100000)); + do_check_true(wrappedProbe(pset, 1593203)); + do_check_false(wrappedProbe(pset, 999)); + do_check_false(wrappedProbe(pset, 0)); + + + checkContents(pset, prefixes); +} + +function testDuplicates() { + let pset = Cc["@mozilla.org/url-classifier/prefixset;1"] + .createInstance(Ci.nsIUrlClassifierPrefixSet); + let prefixes = [1,1,2,2,2,3,3,3,3,3,3,5,6,6,7,7,9,9,9]; + pset.setPrefixes(prefixes, prefixes.length); + + do_check_true(wrappedProbe(pset, 1)); + do_check_true(wrappedProbe(pset, 2)); + do_check_true(wrappedProbe(pset, 5)); + do_check_true(wrappedProbe(pset, 9)); + do_check_false(wrappedProbe(pset, 4)); + do_check_false(wrappedProbe(pset, 8)); + + + checkContents(pset, prefixes); +} + +function testSimplePset() { + let pset = newPset(); + let prefixes = [1,2,100,400,123456789]; + pset.setPrefixes(prefixes, prefixes.length); + + doRandomLookups(pset, prefixes, 100); + doExpectedLookups(pset, prefixes, 1); + + + checkContents(pset, prefixes); +} + +function testReSetPrefixes() { + let pset = newPset(); + let prefixes = [1, 5, 100, 1000, 150000]; + pset.setPrefixes(prefixes, prefixes.length); + + doExpectedLookups(pset, prefixes, 1); + + let secondPrefixes = [12, 50, 300, 2000, 5000, 200000]; + pset.setPrefixes(secondPrefixes, secondPrefixes.length); + + doExpectedLookups(pset, secondPrefixes, 1); + for (let i = 0; i < prefixes.length; i++) { + do_check_false(wrappedProbe(pset, prefixes[i])); + } + + + checkContents(pset, secondPrefixes); +} + +function testLoadSaveLargeSet() { + let N = 1000; + let arr = []; + + for (let i = 0; i < N; i++) { + let randInt = Math.floor(Math.random() * Math.pow(2, 32)); + arr.push(randInt); + } + + arr.sort((x,y) => x - y); + + let pset = newPset(); + pset.setPrefixes(arr, arr.length); + + doExpectedLookups(pset, arr, 1); + doRandomLookups(pset, arr, 1000); + + checkContents(pset, arr); + + // Now try to save, restore, and redo the lookups + var file = dirSvc.get('ProfLD', Ci.nsIFile); + file.append("testLarge.pset"); + + pset.storeToFile(file); + + let psetLoaded = newPset(); + psetLoaded.loadFromFile(file); + + doExpectedLookups(psetLoaded, arr, 1); + doRandomLookups(psetLoaded, arr, 1000); + + checkContents(psetLoaded, arr); +} + +function testTinySet() { + let pset = Cc["@mozilla.org/url-classifier/prefixset;1"] + .createInstance(Ci.nsIUrlClassifierPrefixSet); + let prefixes = [1]; + pset.setPrefixes(prefixes, prefixes.length); + + do_check_true(wrappedProbe(pset, 1)); + do_check_false(wrappedProbe(pset, 100000)); + checkContents(pset, prefixes); + + prefixes = []; + pset.setPrefixes(prefixes, prefixes.length); + do_check_false(wrappedProbe(pset, 1)); + checkContents(pset, prefixes); +} + +function testLoadSaveNoDelta() { + let N = 100; + let arr = []; + + for (let i = 0; i < N; i++) { + // construct a tree without deltas by making the distance + // between entries larger than 16 bits + arr.push(((1 << 16) + 1) * i); + } + + let pset = newPset(); + pset.setPrefixes(arr, arr.length); + + doExpectedLookups(pset, arr, 1); + + var file = dirSvc.get('ProfLD', Ci.nsIFile); + file.append("testNoDelta.pset"); + + pset.storeToFile(file); + pset.loadFromFile(file); + + doExpectedLookups(pset, arr, 1); +} + +var tests = [testBasicPset, + testSimplePset, + testReSetPrefixes, + testLoadSaveLargeSet, + testDuplicates, + testTinySet, + testLoadSaveNoDelta]; + +function run_test() { + // None of the tests use |executeSoon| or any sort of callbacks, so we can + // just run them in succession. + for (let i = 0; i < tests.length; i++) { + dump("Running " + tests[i].name + "\n"); + tests[i](); + } +} diff --git a/toolkit/components/url-classifier/tests/unit/test_provider_url.js b/toolkit/components/url-classifier/tests/unit/test_provider_url.js new file mode 100644 index 000000000..9a946dc3f --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/test_provider_url.js @@ -0,0 +1,34 @@ +Cu.import("resource://testing-common/AppInfo.jsm", this); +Cu.import("resource://gre/modules/Services.jsm"); + +function updateVersion(version) { + updateAppInfo({ version }); +} + +add_test(function test_provider_url() { + let urls = [ + "browser.safebrowsing.provider.google.updateURL", + "browser.safebrowsing.provider.google.gethashURL", + "browser.safebrowsing.provider.mozilla.updateURL", + "browser.safebrowsing.provider.mozilla.gethashURL" + ]; + + let versions = [ + "49.0", + "49.0.1", + "49.0a1", + "49.0b1", + "49.0esr", + "49.0.1esr" + ]; + + for (let version of versions) { + for (let url of urls) { + updateVersion(version); + let value = Services.urlFormatter.formatURLPref(url); + Assert.notEqual(value.indexOf("&appver=49.0&"), -1); + } + } + + run_next_test(); +}); diff --git a/toolkit/components/url-classifier/tests/unit/test_safebrowsing_protobuf.js b/toolkit/components/url-classifier/tests/unit/test_safebrowsing_protobuf.js new file mode 100644 index 000000000..45309ba54 --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/test_safebrowsing_protobuf.js @@ -0,0 +1,23 @@ +function run_test() { + let urlUtils = Cc["@mozilla.org/url-classifier/utils;1"] + .getService(Ci.nsIUrlClassifierUtils); + + // No list at all. + let requestNoList = urlUtils.makeUpdateRequestV4([], [], 0); + + // Only one valid list name. + let requestOneValid = + urlUtils.makeUpdateRequestV4(["goog-phish-proto"], ["AAAAAA"], 1); + + // Only one invalid list name. + let requestOneInvalid = + urlUtils.makeUpdateRequestV4(["bad-list-name"], ["AAAAAA"], 1); + + // One valid and one invalid list name. + let requestOneInvalidOneValid = + urlUtils.makeUpdateRequestV4(["goog-phish-proto", "bad-list-name"], + ["AAAAAA", "AAAAAA"], 2); + + equal(requestNoList, requestOneInvalid); + equal(requestOneValid, requestOneInvalidOneValid); +}
\ No newline at end of file diff --git a/toolkit/components/url-classifier/tests/unit/test_streamupdater.js b/toolkit/components/url-classifier/tests/unit/test_streamupdater.js new file mode 100644 index 000000000..e5abc4e91 --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/test_streamupdater.js @@ -0,0 +1,288 @@ +function doTest(updates, assertions, expectError) +{ + if (expectError) { + doUpdateTest(updates, assertions, updateError, runNextTest); + } else { + doUpdateTest(updates, assertions, runNextTest, updateError); + } +} + +// Never use the same URLs for multiple tests, because we aren't guaranteed +// to reset the database between tests. +function testFillDb() { + var add1Urls = [ "zaz.com/a", "yxz.com/c" ]; + + var update = "n:1000\n"; + update += "i:test-phish-simple\n"; + + var update1 = buildBareUpdate( + [{ "chunkNum" : 1, + "urls" : add1Urls }]); + update += "u:data:," + encodeURIComponent(update1) + "\n"; + + var assertions = { + "tableData" : "test-phish-simple;a:1", + "urlsExist" : add1Urls + }; + + doTest([update], assertions, false); +} + +function testSimpleForward() { + var add1Urls = [ "foo-simple.com/a", "bar-simple.com/c" ]; + var add2Urls = [ "foo-simple.com/b" ]; + var add3Urls = [ "bar-simple.com/d" ]; + + var update = "n:1000\n"; + update += "i:test-phish-simple\n"; + + var update1 = buildBareUpdate( + [{ "chunkNum" : 1, + "urls" : add1Urls }]); + update += "u:data:," + encodeURIComponent(update1) + "\n"; + + var update2 = buildBareUpdate( + [{ "chunkNum" : 2, + "urls" : add2Urls }]); + update += "u:data:," + encodeURIComponent(update2) + "\n"; + + var update3 = buildBareUpdate( + [{ "chunkNum" : 3, + "urls" : add3Urls }]); + update += "u:data:," + encodeURIComponent(update3) + "\n"; + + var assertions = { + "tableData" : "test-phish-simple;a:1-3", + "urlsExist" : add1Urls.concat(add2Urls).concat(add3Urls) + }; + + doTest([update], assertions, false); +} + +// Make sure that a nested forward (a forward within a forward) causes +// the update to fail. +function testNestedForward() { + var add1Urls = [ "foo-nested.com/a", "bar-nested.com/c" ]; + var add2Urls = [ "foo-nested.com/b" ]; + + var update = "n:1000\n"; + update += "i:test-phish-simple\n"; + + var update1 = buildBareUpdate( + [{ "chunkNum" : 1, + "urls" : add1Urls }]); + update += "u:data:," + encodeURIComponent(update1) + "\n"; + + var update2 = buildBareUpdate( + [{ "chunkNum" : 2 }]); + var update3 = buildBareUpdate( + [{ "chunkNum" : 3, + "urls" : add1Urls }]); + + update2 += "u:data:," + encodeURIComponent(update3) + "\n"; + + update += "u:data:," + encodeURIComponent(update2) + "\n"; + + var assertions = { + "tableData" : "", + "urlsDontExist" : add1Urls.concat(add2Urls) + }; + + doTest([update], assertions, true); +} + +// An invalid URL forward causes the update to fail. +function testInvalidUrlForward() { + var add1Urls = [ "foo-invalid.com/a", "bar-invalid.com/c" ]; + + var update = buildPhishingUpdate( + [{ "chunkNum" : 1, + "urls" : add1Urls }]); + update += "u:asdf://blah/blah\n"; // invalid URL scheme + + // add1Urls is present, but that is an artifact of the way we do the test. + var assertions = { + "tableData" : "", + "urlsExist" : add1Urls + }; + + doTest([update], assertions, true); +} + +// A failed network request causes the update to fail. +function testErrorUrlForward() { + var add1Urls = [ "foo-forward.com/a", "bar-forward.com/c" ]; + + var update = buildPhishingUpdate( + [{ "chunkNum" : 1, + "urls" : add1Urls }]); + update += "u:http://test.invalid/asdf/asdf\n"; // invalid URL scheme + + // add1Urls is present, but that is an artifact of the way we do the test. + var assertions = { + "tableData" : "", + "urlsExist" : add1Urls + }; + + doTest([update], assertions, true); +} + +function testMultipleTables() { + var add1Urls = [ "foo-multiple.com/a", "bar-multiple.com/c" ]; + var add2Urls = [ "foo-multiple.com/b" ]; + var add3Urls = [ "bar-multiple.com/d" ]; + var add4Urls = [ "bar-multiple.com/e" ]; + var add6Urls = [ "bar-multiple.com/g" ]; + + var update = "n:1000\n"; + update += "i:test-phish-simple\n"; + + var update1 = buildBareUpdate( + [{ "chunkNum" : 1, + "urls" : add1Urls }]); + update += "u:data:," + encodeURIComponent(update1) + "\n"; + + var update2 = buildBareUpdate( + [{ "chunkNum" : 2, + "urls" : add2Urls }]); + update += "u:data:," + encodeURIComponent(update2) + "\n"; + + update += "i:test-malware-simple\n"; + + var update3 = buildBareUpdate( + [{ "chunkNum" : 3, + "urls" : add3Urls }]); + update += "u:data:," + encodeURIComponent(update3) + "\n"; + + update += "i:test-unwanted-simple\n"; + var update4 = buildBareUpdate( + [{ "chunkNum" : 4, + "urls" : add4Urls }]); + update += "u:data:," + encodeURIComponent(update4) + "\n"; + + update += "i:test-block-simple\n"; + var update6 = buildBareUpdate( + [{ "chunkNum" : 6, + "urls" : add6Urls }]); + update += "u:data:," + encodeURIComponent(update6) + "\n"; + + var assertions = { + "tableData" : "test-block-simple;a:6\ntest-malware-simple;a:3\ntest-phish-simple;a:1-2\ntest-unwanted-simple;a:4", + "urlsExist" : add1Urls.concat(add2Urls), + "malwareUrlsExist" : add3Urls, + "unwantedUrlsExist" : add4Urls, + "blockedUrlsExist" : add6Urls + }; + + doTest([update], assertions, false); +} + +function testUrlInMultipleTables() { + var add1Urls = [ "foo-forward.com/a" ]; + + var update = "n:1000\n"; + update += "i:test-phish-simple\n"; + + var update1 = buildBareUpdate( + [{ "chunkNum" : 1, + "urls" : add1Urls }]); + update += "u:data:," + encodeURIComponent(update1) + "\n"; + + update += "i:test-malware-simple\n"; + var update2 = buildBareUpdate( + [{ "chunkNum" : 2, + "urls" : add1Urls }]); + update += "u:data:," + encodeURIComponent(update2) + "\n"; + + update += "i:test-unwanted-simple\n"; + var update3 = buildBareUpdate( + [{ "chunkNum" : 3, + "urls" : add1Urls }]); + update += "u:data:," + encodeURIComponent(update3) + "\n"; + + var assertions = { + "tableData" : "test-malware-simple;a:2\ntest-phish-simple;a:1\ntest-unwanted-simple;a:3", + "urlExistInMultipleTables" : { url: add1Urls, + tables: "test-malware-simple,test-phish-simple,test-unwanted-simple" } + }; + + doTest([update], assertions, false); +} + +function Observer(callback) { + this.observe = callback; +} + +Observer.prototype = +{ +QueryInterface: function(iid) +{ + if (!iid.equals(Ci.nsISupports) && + !iid.equals(Ci.nsIObserver)) { + throw Cr.NS_ERROR_NO_INTERFACE; + } + return this; +} +}; + +// Tests a database reset request. +function testReset() { + // The moz-phish-simple table is populated separately from the other update in + // a separate update request. Therefore it should not be reset when we run the + // updates later in this function. + var mozAddUrls = [ "moz-reset.com/a" ]; + var mozUpdate = buildMozPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : mozAddUrls + }]); + + var dataUpdate = "data:," + encodeURIComponent(mozUpdate); + + streamUpdater.downloadUpdates(mozTables, "", true, + dataUpdate, () => {}, updateError, updateError); + + var addUrls1 = [ "foo-reset.com/a", "foo-reset.com/b" ]; + var update1 = buildPhishingUpdate( + [ + { "chunkNum" : 1, + "urls" : addUrls1 + }]); + + var update2 = "n:1000\nr:pleasereset\n"; + + var addUrls3 = [ "bar-reset.com/a", "bar-reset.com/b" ]; + var update3 = buildPhishingUpdate( + [ + { "chunkNum" : 3, + "urls" : addUrls3 + }]); + + var assertions = { + "tableData" : "moz-phish-simple;a:1\ntest-phish-simple;a:3", // tables that should still be there. + "mozPhishingUrlsExist" : mozAddUrls, // mozAddUrls added prior to the reset + // but it should still exist after reset. + "urlsExist" : addUrls3, // addUrls3 added after the reset. + "urlsDontExist" : addUrls1 // addUrls1 added prior to the reset + }; + + // Use these update responses in order. The update request only + // contains test-*-simple tables so the reset will only apply to these. + doTest([update1, update2, update3], assertions, false); +} + + +function run_test() +{ + runTests([ + testSimpleForward, + testNestedForward, + testInvalidUrlForward, + testErrorUrlForward, + testMultipleTables, + testUrlInMultipleTables, + testReset + ]); +} + +do_test_pending(); diff --git a/toolkit/components/url-classifier/tests/unit/test_threat_type_conversion.js b/toolkit/components/url-classifier/tests/unit/test_threat_type_conversion.js new file mode 100644 index 000000000..f7c51b956 --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/test_threat_type_conversion.js @@ -0,0 +1,37 @@ +function run_test() { + let urlUtils = Cc["@mozilla.org/url-classifier/utils;1"] + .getService(Ci.nsIUrlClassifierUtils); + + // Test list name to threat type conversion. + + equal(urlUtils.convertListNameToThreatType("goog-malware-proto"), 1); + equal(urlUtils.convertListNameToThreatType("googpub-phish-proto"), 2); + equal(urlUtils.convertListNameToThreatType("goog-unwanted-proto"), 3); + equal(urlUtils.convertListNameToThreatType("goog-phish-proto"), 5); + + try { + urlUtils.convertListNameToThreatType("bad-list-name"); + ok(false, "Bad list name should lead to exception."); + } catch (e) {} + + try { + urlUtils.convertListNameToThreatType("bad-list-name"); + ok(false, "Bad list name should lead to exception."); + } catch (e) {} + + // Test threat type to list name conversion. + equal(urlUtils.convertThreatTypeToListNames(1), "goog-malware-proto"); + equal(urlUtils.convertThreatTypeToListNames(2), "googpub-phish-proto,test-phish-proto"); + equal(urlUtils.convertThreatTypeToListNames(3), "goog-unwanted-proto,test-unwanted-proto"); + equal(urlUtils.convertThreatTypeToListNames(5), "goog-phish-proto"); + + try { + urlUtils.convertThreatTypeToListNames(0); + ok(false, "Bad threat type should lead to exception."); + } catch (e) {} + + try { + urlUtils.convertThreatTypeToListNames(100); + ok(false, "Bad threat type should lead to exception."); + } catch (e) {} +}
\ No newline at end of file diff --git a/toolkit/components/url-classifier/tests/unit/xpcshell.ini b/toolkit/components/url-classifier/tests/unit/xpcshell.ini new file mode 100644 index 000000000..c34d575c6 --- /dev/null +++ b/toolkit/components/url-classifier/tests/unit/xpcshell.ini @@ -0,0 +1,24 @@ +[DEFAULT] +head = head_urlclassifier.js +tail = tail_urlclassifier.js +skip-if = toolkit == 'android' +support-files = + data/digest1.chunk + data/digest2.chunk + +[test_addsub.js] +[test_bug1274685_unowned_list.js] +[test_backoff.js] +[test_dbservice.js] +[test_hashcompleter.js] +# Bug 752243: Profile cleanup frequently fails +#skip-if = os == "mac" || os == "linux" +[test_partial.js] +[test_prefixset.js] +[test_threat_type_conversion.js] +[test_provider_url.js] +[test_streamupdater.js] +[test_digest256.js] +[test_listmanager.js] +[test_pref.js] +[test_safebrowsing_protobuf.js] diff --git a/toolkit/components/url-classifier/tests/unittests.xul b/toolkit/components/url-classifier/tests/unittests.xul new file mode 100644 index 000000000..0c9ce898b --- /dev/null +++ b/toolkit/components/url-classifier/tests/unittests.xul @@ -0,0 +1,188 @@ +<?xml version="1.0"?> +<window id="PROT_unittest" + xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + onload="onProtUnittestLoad();" + title="prot unittests"> + +<script><![CDATA[ + const Cc = Components.classes; + const Ci = Components.interfaces; + + function G_Debug(zone, s) { + var label = document.createElement('label'); + var txt = "[" + zone + "] " + s; + label.appendChild(document.createTextNode(txt)); + + document.documentElement.appendChild(label); + } + + function G_Assert(zone, cond, msg) { + if (!cond) { + G_Debug(zone, msg); + throw msg; + } + } + + function ProtectionTableTests() { + var z = "trtable UNITTEST"; + + G_Debug(z, "Starting"); + + var url = "http://www.yahoo.com?foo=bar"; + var url2 = "http://168.188.99.26/.secure/www.ebay.com/"; + var urlTable = Cc['@mozilla.org/url-classifier/table;1?type=url'] + .createInstance(Ci.nsIUrlClassifierTable); + urlTable.insert(url, "1"); + urlTable.insert(url2, "1"); + G_Assert(z, urlTable.exists(url), "URL lookups broken"); + G_Assert(z, !urlTable.exists("about:config"), "about:config breaks domlook"); + G_Assert(z, urlTable.exists(url2), "URL lookups broken"); + G_Assert(z, urlTable.exists("http://%31%36%38%2e%31%38%38%2e%39%39%2e%32%36/%2E%73%65%63%75%72%65/%77%77%77%2E%65%62%61%79%2E%63%6F%6D/") == true, + "URL Canonicalization broken"); + G_Assert(z, urlTable.count == 2, 'urlTable: wrong size'); + + var dom1 = "bar.com"; + var dom2 = "amazon.co.uk"; + var dom3 = "127.0.0.1"; + var domainTable = Cc['@mozilla.org/url-classifier/table;1?type=domain'] + .createInstance(Ci.nsIUrlClassifierTable); + domainTable.insert(dom1, "1"); + domainTable.insert(dom2, "1"); + domainTable.insert(dom3, "1"); + G_Assert(z, domainTable.exists("http://www.bar.com/?zaz=asdf#url"), + "Domain lookups broken (single dot)"); + G_Assert(z, domainTable.exists("http://www.amazon.co.uk/?z=af#url"), + "Domain lookups broken (two dots)"); + G_Assert(z, domainTable.exists("http://127.0.0.1/?z=af#url"), + "Domain lookups broken (IP)"); + G_Assert(z, domainTable.count == 3, 'domainTable: wrong size'); + + var site1 = "google.com/safebrowsing/"; + var site2 = "www.foo.bar/"; + var site3 = "127.0.0.1/"; + var siteTable = Cc['@mozilla.org/url-classifier/table;1?type=site'] + .createInstance(Ci.nsIUrlClassifierTable); + siteTable.insert(site1, "1"); + siteTable.insert(site2, "1"); + siteTable.insert(site3, "1"); + G_Assert(z, siteTable.exists("http://www.google.com/safebrowsing/1.php"), + "Site lookups broken - reducing"); + G_Assert(z, siteTable.exists("http://www.foo.bar/some/random/path"), + "Site lookups broken - fqdn"); + G_Assert(z, siteTable.exists("http://127.0.0.1/something?hello=1"), + "Site lookups broken - IP"); + G_Assert(z, !siteTable.exists("http://www.google.com/search/"), + "Site lookups broken - overreaching"); + G_Assert(z, siteTable.count == 3, 'siteTable: wrong size'); + + var url1 = "http://poseidon.marinet.gr/~eleni/eBay/index.php"; + var domainHash = "01844755C8143C4579BB28DD59C23747"; + var enchashTable = Cc['@mozilla.org/url-classifier/table;1?type=enchash'] + .createInstance(Ci.nsIUrlClassifierTable); + enchashTable.insert(domainHash, "bGtEQWJuMl9FA3Kl5RiXMpgFU8nDJl9J0hXjUck9+" + + "mMUQwAN6llf0gJeY5DIPPc2f+a8MSBFJN17ANGJ" + + "Zl5oZVsQfSW4i12rlScsx4tweZAE"); + G_Assert(z, enchashTable.exists(url1), 'enchash lookup failed'); + G_Assert(z, !enchashTable.exists(url1 + '/foo'), + "enchash lookup broken - overreaching"); + G_Assert(z, enchashTable.count == 1, 'enchashTable: wrong size'); + + // TODO: test replace + G_Debug(z, "PASSED"); + } + + function ProtectionListManagerTests() { + var z = "listmanager UNITTEST"; + G_Debug(z, "Starting"); + + // test lookup and register + var listManagerInst = Cc["@mozilla.org/url-classifier/listmanager;1"] + .createInstance(Ci.nsIUrlListManager); + var listName = 'foo-bar-url'; + listManagerInst.registerTable(listName, false); + listManagerInst.safeInsert(listName, 'test', '1'); + G_Assert(z, listManagerInst.safeExists(listName, 'test'), + 'insert/exist failed'); + + // test serialization + var baseName = (new Date().getTime()) + ".tmp"; + var tempDir = Cc["@mozilla.org/file/directory_service;1"] + .getService(Ci.nsIProperties) + .get("TmpD", Ci.nsILocalFile); + tempDir.append(baseName); + tempDir.createUnique(tempDir.DIRECTORY_TYPE, 0744); + + var listManager = Cc["@mozilla.org/url-classifier/listmanager;1"] + .getService(Ci.nsIUrlListManager); + listManager.setAppDir(tempDir); + + var data = ""; + + var set1Name = "test1-foo-domain"; + data += "[" + set1Name + " 1.2]\n"; + var set1 = {}; + for (var i = 0; i < 10; i++) { + set1["http://" + i + ".com"] = 1; + data += "+" + i + ".com\t1\n"; + } + + data += "\n"; + var set2Name = "test2-foo-domain"; + // TODO must have blank line + data += "\n[" + set2Name + " 1.7]\n"; + var set2 = {}; + for (var i = 0; i < 5; i++) { + set2["http://" + i + ".com"] = 1; + data += "+" + i + ".com\t1\n"; + } + + function deserialized(tablesKnown, tablesData) { + listManager.wrappedJSObject.dataReady(tablesKnown, tablesData); + + var file = tempDir.clone(); + file.append(set1Name + ".sst"); + G_Assert(z, file.exists() && file.isFile() && file.isReadable(), + "Failed to write out: " + file.path); + + file = tempDir.clone(); + file.append(set2Name + ".sst"); + G_Assert(z, file.exists() && file.isFile() && file.isReadable(), + "Failed to write out: " + file.path); + + // now try to read them back from disk + listManager = Cc["@mozilla.org/url-classifier/listmanager;1"] + .createInstance(Ci.nsIUrlListManager); + listManager.setAppDir(tempDir); + var tables = [ set1Name, set2Name ]; + listManager.enableUpdate(set1Name); + listManager.enableUpdate(set2Name); + listManager.wrappedJSObject.readDataFiles(); + + // assert that the values match + for (var prop in set1) { + G_Assert(z, + listManager.wrappedJSObject.tablesData[set1Name].exists(prop), + "Couldn't find member " + prop + "of set1 from disk."); + } + + for (var prop in set2) { + G_Assert(z, + listManager.wrappedJSObject.tablesData[set2Name].exists(prop), + "Couldn't find member " + prop + "of set2 from disk."); + } + + tempDir.remove(true); + + G_Debug(z, "PASSED"); + }; + + // Use the unwrapped object for the unittest + listManager.wrappedJSObject.deserialize_(data, deserialized); + } + + function onProtUnittestLoad() { + ProtectionTableTests(); + ProtectionListManagerTests(); + } +]]></script> +</window> |
