diff options
Diffstat (limited to 'toolkit/components/url-classifier/ProtocolParser.cpp')
| -rw-r--r-- | toolkit/components/url-classifier/ProtocolParser.cpp | 1108 |
1 files changed, 1108 insertions, 0 deletions
diff --git a/toolkit/components/url-classifier/ProtocolParser.cpp b/toolkit/components/url-classifier/ProtocolParser.cpp new file mode 100644 index 000000000..5da7787be --- /dev/null +++ b/toolkit/components/url-classifier/ProtocolParser.cpp @@ -0,0 +1,1108 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ProtocolParser.h" +#include "LookupCache.h" +#include "nsNetCID.h" +#include "mozilla/Logging.h" +#include "prnetdb.h" +#include "prprf.h" + +#include "nsUrlClassifierDBService.h" +#include "nsUrlClassifierUtils.h" +#include "nsPrintfCString.h" +#include "mozilla/Base64.h" +#include "RiceDeltaDecoder.h" +#include "mozilla/EndianUtils.h" + +// MOZ_LOG=UrlClassifierProtocolParser:5 +mozilla::LazyLogModule gUrlClassifierProtocolParserLog("UrlClassifierProtocolParser"); +#define PARSER_LOG(args) MOZ_LOG(gUrlClassifierProtocolParserLog, mozilla::LogLevel::Debug, args) + +namespace mozilla { +namespace safebrowsing { + +// Updates will fail if fed chunks larger than this +const uint32_t MAX_CHUNK_SIZE = (1024 * 1024); +// Updates will fail if the total number of tocuhed chunks is larger than this +const uint32_t MAX_CHUNK_RANGE = 1000000; + +const uint32_t DOMAIN_SIZE = 4; + +// Parse one stringified range of chunks of the form "n" or "n-m" from a +// comma-separated list of chunks. Upon return, 'begin' will point to the +// next range of chunks in the list of chunks. +static bool +ParseChunkRange(nsACString::const_iterator& aBegin, + const nsACString::const_iterator& aEnd, + uint32_t* aFirst, uint32_t* aLast) +{ + nsACString::const_iterator iter = aBegin; + FindCharInReadable(',', iter, aEnd); + + nsAutoCString element(Substring(aBegin, iter)); + aBegin = iter; + if (aBegin != aEnd) + aBegin++; + + uint32_t numRead = PR_sscanf(element.get(), "%u-%u", aFirst, aLast); + if (numRead == 2) { + if (*aFirst > *aLast) { + uint32_t tmp = *aFirst; + *aFirst = *aLast; + *aLast = tmp; + } + return true; + } + + if (numRead == 1) { + *aLast = *aFirst; + return true; + } + + return false; +} + +/////////////////////////////////////////////////////////////// +// ProtocolParser implementation + +ProtocolParser::ProtocolParser() + : mUpdateStatus(NS_OK) + , mUpdateWaitSec(0) +{ +} + +ProtocolParser::~ProtocolParser() +{ + CleanupUpdates(); +} + +nsresult +ProtocolParser::Init(nsICryptoHash* aHasher) +{ + mCryptoHash = aHasher; + return NS_OK; +} + +void +ProtocolParser::CleanupUpdates() +{ + for (uint32_t i = 0; i < mTableUpdates.Length(); i++) { + delete mTableUpdates[i]; + } + mTableUpdates.Clear(); +} + +TableUpdate * +ProtocolParser::GetTableUpdate(const nsACString& aTable) +{ + for (uint32_t i = 0; i < mTableUpdates.Length(); i++) { + if (aTable.Equals(mTableUpdates[i]->TableName())) { + return mTableUpdates[i]; + } + } + + // We free automatically on destruction, ownership of these + // updates can be transferred to DBServiceWorker, which passes + // them back to Classifier when doing the updates, and that + // will free them. + TableUpdate *update = CreateTableUpdate(aTable); + mTableUpdates.AppendElement(update); + return update; +} + +/////////////////////////////////////////////////////////////////////// +// ProtocolParserV2 + +ProtocolParserV2::ProtocolParserV2() + : mState(PROTOCOL_STATE_CONTROL) + , mResetRequested(false) + , mTableUpdate(nullptr) +{ +} + +ProtocolParserV2::~ProtocolParserV2() +{ +} + +void +ProtocolParserV2::SetCurrentTable(const nsACString& aTable) +{ + auto update = GetTableUpdate(aTable); + mTableUpdate = TableUpdate::Cast<TableUpdateV2>(update); +} + +nsresult +ProtocolParserV2::AppendStream(const nsACString& aData) +{ + if (NS_FAILED(mUpdateStatus)) + return mUpdateStatus; + + nsresult rv; + mPending.Append(aData); + + bool done = false; + while (!done) { + if (nsUrlClassifierDBService::ShutdownHasStarted()) { + return NS_ERROR_ABORT; + } + + if (mState == PROTOCOL_STATE_CONTROL) { + rv = ProcessControl(&done); + } else if (mState == PROTOCOL_STATE_CHUNK) { + rv = ProcessChunk(&done); + } else { + NS_ERROR("Unexpected protocol state"); + rv = NS_ERROR_FAILURE; + } + if (NS_FAILED(rv)) { + mUpdateStatus = rv; + return rv; + } + } + return NS_OK; +} + +void +ProtocolParserV2::End() +{ + // Inbound data has already been processed in every AppendStream() call. +} + +nsresult +ProtocolParserV2::ProcessControl(bool* aDone) +{ + nsresult rv; + + nsAutoCString line; + *aDone = true; + while (NextLine(line)) { + PARSER_LOG(("Processing %s\n", line.get())); + + if (StringBeginsWith(line, NS_LITERAL_CSTRING("i:"))) { + // Set the table name from the table header line. + SetCurrentTable(Substring(line, 2)); + } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("n:"))) { + if (PR_sscanf(line.get(), "n:%d", &mUpdateWaitSec) != 1) { + PARSER_LOG(("Error parsing n: '%s' (%d)", line.get(), mUpdateWaitSec)); + return NS_ERROR_FAILURE; + } + } else if (line.EqualsLiteral("r:pleasereset")) { + mResetRequested = true; + } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("u:"))) { + rv = ProcessForward(line); + NS_ENSURE_SUCCESS(rv, rv); + } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("a:")) || + StringBeginsWith(line, NS_LITERAL_CSTRING("s:"))) { + rv = ProcessChunkControl(line); + NS_ENSURE_SUCCESS(rv, rv); + *aDone = false; + return NS_OK; + } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("ad:")) || + StringBeginsWith(line, NS_LITERAL_CSTRING("sd:"))) { + rv = ProcessExpirations(line); + NS_ENSURE_SUCCESS(rv, rv); + } + } + + *aDone = true; + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessExpirations(const nsCString& aLine) +{ + if (!mTableUpdate) { + NS_WARNING("Got an expiration without a table."); + return NS_ERROR_FAILURE; + } + const nsCSubstring &list = Substring(aLine, 3); + nsACString::const_iterator begin, end; + list.BeginReading(begin); + list.EndReading(end); + while (begin != end) { + uint32_t first, last; + if (ParseChunkRange(begin, end, &first, &last)) { + if (last < first) return NS_ERROR_FAILURE; + if (last - first > MAX_CHUNK_RANGE) return NS_ERROR_FAILURE; + for (uint32_t num = first; num <= last; num++) { + if (aLine[0] == 'a') { + nsresult rv = mTableUpdate->NewAddExpiration(num); + if (NS_FAILED(rv)) { + return rv; + } + } else { + nsresult rv = mTableUpdate->NewSubExpiration(num); + if (NS_FAILED(rv)) { + return rv; + } + } + } + } else { + return NS_ERROR_FAILURE; + } + } + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessChunkControl(const nsCString& aLine) +{ + if (!mTableUpdate) { + NS_WARNING("Got a chunk before getting a table."); + return NS_ERROR_FAILURE; + } + + mState = PROTOCOL_STATE_CHUNK; + char command; + + mChunkState.Clear(); + + if (PR_sscanf(aLine.get(), + "%c:%d:%d:%d", + &command, + &mChunkState.num, &mChunkState.hashSize, &mChunkState.length) + != 4) + { + NS_WARNING(("PR_sscanf failed")); + return NS_ERROR_FAILURE; + } + + if (mChunkState.length > MAX_CHUNK_SIZE) { + NS_WARNING("Invalid length specified in update."); + return NS_ERROR_FAILURE; + } + + if (!(mChunkState.hashSize == PREFIX_SIZE || mChunkState.hashSize == COMPLETE_SIZE)) { + NS_WARNING("Invalid hash size specified in update."); + return NS_ERROR_FAILURE; + } + + if (StringEndsWith(mTableUpdate->TableName(), + NS_LITERAL_CSTRING("-shavar")) || + StringEndsWith(mTableUpdate->TableName(), + NS_LITERAL_CSTRING("-simple"))) { + // Accommodate test tables ending in -simple for now. + mChunkState.type = (command == 'a') ? CHUNK_ADD : CHUNK_SUB; + } else if (StringEndsWith(mTableUpdate->TableName(), + NS_LITERAL_CSTRING("-digest256"))) { + mChunkState.type = (command == 'a') ? CHUNK_ADD_DIGEST : CHUNK_SUB_DIGEST; + } + nsresult rv; + switch (mChunkState.type) { + case CHUNK_ADD: + rv = mTableUpdate->NewAddChunk(mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + break; + case CHUNK_SUB: + rv = mTableUpdate->NewSubChunk(mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + break; + case CHUNK_ADD_DIGEST: + rv = mTableUpdate->NewAddChunk(mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + break; + case CHUNK_SUB_DIGEST: + rv = mTableUpdate->NewSubChunk(mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + break; + } + + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessForward(const nsCString& aLine) +{ + const nsCSubstring &forward = Substring(aLine, 2); + return AddForward(forward); +} + +nsresult +ProtocolParserV2::AddForward(const nsACString& aUrl) +{ + if (!mTableUpdate) { + NS_WARNING("Forward without a table name."); + return NS_ERROR_FAILURE; + } + + ForwardedUpdate *forward = mForwards.AppendElement(); + forward->table = mTableUpdate->TableName(); + forward->url.Assign(aUrl); + + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessChunk(bool* aDone) +{ + if (!mTableUpdate) { + NS_WARNING("Processing chunk without an active table."); + return NS_ERROR_FAILURE; + } + + NS_ASSERTION(mChunkState.num != 0, "Must have a chunk number."); + + if (mPending.Length() < mChunkState.length) { + *aDone = true; + return NS_OK; + } + + // Pull the chunk out of the pending stream data. + nsAutoCString chunk; + chunk.Assign(Substring(mPending, 0, mChunkState.length)); + mPending.Cut(0, mChunkState.length); + + *aDone = false; + mState = PROTOCOL_STATE_CONTROL; + + if (StringEndsWith(mTableUpdate->TableName(), + NS_LITERAL_CSTRING("-shavar"))) { + return ProcessShaChunk(chunk); + } + if (StringEndsWith(mTableUpdate->TableName(), + NS_LITERAL_CSTRING("-digest256"))) { + return ProcessDigestChunk(chunk); + } + return ProcessPlaintextChunk(chunk); +} + +/** + * Process a plaintext chunk (currently only used in unit tests). + */ +nsresult +ProtocolParserV2::ProcessPlaintextChunk(const nsACString& aChunk) +{ + if (!mTableUpdate) { + NS_WARNING("Chunk received with no table."); + return NS_ERROR_FAILURE; + } + + PARSER_LOG(("Handling a %d-byte simple chunk", aChunk.Length())); + + nsTArray<nsCString> lines; + ParseString(PromiseFlatCString(aChunk), '\n', lines); + + // non-hashed tables need to be hashed + for (uint32_t i = 0; i < lines.Length(); i++) { + nsCString& line = lines[i]; + + if (mChunkState.type == CHUNK_ADD) { + if (mChunkState.hashSize == COMPLETE_SIZE) { + Completion hash; + hash.FromPlaintext(line, mCryptoHash); + nsresult rv = mTableUpdate->NewAddComplete(mChunkState.num, hash); + if (NS_FAILED(rv)) { + return rv; + } + } else { + NS_ASSERTION(mChunkState.hashSize == 4, "Only 32- or 4-byte hashes can be used for add chunks."); + Prefix hash; + hash.FromPlaintext(line, mCryptoHash); + nsresult rv = mTableUpdate->NewAddPrefix(mChunkState.num, hash); + if (NS_FAILED(rv)) { + return rv; + } + } + } else { + nsCString::const_iterator begin, iter, end; + line.BeginReading(begin); + line.EndReading(end); + iter = begin; + uint32_t addChunk; + if (!FindCharInReadable(':', iter, end) || + PR_sscanf(lines[i].get(), "%d:", &addChunk) != 1) { + NS_WARNING("Received sub chunk without associated add chunk."); + return NS_ERROR_FAILURE; + } + iter++; + + if (mChunkState.hashSize == COMPLETE_SIZE) { + Completion hash; + hash.FromPlaintext(Substring(iter, end), mCryptoHash); + nsresult rv = mTableUpdate->NewSubComplete(addChunk, hash, mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + } else { + NS_ASSERTION(mChunkState.hashSize == 4, "Only 32- or 4-byte hashes can be used for add chunks."); + Prefix hash; + hash.FromPlaintext(Substring(iter, end), mCryptoHash); + nsresult rv = mTableUpdate->NewSubPrefix(addChunk, hash, mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + } + } + } + + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessShaChunk(const nsACString& aChunk) +{ + uint32_t start = 0; + while (start < aChunk.Length()) { + // First four bytes are the domain key. + Prefix domain; + domain.Assign(Substring(aChunk, start, DOMAIN_SIZE)); + start += DOMAIN_SIZE; + + // Then a count of entries. + uint8_t numEntries = static_cast<uint8_t>(aChunk[start]); + start++; + + PARSER_LOG(("Handling a %d-byte shavar chunk containing %u entries" + " for domain %X", aChunk.Length(), numEntries, + domain.ToUint32())); + + nsresult rv; + if (mChunkState.type == CHUNK_ADD && mChunkState.hashSize == PREFIX_SIZE) { + rv = ProcessHostAdd(domain, numEntries, aChunk, &start); + } else if (mChunkState.type == CHUNK_ADD && mChunkState.hashSize == COMPLETE_SIZE) { + rv = ProcessHostAddComplete(numEntries, aChunk, &start); + } else if (mChunkState.type == CHUNK_SUB && mChunkState.hashSize == PREFIX_SIZE) { + rv = ProcessHostSub(domain, numEntries, aChunk, &start); + } else if (mChunkState.type == CHUNK_SUB && mChunkState.hashSize == COMPLETE_SIZE) { + rv = ProcessHostSubComplete(numEntries, aChunk, &start); + } else { + NS_WARNING("Unexpected chunk type/hash size!"); + PARSER_LOG(("Got an unexpected chunk type/hash size: %s:%d", + mChunkState.type == CHUNK_ADD ? "add" : "sub", + mChunkState.hashSize)); + return NS_ERROR_FAILURE; + } + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessDigestChunk(const nsACString& aChunk) +{ + PARSER_LOG(("Handling a %d-byte digest256 chunk", aChunk.Length())); + + if (mChunkState.type == CHUNK_ADD_DIGEST) { + return ProcessDigestAdd(aChunk); + } + if (mChunkState.type == CHUNK_SUB_DIGEST) { + return ProcessDigestSub(aChunk); + } + return NS_ERROR_UNEXPECTED; +} + +nsresult +ProtocolParserV2::ProcessDigestAdd(const nsACString& aChunk) +{ + // The ABNF format for add chunks is (HASH)+, where HASH is 32 bytes. + MOZ_ASSERT(aChunk.Length() % 32 == 0, + "Chunk length in bytes must be divisible by 4"); + uint32_t start = 0; + while (start < aChunk.Length()) { + Completion hash; + hash.Assign(Substring(aChunk, start, COMPLETE_SIZE)); + start += COMPLETE_SIZE; + nsresult rv = mTableUpdate->NewAddComplete(mChunkState.num, hash); + if (NS_FAILED(rv)) { + return rv; + } + } + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessDigestSub(const nsACString& aChunk) +{ + // The ABNF format for sub chunks is (ADDCHUNKNUM HASH)+, where ADDCHUNKNUM + // is a 4 byte chunk number, and HASH is 32 bytes. + MOZ_ASSERT(aChunk.Length() % 36 == 0, + "Chunk length in bytes must be divisible by 36"); + uint32_t start = 0; + while (start < aChunk.Length()) { + // Read ADDCHUNKNUM + const nsCSubstring& addChunkStr = Substring(aChunk, start, 4); + start += 4; + + uint32_t addChunk; + memcpy(&addChunk, addChunkStr.BeginReading(), 4); + addChunk = PR_ntohl(addChunk); + + // Read the hash + Completion hash; + hash.Assign(Substring(aChunk, start, COMPLETE_SIZE)); + start += COMPLETE_SIZE; + + nsresult rv = mTableUpdate->NewSubComplete(addChunk, hash, mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + } + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessHostAdd(const Prefix& aDomain, uint8_t aNumEntries, + const nsACString& aChunk, uint32_t* aStart) +{ + NS_ASSERTION(mChunkState.hashSize == PREFIX_SIZE, + "ProcessHostAdd should only be called for prefix hashes."); + + if (aNumEntries == 0) { + nsresult rv = mTableUpdate->NewAddPrefix(mChunkState.num, aDomain); + if (NS_FAILED(rv)) { + return rv; + } + return NS_OK; + } + + if (*aStart + (PREFIX_SIZE * aNumEntries) > aChunk.Length()) { + NS_WARNING("Chunk is not long enough to contain the expected entries."); + return NS_ERROR_FAILURE; + } + + for (uint8_t i = 0; i < aNumEntries; i++) { + Prefix hash; + hash.Assign(Substring(aChunk, *aStart, PREFIX_SIZE)); + PARSER_LOG(("Add prefix %X", hash.ToUint32())); + nsresult rv = mTableUpdate->NewAddPrefix(mChunkState.num, hash); + if (NS_FAILED(rv)) { + return rv; + } + *aStart += PREFIX_SIZE; + } + + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessHostSub(const Prefix& aDomain, uint8_t aNumEntries, + const nsACString& aChunk, uint32_t *aStart) +{ + NS_ASSERTION(mChunkState.hashSize == PREFIX_SIZE, + "ProcessHostSub should only be called for prefix hashes."); + + if (aNumEntries == 0) { + if ((*aStart) + 4 > aChunk.Length()) { + NS_WARNING("Received a zero-entry sub chunk without an associated add."); + return NS_ERROR_FAILURE; + } + + const nsCSubstring& addChunkStr = Substring(aChunk, *aStart, 4); + *aStart += 4; + + uint32_t addChunk; + memcpy(&addChunk, addChunkStr.BeginReading(), 4); + addChunk = PR_ntohl(addChunk); + + PARSER_LOG(("Sub prefix (addchunk=%u)", addChunk)); + nsresult rv = mTableUpdate->NewSubPrefix(addChunk, aDomain, mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + return NS_OK; + } + + if (*aStart + ((PREFIX_SIZE + 4) * aNumEntries) > aChunk.Length()) { + NS_WARNING("Chunk is not long enough to contain the expected entries."); + return NS_ERROR_FAILURE; + } + + for (uint8_t i = 0; i < aNumEntries; i++) { + const nsCSubstring& addChunkStr = Substring(aChunk, *aStart, 4); + *aStart += 4; + + uint32_t addChunk; + memcpy(&addChunk, addChunkStr.BeginReading(), 4); + addChunk = PR_ntohl(addChunk); + + Prefix prefix; + prefix.Assign(Substring(aChunk, *aStart, PREFIX_SIZE)); + *aStart += PREFIX_SIZE; + + PARSER_LOG(("Sub prefix %X (addchunk=%u)", prefix.ToUint32(), addChunk)); + nsresult rv = mTableUpdate->NewSubPrefix(addChunk, prefix, mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + } + + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessHostAddComplete(uint8_t aNumEntries, + const nsACString& aChunk, uint32_t* aStart) +{ + NS_ASSERTION(mChunkState.hashSize == COMPLETE_SIZE, + "ProcessHostAddComplete should only be called for complete hashes."); + + if (aNumEntries == 0) { + // this is totally comprehensible. + // My sarcasm detector is going off! + NS_WARNING("Expected > 0 entries for a 32-byte hash add."); + return NS_OK; + } + + if (*aStart + (COMPLETE_SIZE * aNumEntries) > aChunk.Length()) { + NS_WARNING("Chunk is not long enough to contain the expected entries."); + return NS_ERROR_FAILURE; + } + + for (uint8_t i = 0; i < aNumEntries; i++) { + Completion hash; + hash.Assign(Substring(aChunk, *aStart, COMPLETE_SIZE)); + nsresult rv = mTableUpdate->NewAddComplete(mChunkState.num, hash); + if (NS_FAILED(rv)) { + return rv; + } + *aStart += COMPLETE_SIZE; + } + + return NS_OK; +} + +nsresult +ProtocolParserV2::ProcessHostSubComplete(uint8_t aNumEntries, + const nsACString& aChunk, uint32_t* aStart) +{ + NS_ASSERTION(mChunkState.hashSize == COMPLETE_SIZE, + "ProcessHostSubComplete should only be called for complete hashes."); + + if (aNumEntries == 0) { + // this is totally comprehensible. + NS_WARNING("Expected > 0 entries for a 32-byte hash sub."); + return NS_OK; + } + + if (*aStart + ((COMPLETE_SIZE + 4) * aNumEntries) > aChunk.Length()) { + NS_WARNING("Chunk is not long enough to contain the expected entries."); + return NS_ERROR_FAILURE; + } + + for (uint8_t i = 0; i < aNumEntries; i++) { + Completion hash; + hash.Assign(Substring(aChunk, *aStart, COMPLETE_SIZE)); + *aStart += COMPLETE_SIZE; + + const nsCSubstring& addChunkStr = Substring(aChunk, *aStart, 4); + *aStart += 4; + + uint32_t addChunk; + memcpy(&addChunk, addChunkStr.BeginReading(), 4); + addChunk = PR_ntohl(addChunk); + + nsresult rv = mTableUpdate->NewSubComplete(addChunk, hash, mChunkState.num); + if (NS_FAILED(rv)) { + return rv; + } + } + + return NS_OK; +} + +bool +ProtocolParserV2::NextLine(nsACString& aLine) +{ + int32_t newline = mPending.FindChar('\n'); + if (newline == kNotFound) { + return false; + } + aLine.Assign(Substring(mPending, 0, newline)); + mPending.Cut(0, newline + 1); + return true; +} + +TableUpdate* +ProtocolParserV2::CreateTableUpdate(const nsACString& aTableName) const +{ + return new TableUpdateV2(aTableName); +} + +/////////////////////////////////////////////////////////////////////// +// ProtocolParserProtobuf + +ProtocolParserProtobuf::ProtocolParserProtobuf() +{ +} + +ProtocolParserProtobuf::~ProtocolParserProtobuf() +{ +} + +void +ProtocolParserProtobuf::SetCurrentTable(const nsACString& aTable) +{ + // Should never occur. + MOZ_ASSERT_UNREACHABLE("SetCurrentTable shouldn't be called"); +} + + +TableUpdate* +ProtocolParserProtobuf::CreateTableUpdate(const nsACString& aTableName) const +{ + return new TableUpdateV4(aTableName); +} + +nsresult +ProtocolParserProtobuf::AppendStream(const nsACString& aData) +{ + // Protobuf data cannot be parsed progressively. Just save the incoming data. + mPending.Append(aData); + return NS_OK; +} + +void +ProtocolParserProtobuf::End() +{ + // mUpdateStatus will be updated to success as long as not all + // the responses are invalid. + mUpdateStatus = NS_ERROR_FAILURE; + + FetchThreatListUpdatesResponse response; + if (!response.ParseFromArray(mPending.get(), mPending.Length())) { + NS_WARNING("ProtocolParserProtobuf failed parsing data."); + return; + } + + auto minWaitDuration = response.minimum_wait_duration(); + mUpdateWaitSec = minWaitDuration.seconds() + + minWaitDuration.nanos() / 1000000000; + + for (int i = 0; i < response.list_update_responses_size(); i++) { + auto r = response.list_update_responses(i); + nsresult rv = ProcessOneResponse(r); + if (NS_SUCCEEDED(rv)) { + mUpdateStatus = rv; + } else { + NS_WARNING("Failed to process one response."); + } + } +} + +nsresult +ProtocolParserProtobuf::ProcessOneResponse(const ListUpdateResponse& aResponse) +{ + // A response must have a threat type. + if (!aResponse.has_threat_type()) { + NS_WARNING("Threat type not initialized. This seems to be an invalid response."); + return NS_ERROR_FAILURE; + } + + // Convert threat type to list name. + nsCOMPtr<nsIUrlClassifierUtils> urlUtil = + do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID); + nsCString possibleListNames; + nsresult rv = urlUtil->ConvertThreatTypeToListNames(aResponse.threat_type(), + possibleListNames); + if (NS_FAILED(rv)) { + PARSER_LOG((nsPrintfCString("Threat type to list name conversion error: %d", + aResponse.threat_type())).get()); + return NS_ERROR_FAILURE; + } + + // Match the table name we received with one of the ones we requested. + // We ignore the case where a threat type matches more than one list + // per provider and return the first one. See bug 1287059." + nsCString listName; + nsTArray<nsCString> possibleListNameArray; + Classifier::SplitTables(possibleListNames, possibleListNameArray); + for (auto possibleName : possibleListNameArray) { + if (mRequestedTables.Contains(possibleName)) { + listName = possibleName; + break; + } + } + + if (listName.IsEmpty()) { + PARSER_LOG(("We received an update for a list we didn't ask for. Ignoring it.")); + return NS_ERROR_FAILURE; + } + + // Test if this is a full update. + bool isFullUpdate = false; + if (aResponse.has_response_type()) { + isFullUpdate = + aResponse.response_type() == ListUpdateResponse::FULL_UPDATE; + } else { + NS_WARNING("Response type not initialized."); + return NS_ERROR_FAILURE; + } + + // Warn if there's no new state. + if (!aResponse.has_new_client_state()) { + NS_WARNING("New state not initialized."); + return NS_ERROR_FAILURE; + } + + auto tu = GetTableUpdate(nsCString(listName.get())); + auto tuV4 = TableUpdate::Cast<TableUpdateV4>(tu); + NS_ENSURE_TRUE(tuV4, NS_ERROR_FAILURE); + + nsCString state(aResponse.new_client_state().c_str(), + aResponse.new_client_state().size()); + tuV4->SetNewClientState(state); + + if (aResponse.has_checksum()) { + tuV4->NewChecksum(aResponse.checksum().sha256()); + } + + PARSER_LOG(("==== Update for threat type '%d' ====", aResponse.threat_type())); + PARSER_LOG(("* listName: %s\n", listName.get())); + PARSER_LOG(("* newState: %s\n", aResponse.new_client_state().c_str())); + PARSER_LOG(("* isFullUpdate: %s\n", (isFullUpdate ? "yes" : "no"))); + PARSER_LOG(("* hasChecksum: %s\n", (aResponse.has_checksum() ? "yes" : "no"))); + + tuV4->SetFullUpdate(isFullUpdate); + + rv = ProcessAdditionOrRemoval(*tuV4, aResponse.additions(), true /*aIsAddition*/); + NS_ENSURE_SUCCESS(rv, rv); + rv = ProcessAdditionOrRemoval(*tuV4, aResponse.removals(), false); + NS_ENSURE_SUCCESS(rv, rv); + + PARSER_LOG(("\n\n")); + + return NS_OK; +} + +nsresult +ProtocolParserProtobuf::ProcessAdditionOrRemoval(TableUpdateV4& aTableUpdate, + const ThreatEntrySetList& aUpdate, + bool aIsAddition) +{ + nsresult ret = NS_OK; + + for (int i = 0; i < aUpdate.size(); i++) { + auto update = aUpdate.Get(i); + if (!update.has_compression_type()) { + NS_WARNING(nsPrintfCString("%s with no compression type.", + aIsAddition ? "Addition" : "Removal").get()); + continue; + } + + switch (update.compression_type()) { + case COMPRESSION_TYPE_UNSPECIFIED: + NS_WARNING("Unspecified compression type."); + break; + + case RAW: + ret = (aIsAddition ? ProcessRawAddition(aTableUpdate, update) + : ProcessRawRemoval(aTableUpdate, update)); + break; + + case RICE: + ret = (aIsAddition ? ProcessEncodedAddition(aTableUpdate, update) + : ProcessEncodedRemoval(aTableUpdate, update)); + break; + } + } + + return ret; +} + +nsresult +ProtocolParserProtobuf::ProcessRawAddition(TableUpdateV4& aTableUpdate, + const ThreatEntrySet& aAddition) +{ + if (!aAddition.has_raw_hashes()) { + PARSER_LOG(("* No raw addition.")); + return NS_OK; + } + + auto rawHashes = aAddition.raw_hashes(); + if (!rawHashes.has_prefix_size()) { + NS_WARNING("Raw hash has no prefix size"); + return NS_OK; + } + + auto prefixes = rawHashes.raw_hashes(); + if (4 == rawHashes.prefix_size()) { + // Process fixed length prefixes separately. + uint32_t* fixedLengthPrefixes = (uint32_t*)prefixes.c_str(); + size_t numOfFixedLengthPrefixes = prefixes.size() / 4; + PARSER_LOG(("* Raw addition (4 bytes)")); + PARSER_LOG((" - # of prefixes: %d", numOfFixedLengthPrefixes)); + PARSER_LOG((" - Memory address: 0x%p", fixedLengthPrefixes)); + } else { + // TODO: Process variable length prefixes including full hashes. + // See Bug 1283009. + PARSER_LOG((" Raw addition (%d bytes)", rawHashes.prefix_size())); + } + + if (!rawHashes.mutable_raw_hashes()) { + PARSER_LOG(("Unable to get mutable raw hashes. Can't perform a string move.")); + return NS_ERROR_FAILURE; + } + + aTableUpdate.NewPrefixes(rawHashes.prefix_size(), + *rawHashes.mutable_raw_hashes()); + + return NS_OK; +} + +nsresult +ProtocolParserProtobuf::ProcessRawRemoval(TableUpdateV4& aTableUpdate, + const ThreatEntrySet& aRemoval) +{ + if (!aRemoval.has_raw_indices()) { + NS_WARNING("A removal has no indices."); + return NS_OK; + } + + // indices is an array of int32. + auto indices = aRemoval.raw_indices().indices(); + PARSER_LOG(("* Raw removal")); + PARSER_LOG((" - # of removal: %d", indices.size())); + + aTableUpdate.NewRemovalIndices((const uint32_t*)indices.data(), + indices.size()); + + return NS_OK; +} + +static nsresult +DoRiceDeltaDecode(const RiceDeltaEncoding& aEncoding, + nsTArray<uint32_t>& aDecoded) +{ + if (!aEncoding.has_first_value()) { + PARSER_LOG(("The encoding info is incomplete.")); + return NS_ERROR_FAILURE; + } + if (aEncoding.num_entries() > 0 && + (!aEncoding.has_rice_parameter() || !aEncoding.has_encoded_data())) { + PARSER_LOG(("Rice parameter or encoded data is missing.")); + return NS_ERROR_FAILURE; + } + + PARSER_LOG(("* Encoding info:")); + PARSER_LOG((" - First value: %d", aEncoding.first_value())); + PARSER_LOG((" - Num of entries: %d", aEncoding.num_entries())); + PARSER_LOG((" - Rice parameter: %d", aEncoding.rice_parameter())); + + // Set up the input buffer. Note that the bits should be read + // from LSB to MSB so that we in-place reverse the bits before + // feeding to the decoder. + auto encoded = const_cast<RiceDeltaEncoding&>(aEncoding).mutable_encoded_data(); + RiceDeltaDecoder decoder((uint8_t*)encoded->c_str(), encoded->size()); + + // Setup the output buffer. The "first value" is included in + // the output buffer. + aDecoded.SetLength(aEncoding.num_entries() + 1); + + // Decode! + bool rv = decoder.Decode(aEncoding.rice_parameter(), + aEncoding.first_value(), // first value. + aEncoding.num_entries(), // # of entries (first value not included). + &aDecoded[0]); + + NS_ENSURE_TRUE(rv, NS_ERROR_FAILURE); + + return NS_OK; +} + +nsresult +ProtocolParserProtobuf::ProcessEncodedAddition(TableUpdateV4& aTableUpdate, + const ThreatEntrySet& aAddition) +{ + if (!aAddition.has_rice_hashes()) { + PARSER_LOG(("* No rice encoded addition.")); + return NS_OK; + } + + nsTArray<uint32_t> decoded; + nsresult rv = DoRiceDeltaDecode(aAddition.rice_hashes(), decoded); + if (NS_FAILED(rv)) { + PARSER_LOG(("Failed to parse encoded prefixes.")); + return rv; + } + + // Say we have the following raw prefixes + // BE LE + // 00 00 00 01 1 16777216 + // 00 00 02 00 512 131072 + // 00 03 00 00 196608 768 + // 04 00 00 00 67108864 4 + // + // which can be treated as uint32 (big-endian) sorted in increasing order: + // + // [1, 512, 196608, 67108864] + // + // According to https://developers.google.com/safe-browsing/v4/compression, + // the following should be done prior to compression: + // + // 1) re-interpret in little-endian ==> [16777216, 131072, 768, 4] + // 2) sort in increasing order ==> [4, 768, 131072, 16777216] + // + // In order to get the original byte stream from |decoded| + // ([4, 768, 131072, 16777216] in this case), we have to: + // + // 1) sort in big-endian order ==> [16777216, 131072, 768, 4] + // 2) copy each uint32 in little-endian to the result string + // + + // The 4-byte prefixes have to be re-sorted in Big-endian increasing order. + struct CompareBigEndian + { + bool Equals(const uint32_t& aA, const uint32_t& aB) const + { + return aA == aB; + } + + bool LessThan(const uint32_t& aA, const uint32_t& aB) const + { + return NativeEndian::swapToBigEndian(aA) < + NativeEndian::swapToBigEndian(aB); + } + }; + decoded.Sort(CompareBigEndian()); + + // The encoded prefixes are always 4 bytes. + std::string prefixes; + for (size_t i = 0; i < decoded.Length(); i++) { + // Note that the third argument is the number of elements we want + // to copy (and swap) but not the number of bytes we want to copy. + char p[4]; + NativeEndian::copyAndSwapToLittleEndian(p, &decoded[i], 1); + prefixes.append(p, 4); + } + + aTableUpdate.NewPrefixes(4, prefixes); + + return NS_OK; +} + +nsresult +ProtocolParserProtobuf::ProcessEncodedRemoval(TableUpdateV4& aTableUpdate, + const ThreatEntrySet& aRemoval) +{ + if (!aRemoval.has_rice_indices()) { + PARSER_LOG(("* No rice encoded removal.")); + return NS_OK; + } + + nsTArray<uint32_t> decoded; + nsresult rv = DoRiceDeltaDecode(aRemoval.rice_indices(), decoded); + if (NS_FAILED(rv)) { + PARSER_LOG(("Failed to decode encoded removal indices.")); + return rv; + } + + // The encoded prefixes are always 4 bytes. + aTableUpdate.NewRemovalIndices(&decoded[0], decoded.Length()); + + return NS_OK; +} + +} // namespace safebrowsing +} // namespace mozilla |