summaryrefslogtreecommitdiffstats
path: root/toolkit/components/url-classifier/tests/gtest
diff options
context:
space:
mode:
Diffstat (limited to 'toolkit/components/url-classifier/tests/gtest')
-rw-r--r--toolkit/components/url-classifier/tests/gtest/Common.cpp219
-rw-r--r--toolkit/components/url-classifier/tests/gtest/Common.h147
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestCaching.cpp271
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestChunkSet.cpp281
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestClassifier.cpp83
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestFailUpdate.cpp109
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestFindFullHash.cpp216
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestLookupCacheV4.cpp152
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestPerProviderDirectory.cpp127
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestPrefixSet.cpp87
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestProtocolParser.cpp140
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestRiceDeltaDecoder.cpp173
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestSafeBrowsingProtobuf.cpp30
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestSafebrowsingHash.cpp58
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestTable.cpp59
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestURLsAndHashing.cpp71
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestUrlClassifierTableUpdateV4.cpp785
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestUrlClassifierUtils.cpp254
-rw-r--r--toolkit/components/url-classifier/tests/gtest/TestVariableLengthPrefixSet.cpp486
-rw-r--r--toolkit/components/url-classifier/tests/gtest/moz.build40
20 files changed, 3788 insertions, 0 deletions
diff --git a/toolkit/components/url-classifier/tests/gtest/Common.cpp b/toolkit/components/url-classifier/tests/gtest/Common.cpp
new file mode 100644
index 0000000000..172ded051f
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/Common.cpp
@@ -0,0 +1,219 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "Common.h"
+
+#include "Classifier.h"
+#include "HashStore.h"
+#include "mozilla/Components.h"
+#include "mozilla/SpinEventLoopUntil.h"
+#include "mozilla/Unused.h"
+#include "nsAppDirectoryServiceDefs.h"
+#include "nsIThread.h"
+#include "nsTArray.h"
+#include "nsThreadUtils.h"
+#include "nsUrlClassifierUtils.h"
+
+using namespace mozilla;
+using namespace mozilla::safebrowsing;
+
+nsresult SyncApplyUpdates(TableUpdateArray& aUpdates) {
+ // We need to spin a new thread specifically because the callback
+ // will be on the caller thread. If we call Classifier::AsyncApplyUpdates
+ // and wait on the same thread, this function will never return.
+
+ nsresult ret = NS_ERROR_FAILURE;
+ bool done = false;
+ auto onUpdateComplete = [&done, &ret](nsresult rv) {
+ // We are on the "ApplyUpdate" thread. Post an event to main thread
+ // so that we can avoid busy waiting on the main thread.
+ nsCOMPtr<nsIRunnable> r =
+ NS_NewRunnableFunction("SyncApplyUpdates", [&done, &ret, rv] {
+ ret = rv;
+ done = true;
+ });
+ NS_DispatchToMainThread(r);
+ };
+
+ nsCOMPtr<nsIFile> file;
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
+
+ nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction("SyncApplyUpdates", [&]() {
+ RefPtr<Classifier> classifier = new Classifier();
+ classifier->Open(*file);
+
+ nsresult rv = classifier->AsyncApplyUpdates(aUpdates, onUpdateComplete);
+ if (NS_FAILED(rv)) {
+ onUpdateComplete(rv);
+ }
+ });
+
+ nsCOMPtr<nsIThread> testingThread;
+ NS_NewNamedThread("ApplyUpdates", getter_AddRefs(testingThread));
+ if (!testingThread) {
+ return NS_ERROR_FAILURE;
+ }
+
+ testingThread->Dispatch(r, NS_DISPATCH_NORMAL);
+
+ // NS_NewCheckSummedOutputStream in HashStore::WriteFile
+ // will synchronously init NS_CRYPTO_HASH_CONTRACTID on
+ // the main thread. As a result we have to keep processing
+ // pending event until |done| becomes true. If there's no
+ // more pending event, what we only can do is wait.
+ MOZ_ALWAYS_TRUE(SpinEventLoopUntil("url-classifier:SyncApplyUpdates"_ns,
+ [&]() { return done; }));
+
+ return ret;
+}
+
+already_AddRefed<nsIFile> GetFile(const nsTArray<nsString>& path) {
+ nsCOMPtr<nsIFile> file;
+ nsresult rv =
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
+ if (NS_WARN_IF(NS_FAILED(rv))) {
+ return nullptr;
+ }
+
+ for (uint32_t i = 0; i < path.Length(); i++) {
+ file->Append(path[i]);
+ }
+ return file.forget();
+}
+
+void ApplyUpdate(TableUpdateArray& updates) {
+ // Force nsUrlClassifierUtils loading on main thread
+ // because nsIUrlClassifierDBService will not run in advance
+ // in gtest.
+ nsUrlClassifierUtils::GetInstance();
+
+ SyncApplyUpdates(updates);
+}
+
+void ApplyUpdate(TableUpdate* update) {
+ TableUpdateArray updates = {update};
+ ApplyUpdate(updates);
+}
+
+nsresult PrefixArrayToPrefixStringMap(const _PrefixArray& aPrefixArray,
+ PrefixStringMap& aOut) {
+ aOut.Clear();
+
+ // Buckets are keyed by prefix length and contain an array of
+ // all prefixes of that length.
+ nsClassHashtable<nsUint32HashKey, _PrefixArray> table;
+ for (const auto& prefix : aPrefixArray) {
+ _PrefixArray* array = table.GetOrInsertNew(prefix.Length());
+ array->AppendElement(prefix);
+ }
+
+ // The resulting map entries will be a concatenation of all
+ // prefix data for the prefixes of a given size.
+ for (const auto& entry : table) {
+ uint32_t size = entry.GetKey();
+ uint32_t count = entry.GetData()->Length();
+
+ auto str = MakeUnique<_Prefix>();
+ str->SetLength(size * count);
+
+ char* dst = str->BeginWriting();
+
+ entry.GetData()->Sort();
+ for (uint32_t i = 0; i < count; i++) {
+ memcpy(dst, entry.GetData()->ElementAt(i).get(), size);
+ dst += size;
+ }
+
+ aOut.InsertOrUpdate(size, std::move(str));
+ }
+
+ return NS_OK;
+}
+
+nsresult PrefixArrayToAddPrefixArray(const _PrefixArray& aPrefixArray,
+ AddPrefixArray& aOut) {
+ aOut.Clear();
+
+ for (const auto& prefix : aPrefixArray) {
+ // Create prefix hash from string
+ AddPrefix* add = aOut.AppendElement(fallible);
+ if (!add) {
+ return NS_ERROR_OUT_OF_MEMORY;
+ }
+
+ add->addChunk = 1;
+ add->prefix.Assign(prefix);
+ }
+
+ EntrySort(aOut);
+
+ return NS_OK;
+}
+
+_Prefix CreatePrefixFromURL(const char* aURL, uint8_t aPrefixSize) {
+ return CreatePrefixFromURL(nsCString(aURL), aPrefixSize);
+}
+
+_Prefix CreatePrefixFromURL(const nsCString& aURL, uint8_t aPrefixSize) {
+ Completion complete;
+ complete.FromPlaintext(aURL);
+
+ _Prefix prefix;
+ prefix.Assign((const char*)complete.buf, aPrefixSize);
+ return prefix;
+}
+
+void CheckContent(LookupCacheV4* aCache, const _PrefixArray& aPrefixArray) {
+ PrefixStringMap vlPSetMap;
+ aCache->GetPrefixes(vlPSetMap);
+
+ PrefixStringMap expected;
+ PrefixArrayToPrefixStringMap(aPrefixArray, expected);
+
+ for (const auto& entry : vlPSetMap) {
+ nsCString* expectedPrefix = expected.Get(entry.GetKey());
+ nsCString* resultPrefix = entry.GetWeak();
+
+ ASSERT_TRUE(resultPrefix->Equals(*expectedPrefix));
+ }
+}
+
+nsresult BuildLookupCache(const RefPtr<Classifier>& classifier,
+ const nsACString& aTable,
+ _PrefixArray& aPrefixArray) {
+ RefPtr<LookupCache> cache = classifier->GetLookupCache(aTable, false);
+ if (!cache) {
+ return NS_ERROR_FAILURE;
+ }
+
+ if (LookupCache::Cast<LookupCacheV4>(cache)) {
+ // V4
+ RefPtr<LookupCacheV4> cacheV4 = LookupCache::Cast<LookupCacheV4>(cache);
+
+ PrefixStringMap map;
+ PrefixArrayToPrefixStringMap(aPrefixArray, map);
+ return cacheV4->Build(map);
+ } else {
+ // V2
+ RefPtr<LookupCacheV2> cacheV2 = LookupCache::Cast<LookupCacheV2>(cache);
+
+ AddPrefixArray addPrefixes;
+ AddCompleteArray addComples;
+
+ PrefixArrayToAddPrefixArray(aPrefixArray, addPrefixes);
+ return cacheV2->Build(addPrefixes, addComples);
+ }
+}
+
+RefPtr<Classifier> GetClassifier() {
+ nsCOMPtr<nsIFile> file;
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
+
+ RefPtr<Classifier> classifier = new Classifier();
+ nsresult rv = classifier->Open(*file);
+ EXPECT_TRUE(rv == NS_OK);
+
+ return classifier;
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/Common.h b/toolkit/components/url-classifier/tests/gtest/Common.h
new file mode 100644
index 0000000000..9c196abafa
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/Common.h
@@ -0,0 +1,147 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef nsUrlClassifierGTestCommon_h__
+#define nsUrlClassifierGTestCommon_h__
+
+#include "Entries.h"
+#include "nsAppDirectoryServiceDefs.h"
+#include "nsIFile.h"
+#include "nsTArray.h"
+#include "nsIThread.h"
+#include "nsThreadUtils.h"
+#include "HashStore.h"
+
+#include "gtest/gtest.h"
+#include "mozilla/gtest/MozAssertions.h"
+#include "LookupCacheV4.h"
+
+using namespace mozilla::safebrowsing;
+
+namespace mozilla {
+namespace safebrowsing {
+class Classifier;
+class LookupCacheV4;
+class TableUpdate;
+} // namespace safebrowsing
+} // namespace mozilla
+
+#define GTEST_SAFEBROWSING_DIR "safebrowsing"_ns
+#define GTEST_TABLE_V4 "gtest-malware-proto"_ns
+#define GTEST_TABLE_V2 "gtest-malware-simple"_ns
+
+template <typename Function>
+void RunTestInNewThread(Function&& aFunction) {
+ nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction(
+ "RunTestInNewThread", std::forward<Function>(aFunction));
+ nsCOMPtr<nsIThread> testingThread;
+ nsresult rv =
+ NS_NewNamedThread("Testing Thread", getter_AddRefs(testingThread), r);
+ ASSERT_EQ(rv, NS_OK);
+ testingThread->Shutdown();
+}
+
+// Synchronously apply updates by calling Classifier::AsyncApplyUpdates.
+nsresult SyncApplyUpdates(Classifier* aClassifier,
+ nsTArray<TableUpdate*>* aUpdates);
+nsresult SyncApplyUpdates(TableUpdateArray& aUpdates);
+
+// Return nsIFile with root directory - NS_APP_USER_PROFILE_50_DIR
+// Sub-directories are passed in path argument.
+already_AddRefed<nsIFile> GetFile(const nsTArray<nsString>& aPath);
+
+// ApplyUpdate will call |ApplyUpdates| of Classifier within a new thread
+void ApplyUpdate(nsTArray<TableUpdate*>& aUpdates);
+
+void ApplyUpdate(TableUpdate* aUpdate);
+
+/**
+ * Prefix processing utility functions
+ */
+
+typedef nsCString _Prefix;
+typedef nsTArray<nsCString> _PrefixArray;
+
+// This function converts a lexigraphic-sorted prefixes array
+// to a hash table keyed by prefix size(PrefixStringMap is defined in Entries.h)
+nsresult PrefixArrayToPrefixStringMap(const _PrefixArray& aPrefixArray,
+ PrefixStringMap& aOut);
+
+// This function converts a lexigraphic-sorted prefixes array
+// to an array containing AddPrefix(AddPrefix is defined in Entries.h)
+nsresult PrefixArrayToAddPrefixArray(const _PrefixArray& aPrefixArray,
+ AddPrefixArray& aOut);
+
+_Prefix CreatePrefixFromURL(const char* aURL, uint8_t aPrefixSize);
+
+_Prefix CreatePrefixFromURL(const nsCString& aURL, uint8_t aPrefixSize);
+
+// To test if the content is equal
+void CheckContent(LookupCacheV4* cache, const _PrefixArray& aPrefixArray);
+
+/**
+ * Utility function to generate safebrowsing internal structure
+ */
+
+static inline nsresult BuildCache(LookupCacheV2* cache,
+ const _PrefixArray& aPrefixArray) {
+ AddPrefixArray prefixes;
+ AddCompleteArray completions;
+ nsresult rv = PrefixArrayToAddPrefixArray(aPrefixArray, prefixes);
+ if (NS_FAILED(rv)) {
+ return rv;
+ }
+
+ return cache->Build(prefixes, completions);
+}
+
+static inline nsresult BuildCache(LookupCacheV4* cache,
+ const _PrefixArray& aPrefixArray) {
+ PrefixStringMap map;
+ PrefixArrayToPrefixStringMap(aPrefixArray, map);
+ return cache->Build(map);
+}
+
+// Create a LookupCacheV4 object with sepecified prefix array.
+template <typename T>
+RefPtr<T> SetupLookupCache(const _PrefixArray& aPrefixArray,
+ nsCOMPtr<nsIFile>& aFile) {
+ RefPtr<T> cache = new T(GTEST_TABLE_V4, ""_ns, aFile);
+
+ nsresult rv = cache->Init();
+ EXPECT_EQ(rv, NS_OK);
+
+ rv = BuildCache(cache, aPrefixArray);
+ EXPECT_EQ(rv, NS_OK);
+
+ return cache;
+}
+
+template <typename T>
+RefPtr<T> SetupLookupCache(const _PrefixArray& aPrefixArray) {
+ nsCOMPtr<nsIFile> file;
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
+
+ file->AppendNative(GTEST_SAFEBROWSING_DIR);
+
+ RefPtr<T> cache = new T(GTEST_TABLE_V4, ""_ns, file);
+ nsresult rv = cache->Init();
+ EXPECT_EQ(rv, NS_OK);
+
+ rv = BuildCache(cache, aPrefixArray);
+ EXPECT_EQ(rv, NS_OK);
+
+ return cache;
+}
+
+/**
+ * Retrieve Classifer class
+ */
+RefPtr<Classifier> GetClassifier();
+
+nsresult BuildLookupCache(const RefPtr<Classifier>& aClassifier,
+ const nsACString& aTable, _PrefixArray& aPrefixArray);
+
+#endif // nsUrlClassifierGTestCommon_h__
diff --git a/toolkit/components/url-classifier/tests/gtest/TestCaching.cpp b/toolkit/components/url-classifier/tests/gtest/TestCaching.cpp
new file mode 100644
index 0000000000..520eb35dcb
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestCaching.cpp
@@ -0,0 +1,271 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "Common.h"
+#include "LookupCacheV4.h"
+
+#define EXPIRED_TIME_SEC (PR_Now() / PR_USEC_PER_SEC - 3600)
+#define NOTEXPIRED_TIME_SEC (PR_Now() / PR_USEC_PER_SEC + 3600)
+
+#define CACHED_URL "cache.com/"_ns
+#define NEG_CACHE_EXPIRED_URL "cache.negExpired.com/"_ns
+#define POS_CACHE_EXPIRED_URL "cache.posExpired.com/"_ns
+#define BOTH_CACHE_EXPIRED_URL "cache.negAndposExpired.com/"_ns
+
+static void SetupCacheEntry(LookupCacheV2* aLookupCache,
+ const nsCString& aCompletion,
+ bool aNegExpired = false,
+ bool aPosExpired = false) {
+ AddCompleteArray completes;
+ AddCompleteArray emptyCompletes;
+ MissPrefixArray misses;
+ MissPrefixArray emptyMisses;
+
+ AddComplete* add = completes.AppendElement(mozilla::fallible);
+ add->complete.FromPlaintext(aCompletion);
+
+ Prefix* prefix = misses.AppendElement(mozilla::fallible);
+ prefix->FromPlaintext(aCompletion);
+
+ // Setup positive cache first otherwise negative cache expiry will be
+ // overwritten.
+ int64_t posExpirySec = aPosExpired ? EXPIRED_TIME_SEC : NOTEXPIRED_TIME_SEC;
+ aLookupCache->AddGethashResultToCache(completes, emptyMisses, posExpirySec);
+
+ int64_t negExpirySec = aNegExpired ? EXPIRED_TIME_SEC : NOTEXPIRED_TIME_SEC;
+ aLookupCache->AddGethashResultToCache(emptyCompletes, misses, negExpirySec);
+}
+
+static void SetupCacheEntry(LookupCacheV4* aLookupCache,
+ const nsCString& aCompletion,
+ bool aNegExpired = false,
+ bool aPosExpired = false) {
+ FullHashResponseMap map;
+
+ Prefix prefix;
+ prefix.FromPlaintext(aCompletion);
+
+ CachedFullHashResponse* response = map.GetOrInsertNew(prefix.ToUint32());
+
+ response->negativeCacheExpirySec =
+ aNegExpired ? EXPIRED_TIME_SEC : NOTEXPIRED_TIME_SEC;
+ response->fullHashes.InsertOrUpdate(
+ CreatePrefixFromURL(aCompletion, COMPLETE_SIZE),
+ aPosExpired ? EXPIRED_TIME_SEC : NOTEXPIRED_TIME_SEC);
+
+ aLookupCache->AddFullHashResponseToCache(map);
+}
+
+template <typename T>
+static void TestCache(const Completion aCompletion, bool aExpectedHas,
+ bool aExpectedConfirmed, bool aExpectedInCache,
+ T* aCache = nullptr) {
+ bool has, inCache, confirmed;
+ uint32_t matchLength;
+
+ if (aCache) {
+ aCache->Has(aCompletion, &has, &matchLength, &confirmed);
+ inCache = aCache->IsInCache(aCompletion.ToUint32());
+ } else {
+ _PrefixArray array = {CreatePrefixFromURL("cache.notexpired.com/", 10),
+ CreatePrefixFromURL("cache.expired.com/", 8),
+ CreatePrefixFromURL("gound.com/", 5),
+ CreatePrefixFromURL("small.com/", 4)};
+
+ RefPtr<T> cache = SetupLookupCache<T>(array);
+
+ // Create an expired entry and a non-expired entry
+ SetupCacheEntry(cache, "cache.notexpired.com/"_ns);
+ SetupCacheEntry(cache, "cache.expired.com/"_ns, true, true);
+
+ cache->Has(aCompletion, &has, &matchLength, &confirmed);
+ inCache = cache->IsInCache(aCompletion.ToUint32());
+ }
+
+ EXPECT_EQ(has, aExpectedHas);
+ EXPECT_EQ(confirmed, aExpectedConfirmed);
+ EXPECT_EQ(inCache, aExpectedInCache);
+}
+
+template <typename T>
+static void TestCache(const nsCString& aURL, bool aExpectedHas,
+ bool aExpectedConfirmed, bool aExpectedInCache,
+ T* aCache = nullptr) {
+ Completion lookupHash;
+ lookupHash.FromPlaintext(aURL);
+
+ TestCache<T>(lookupHash, aExpectedHas, aExpectedConfirmed, aExpectedInCache,
+ aCache);
+}
+
+// This testcase check the returned result of |Has| API if fullhash cannot match
+// any prefix in the local database.
+TEST(UrlClassifierCaching, NotFound)
+{
+ TestCache<LookupCacheV2>("nomatch.com/"_ns, false, false, false);
+ TestCache<LookupCacheV4>("nomatch.com/"_ns, false, false, false);
+}
+
+// This testcase check the returned result of |Has| API if fullhash find a match
+// in the local database but not in the cache.
+TEST(UrlClassifierCaching, NotInCache)
+{
+ TestCache<LookupCacheV2>("gound.com/"_ns, true, false, false);
+ TestCache<LookupCacheV4>("gound.com/"_ns, true, false, false);
+}
+
+// This testcase check the returned result of |Has| API if fullhash matches
+// a cache entry in positive cache.
+TEST(UrlClassifierCaching, InPositiveCacheNotExpired)
+{
+ TestCache<LookupCacheV2>("cache.notexpired.com/"_ns, true, true, true);
+ TestCache<LookupCacheV4>("cache.notexpired.com/"_ns, true, true, true);
+}
+
+// This testcase check the returned result of |Has| API if fullhash matches
+// a cache entry in positive cache but that it is expired.
+TEST(UrlClassifierCaching, InPositiveCacheExpired)
+{
+ TestCache<LookupCacheV2>("cache.expired.com/"_ns, true, false, true);
+ TestCache<LookupCacheV4>("cache.expired.com/"_ns, true, false, true);
+}
+
+// This testcase check the returned result of |Has| API if fullhash matches
+// a cache entry in negative cache.
+TEST(UrlClassifierCaching, InNegativeCacheNotExpired)
+{
+ // Create a fullhash whose prefix matches the prefix in negative cache
+ // but completion doesn't match any fullhash in positive cache.
+
+ Completion prefix;
+ prefix.FromPlaintext("cache.notexpired.com/"_ns);
+
+ Completion fullhash;
+ fullhash.FromPlaintext("firefox.com/"_ns);
+
+ // Overwrite the 4-byte prefix of `fullhash` so that it conflicts with
+ // `prefix`. Since "cache.notexpired.com" is added to database in TestCache as
+ // a 10-byte prefix, we should copy more than 10 bytes to fullhash to ensure
+ // it can match the prefix in database.
+ memcpy(fullhash.buf, prefix.buf, 10);
+
+ TestCache<LookupCacheV2>(fullhash, false, false, true);
+ TestCache<LookupCacheV4>(fullhash, false, false, true);
+}
+
+// This testcase check the returned result of |Has| API if fullhash matches
+// a cache entry in negative cache but that entry is expired.
+TEST(UrlClassifierCaching, InNegativeCacheExpired)
+{
+ // Create a fullhash whose prefix is in the cache.
+
+ Completion prefix;
+ prefix.FromPlaintext("cache.expired.com/"_ns);
+
+ Completion fullhash;
+ fullhash.FromPlaintext("firefox.com/"_ns);
+
+ memcpy(fullhash.buf, prefix.buf, 10);
+
+ TestCache<LookupCacheV2>(fullhash, true, false, true);
+ TestCache<LookupCacheV4>(fullhash, true, false, true);
+}
+
+// This testcase create 4 cache entries.
+// 1. unexpired entry.
+// 2. an entry whose negative cache time is expired but whose positive cache
+// is not expired.
+// 3. an entry whose positive cache time is expired
+// 4. an entry whose negative cache time and positive cache time are expired
+// After calling |InvalidateExpiredCacheEntry| API, entries with expired
+// negative time should be removed from cache(2 & 4)
+template <typename T>
+void TestInvalidateExpiredCacheEntry() {
+ _PrefixArray array = {CreatePrefixFromURL(CACHED_URL, 10),
+ CreatePrefixFromURL(NEG_CACHE_EXPIRED_URL, 8),
+ CreatePrefixFromURL(POS_CACHE_EXPIRED_URL, 5),
+ CreatePrefixFromURL(BOTH_CACHE_EXPIRED_URL, 4)};
+ RefPtr<T> cache = SetupLookupCache<T>(array);
+
+ SetupCacheEntry(cache, CACHED_URL, false, false);
+ SetupCacheEntry(cache, NEG_CACHE_EXPIRED_URL, true, false);
+ SetupCacheEntry(cache, POS_CACHE_EXPIRED_URL, false, true);
+ SetupCacheEntry(cache, BOTH_CACHE_EXPIRED_URL, true, true);
+
+ // Before invalidate
+ TestCache<T>(CACHED_URL, true, true, true, cache.get());
+ TestCache<T>(NEG_CACHE_EXPIRED_URL, true, true, true, cache.get());
+ TestCache<T>(POS_CACHE_EXPIRED_URL, true, false, true, cache.get());
+ TestCache<T>(BOTH_CACHE_EXPIRED_URL, true, false, true, cache.get());
+
+ // Call InvalidateExpiredCacheEntry to remove cache entries whose negative
+ // cache time is expired
+ cache->InvalidateExpiredCacheEntries();
+
+ // After invalidate, NEG_CACHE_EXPIRED_URL & BOTH_CACHE_EXPIRED_URL should
+ // not be found in cache.
+ TestCache<T>(NEG_CACHE_EXPIRED_URL, true, false, false, cache.get());
+ TestCache<T>(BOTH_CACHE_EXPIRED_URL, true, false, false, cache.get());
+
+ // Other entries should remain the same result.
+ TestCache<T>(CACHED_URL, true, true, true, cache.get());
+ TestCache<T>(POS_CACHE_EXPIRED_URL, true, false, true, cache.get());
+}
+
+TEST(UrlClassifierCaching, InvalidateExpiredCacheEntryV2)
+{ TestInvalidateExpiredCacheEntry<LookupCacheV2>(); }
+
+TEST(UrlClassifierCaching, InvalidateExpiredCacheEntryV4)
+{ TestInvalidateExpiredCacheEntry<LookupCacheV4>(); }
+
+// This testcase check if an cache entry whose negative cache time is expired
+// and it doesn't have any postive cache entries in it, it should be removed
+// from cache after calling |Has|.
+TEST(UrlClassifierCaching, NegativeCacheExpireV2)
+{
+ _PrefixArray array = {CreatePrefixFromURL(NEG_CACHE_EXPIRED_URL, 8)};
+ RefPtr<LookupCacheV2> cache = SetupLookupCache<LookupCacheV2>(array);
+
+ nsCOMPtr<nsICryptoHash> cryptoHash =
+ do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID);
+
+ MissPrefixArray misses;
+ Prefix* prefix = misses.AppendElement(mozilla::fallible);
+ prefix->FromPlaintext(NEG_CACHE_EXPIRED_URL);
+
+ AddCompleteArray dummy;
+ cache->AddGethashResultToCache(dummy, misses, EXPIRED_TIME_SEC);
+
+ // Ensure it is in cache in the first place.
+ EXPECT_EQ(cache->IsInCache(prefix->ToUint32()), true);
+
+ // It should be removed after calling Has API.
+ TestCache<LookupCacheV2>(NEG_CACHE_EXPIRED_URL, true, false, false,
+ cache.get());
+}
+
+TEST(UrlClassifierCaching, NegativeCacheExpireV4)
+{
+ _PrefixArray array = {CreatePrefixFromURL(NEG_CACHE_EXPIRED_URL, 8)};
+ RefPtr<LookupCacheV4> cache = SetupLookupCache<LookupCacheV4>(array);
+
+ FullHashResponseMap map;
+ Prefix prefix;
+ nsCOMPtr<nsICryptoHash> cryptoHash =
+ do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID);
+ prefix.FromPlaintext(NEG_CACHE_EXPIRED_URL);
+ CachedFullHashResponse* response = map.GetOrInsertNew(prefix.ToUint32());
+
+ response->negativeCacheExpirySec = EXPIRED_TIME_SEC;
+
+ cache->AddFullHashResponseToCache(map);
+
+ // Ensure it is in cache in the first place.
+ EXPECT_EQ(cache->IsInCache(prefix.ToUint32()), true);
+
+ // It should be removed after calling Has API.
+ TestCache<LookupCacheV4>(NEG_CACHE_EXPIRED_URL, true, false, false,
+ cache.get());
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestChunkSet.cpp b/toolkit/components/url-classifier/tests/gtest/TestChunkSet.cpp
new file mode 100644
index 0000000000..6835103b30
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestChunkSet.cpp
@@ -0,0 +1,281 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <set>
+
+#include "ChunkSet.h"
+#include "mozilla/ArrayUtils.h"
+
+#include "Common.h"
+
+TEST(UrlClassifierChunkSet, Empty)
+{
+ mozilla::safebrowsing::ChunkSet chunkSet;
+ mozilla::safebrowsing::ChunkSet removeSet;
+
+ removeSet.Set(0);
+
+ ASSERT_FALSE(chunkSet.Has(0));
+ ASSERT_FALSE(chunkSet.Has(1));
+ ASSERT_TRUE(chunkSet.Remove(removeSet) == NS_OK);
+ ASSERT_TRUE(chunkSet.Length() == 0);
+
+ chunkSet.Set(0);
+
+ ASSERT_TRUE(chunkSet.Has(0));
+ ASSERT_TRUE(chunkSet.Length() == 1);
+ ASSERT_TRUE(chunkSet.Remove(removeSet) == NS_OK);
+ ASSERT_FALSE(chunkSet.Has(0));
+ ASSERT_TRUE(chunkSet.Length() == 0);
+}
+
+TEST(UrlClassifierChunkSet, Main)
+{
+ static int testVals[] = {2, 1, 5, 6, 8, 7, 14, 10, 12, 13};
+
+ mozilla::safebrowsing::ChunkSet chunkSet;
+
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) {
+ chunkSet.Set(testVals[i]);
+ }
+
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) {
+ ASSERT_TRUE(chunkSet.Has(testVals[i]));
+ }
+
+ ASSERT_FALSE(chunkSet.Has(3));
+ ASSERT_FALSE(chunkSet.Has(4));
+ ASSERT_FALSE(chunkSet.Has(9));
+ ASSERT_FALSE(chunkSet.Has(11));
+
+ ASSERT_TRUE(chunkSet.Length() == MOZ_ARRAY_LENGTH(testVals));
+}
+
+TEST(UrlClassifierChunkSet, Merge)
+{
+ static int testVals[] = {2, 1, 5, 6, 8, 7, 14, 10, 12, 13};
+ static int mergeVals[] = {9, 3, 4, 20, 14, 16};
+
+ mozilla::safebrowsing::ChunkSet chunkSet;
+ mozilla::safebrowsing::ChunkSet mergeSet;
+
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) {
+ chunkSet.Set(testVals[i]);
+ }
+
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) {
+ mergeSet.Set(mergeVals[i]);
+ }
+
+ chunkSet.Merge(mergeSet);
+
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) {
+ ASSERT_TRUE(chunkSet.Has(testVals[i]));
+ }
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) {
+ ASSERT_TRUE(chunkSet.Has(mergeVals[i]));
+ }
+
+ // -1 because 14 is duplicated in both sets
+ ASSERT_TRUE(chunkSet.Length() ==
+ MOZ_ARRAY_LENGTH(testVals) + MOZ_ARRAY_LENGTH(mergeVals) - 1);
+
+ ASSERT_FALSE(chunkSet.Has(11));
+ ASSERT_FALSE(chunkSet.Has(15));
+ ASSERT_FALSE(chunkSet.Has(17));
+ ASSERT_FALSE(chunkSet.Has(18));
+ ASSERT_FALSE(chunkSet.Has(19));
+}
+
+TEST(UrlClassifierChunkSet, Merge2)
+{
+ static int testVals[] = {2, 1, 5, 6, 8, 7, 14, 10, 12, 13};
+ static int mergeVals[] = {9, 3, 4, 20, 14, 16};
+ static int mergeVals2[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
+
+ mozilla::safebrowsing::ChunkSet chunkSet;
+ mozilla::safebrowsing::ChunkSet mergeSet;
+ mozilla::safebrowsing::ChunkSet mergeSet2;
+
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) {
+ chunkSet.Set(testVals[i]);
+ }
+
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) {
+ mergeSet.Set(mergeVals[i]);
+ }
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals2); i++) {
+ mergeSet2.Set(mergeVals2[i]);
+ }
+
+ chunkSet.Merge(mergeSet);
+ chunkSet.Merge(mergeSet2);
+
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) {
+ ASSERT_TRUE(chunkSet.Has(testVals[i]));
+ }
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) {
+ ASSERT_TRUE(chunkSet.Has(mergeVals[i]));
+ }
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals2); i++) {
+ ASSERT_TRUE(chunkSet.Has(mergeVals2[i]));
+ }
+
+ ASSERT_FALSE(chunkSet.Has(15));
+ ASSERT_FALSE(chunkSet.Has(17));
+ ASSERT_FALSE(chunkSet.Has(18));
+ ASSERT_FALSE(chunkSet.Has(19));
+}
+
+TEST(UrlClassifierChunkSet, Stress)
+{
+ mozilla::safebrowsing::ChunkSet chunkSet;
+ mozilla::safebrowsing::ChunkSet mergeSet;
+ std::set<int> refSet;
+ std::set<int> refMergeSet;
+ static const int TEST_ITERS = 7000;
+ static const int REMOVE_ITERS = 3000;
+ static const int TEST_RANGE = 10000;
+
+ // Construction by Set
+ for (int i = 0; i < TEST_ITERS; i++) {
+ int chunk = rand() % TEST_RANGE;
+ chunkSet.Set(chunk);
+ refSet.insert(chunk);
+ }
+
+ // Same elements as reference set
+ for (auto it = refSet.begin(); it != refSet.end(); ++it) {
+ ASSERT_TRUE(chunkSet.Has(*it));
+ }
+
+ // Hole punching via Remove
+ for (int i = 0; i < REMOVE_ITERS; i++) {
+ int chunk = rand() % TEST_RANGE;
+ mozilla::safebrowsing::ChunkSet helpChunk;
+ helpChunk.Set(chunk);
+
+ chunkSet.Remove(helpChunk);
+ refSet.erase(chunk);
+
+ ASSERT_FALSE(chunkSet.Has(chunk));
+ }
+
+ // Should have chunks present in reference set
+ // Should not have chunks absent in reference set
+ for (int it = 0; it < TEST_RANGE; ++it) {
+ auto found = refSet.find(it);
+ if (chunkSet.Has(it)) {
+ ASSERT_FALSE(found == refSet.end());
+ } else {
+ ASSERT_TRUE(found == refSet.end());
+ }
+ }
+
+ // Construct set to merge with
+ for (int i = 0; i < TEST_ITERS; i++) {
+ int chunk = rand() % TEST_RANGE;
+ mergeSet.Set(chunk);
+ refMergeSet.insert(chunk);
+ }
+
+ // Merge set constructed correctly
+ for (auto it = refMergeSet.begin(); it != refMergeSet.end(); ++it) {
+ ASSERT_TRUE(mergeSet.Has(*it));
+ }
+
+ mozilla::safebrowsing::ChunkSet origSet;
+ origSet = chunkSet.InfallibleClone();
+
+ chunkSet.Merge(mergeSet);
+ refSet.insert(refMergeSet.begin(), refMergeSet.end());
+
+ // Check for presence of elements from both source
+ // Should not have chunks absent in reference set
+ for (int it = 0; it < TEST_RANGE; ++it) {
+ auto found = refSet.find(it);
+ if (chunkSet.Has(it)) {
+ ASSERT_FALSE(found == refSet.end());
+ } else {
+ ASSERT_TRUE(found == refSet.end());
+ }
+ }
+
+ // Unmerge
+ chunkSet.Remove(origSet);
+ for (int it = 0; it < TEST_RANGE; ++it) {
+ if (origSet.Has(it)) {
+ ASSERT_FALSE(chunkSet.Has(it));
+ } else if (mergeSet.Has(it)) {
+ ASSERT_TRUE(chunkSet.Has(it));
+ }
+ }
+}
+
+TEST(UrlClassifierChunkSet, RemoveClear)
+{
+ static int testVals[] = {2, 1, 5, 6, 8, 7, 14, 10, 12, 13};
+ static int mergeVals[] = {3, 4, 9, 16, 20};
+
+ mozilla::safebrowsing::ChunkSet chunkSet;
+ mozilla::safebrowsing::ChunkSet mergeSet;
+ mozilla::safebrowsing::ChunkSet removeSet;
+
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) {
+ chunkSet.Set(testVals[i]);
+ removeSet.Set(testVals[i]);
+ }
+
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) {
+ mergeSet.Set(mergeVals[i]);
+ }
+
+ ASSERT_TRUE(chunkSet.Merge(mergeSet) == NS_OK);
+ ASSERT_TRUE(chunkSet.Remove(removeSet) == NS_OK);
+
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) {
+ ASSERT_TRUE(chunkSet.Has(mergeVals[i]));
+ }
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) {
+ ASSERT_FALSE(chunkSet.Has(testVals[i]));
+ }
+
+ chunkSet.Clear();
+
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) {
+ ASSERT_FALSE(chunkSet.Has(mergeVals[i]));
+ }
+}
+
+TEST(UrlClassifierChunkSet, Serialize)
+{
+ static int testVals[] = {2, 1, 5, 6, 8, 7, 14, 10, 12, 13};
+ static int mergeVals[] = {3, 4, 9, 16, 20};
+
+ mozilla::safebrowsing::ChunkSet chunkSet;
+ mozilla::safebrowsing::ChunkSet mergeSet;
+
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(testVals); i++) {
+ chunkSet.Set(testVals[i]);
+ }
+
+ for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mergeVals); i++) {
+ mergeSet.Set(mergeVals[i]);
+ }
+
+ chunkSet.Merge(mergeSet);
+
+ nsAutoCString mergeResult;
+ chunkSet.Serialize(mergeResult);
+
+ printf("mergeResult: %s\n", mergeResult.get());
+
+ nsAutoCString expected("1-10,12-14,16,20"_ns);
+
+ ASSERT_TRUE(mergeResult.Equals(expected));
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestClassifier.cpp b/toolkit/components/url-classifier/tests/gtest/TestClassifier.cpp
new file mode 100644
index 0000000000..4000d6b32e
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestClassifier.cpp
@@ -0,0 +1,83 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "Classifier.h"
+#include "LookupCacheV4.h"
+
+#include "Common.h"
+
+static void TestReadNoiseEntries(RefPtr<Classifier> classifier,
+ const nsCString& aTable, const nsCString& aURL,
+ const _PrefixArray& aPrefixArray) {
+ Completion lookupHash;
+ lookupHash.FromPlaintext(aURL);
+ RefPtr<LookupResult> result = new LookupResult;
+ result->hash.complete = lookupHash;
+
+ PrefixArray noiseEntries;
+ uint32_t noiseCount = 3;
+ nsresult rv;
+ rv = classifier->ReadNoiseEntries(result->hash.fixedLengthPrefix, aTable,
+ noiseCount, noiseEntries);
+ ASSERT_TRUE(rv == NS_OK)
+ << "ReadNoiseEntries returns an error";
+ EXPECT_TRUE(noiseEntries.Length() > 0)
+ << "Number of noise entries is less than 0";
+
+ for (uint32_t i = 0; i < noiseEntries.Length(); i++) {
+ // Test the noise entry should not equal the "real" hash request
+ EXPECT_NE(noiseEntries[i], result->hash.fixedLengthPrefix)
+ << "Noise entry is the same as real hash request";
+ // Test the noise entry should exist in the cached prefix array
+ nsAutoCString partialHash;
+ partialHash.Assign(reinterpret_cast<char*>(&noiseEntries[i]), PREFIX_SIZE);
+ EXPECT_TRUE(aPrefixArray.Contains(partialHash))
+ << "Noise entry is not in the cached prefix array";
+ }
+}
+
+TEST(UrlClassifierClassifier, ReadNoiseEntriesV4)
+{
+ RefPtr<Classifier> classifier = GetClassifier();
+ _PrefixArray array = {
+ CreatePrefixFromURL("bravo.com/", 5),
+ CreatePrefixFromURL("browsing.com/", 9),
+ CreatePrefixFromURL("gound.com/", 4),
+ CreatePrefixFromURL("small.com/", 4),
+ CreatePrefixFromURL("gdfad.com/", 4),
+ CreatePrefixFromURL("afdfound.com/", 4),
+ CreatePrefixFromURL("dffa.com/", 4),
+ };
+
+ nsresult rv;
+ rv = BuildLookupCache(classifier, GTEST_TABLE_V4, array);
+ ASSERT_TRUE(rv == NS_OK)
+ << "Fail to build LookupCache";
+
+ TestReadNoiseEntries(classifier, GTEST_TABLE_V4, "gound.com/"_ns, array);
+}
+
+TEST(UrlClassifierClassifier, ReadNoiseEntriesV2)
+{
+ RefPtr<Classifier> classifier = GetClassifier();
+ _PrefixArray array = {
+ CreatePrefixFromURL("helloworld.com/", 4),
+ CreatePrefixFromURL("firefox.com/", 4),
+ CreatePrefixFromURL("chrome.com/", 4),
+ CreatePrefixFromURL("safebrowsing.com/", 4),
+ CreatePrefixFromURL("opera.com/", 4),
+ CreatePrefixFromURL("torbrowser.com/", 4),
+ CreatePrefixFromURL("gfaads.com/", 4),
+ CreatePrefixFromURL("qggdsas.com/", 4),
+ CreatePrefixFromURL("nqtewq.com/", 4),
+ };
+
+ nsresult rv;
+ rv = BuildLookupCache(classifier, GTEST_TABLE_V2, array);
+ ASSERT_TRUE(rv == NS_OK)
+ << "Fail to build LookupCache";
+
+ TestReadNoiseEntries(classifier, GTEST_TABLE_V2, "helloworld.com/"_ns, array);
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestFailUpdate.cpp b/toolkit/components/url-classifier/tests/gtest/TestFailUpdate.cpp
new file mode 100644
index 0000000000..f94852a821
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestFailUpdate.cpp
@@ -0,0 +1,109 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "HashStore.h"
+#include "mozilla/Unused.h"
+#include "nsPrintfCString.h"
+#include "string.h"
+
+#include "Common.h"
+
+static const char* kFilesInV2[] = {".vlpset", ".sbstore"};
+static const char* kFilesInV4[] = {".vlpset", ".metadata"};
+
+#define GTEST_MALWARE_TABLE_V4 "goog-malware-proto"_ns
+#define GTEST_PHISH_TABLE_V4 "goog-phish-proto"_ns
+
+#define ROOT_DIR u"safebrowsing"_ns
+#define SB_FILE(x, y) NS_ConvertUTF8toUTF16(nsPrintfCString("%s%s", x, y))
+
+template <typename T, size_t N>
+static void CheckFileExist(const nsCString& aTable, const T (&aFiles)[N],
+ bool aExpectExists, const char* aMsg = nullptr) {
+ for (uint32_t i = 0; i < N; i++) {
+ // This is just a quick way to know if this is v4 table
+ NS_ConvertUTF8toUTF16 SUB_DIR(strstr(aTable.get(), "-proto") ? "google4"
+ : "");
+ nsCOMPtr<nsIFile> file = GetFile(nsTArray<nsString>{
+ ROOT_DIR, SUB_DIR, SB_FILE(aTable.get(), aFiles[i])});
+
+ bool exists;
+ file->Exists(&exists);
+
+ if (aMsg) {
+ ASSERT_EQ(aExpectExists, exists)
+ << file->HumanReadablePath().get() << " " << aMsg;
+ } else {
+ ASSERT_EQ(aExpectExists, exists) << file->HumanReadablePath().get();
+ }
+ }
+}
+
+TEST(UrlClassifierFailUpdate, CheckTableReset)
+{
+ const bool FULL_UPDATE = true;
+ const bool PARTIAL_UPDATE = false;
+
+ // Apply V2 update
+ {
+ RefPtr<TableUpdateV2> update = new TableUpdateV2(GTEST_TABLE_V2);
+ mozilla::Unused << update->NewAddChunk(1);
+
+ ApplyUpdate(update);
+
+ // A successful V2 update should create .vlpset & .sbstore files
+ CheckFileExist(GTEST_TABLE_V2, kFilesInV2, true,
+ "V2 update doesn't create vlpset or sbstore");
+ }
+
+ // Helper function to generate table update data
+ auto func = [](RefPtr<TableUpdateV4> update, bool full, const char* str) {
+ update->SetFullUpdate(full);
+ nsCString prefix(str);
+ update->NewPrefixes(prefix.Length(), prefix);
+ };
+
+ // Apply V4 update for table1
+ {
+ RefPtr<TableUpdateV4> update = new TableUpdateV4(GTEST_MALWARE_TABLE_V4);
+ func(update, FULL_UPDATE, "test_prefix");
+
+ ApplyUpdate(update);
+
+ // A successful V4 update should create .vlpset & .metadata files
+ CheckFileExist(GTEST_MALWARE_TABLE_V4, kFilesInV4, true,
+ "v4 update doesn't create vlpset or metadata");
+ }
+
+ // Apply V4 update for table2
+ {
+ RefPtr<TableUpdateV4> update = new TableUpdateV4(GTEST_PHISH_TABLE_V4);
+ func(update, FULL_UPDATE, "test_prefix");
+
+ ApplyUpdate(update);
+
+ CheckFileExist(GTEST_PHISH_TABLE_V4, kFilesInV4, true,
+ "v4 update doesn't create vlpset or metadata");
+ }
+
+ // Apply V4 update with the same prefix in previous full udpate
+ // This should cause an update error.
+ {
+ RefPtr<TableUpdateV4> update = new TableUpdateV4(GTEST_MALWARE_TABLE_V4);
+ func(update, PARTIAL_UPDATE, "test_prefix");
+
+ ApplyUpdate(update);
+
+ // A fail update should remove files for that table
+ CheckFileExist(GTEST_MALWARE_TABLE_V4, kFilesInV4, false,
+ "a fail v4 update doesn't remove the tables");
+
+ // A fail update should NOT remove files for the other tables
+ CheckFileExist(GTEST_TABLE_V2, kFilesInV2, true,
+ "a fail v4 update removes a v2 table");
+ CheckFileExist(GTEST_PHISH_TABLE_V4, kFilesInV4, true,
+ "a fail v4 update removes the other v4 table");
+ }
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestFindFullHash.cpp b/toolkit/components/url-classifier/tests/gtest/TestFindFullHash.cpp
new file mode 100644
index 0000000000..f0f5785017
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestFindFullHash.cpp
@@ -0,0 +1,216 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/Base64.h"
+#include "nsUrlClassifierUtils.h"
+#include "safebrowsing.pb.h"
+
+#include "Common.h"
+
+using namespace mozilla;
+
+template <size_t N>
+static void ToBase64EncodedStringArray(nsCString (&aInput)[N],
+ nsTArray<nsCString>& aEncodedArray) {
+ for (size_t i = 0; i < N; i++) {
+ nsCString encoded;
+ nsresult rv = mozilla::Base64Encode(aInput[i], encoded);
+ NS_ENSURE_SUCCESS_VOID(rv);
+ aEncodedArray.AppendElement(std::move(encoded));
+ }
+}
+
+TEST(UrlClassifierFindFullHash, Request)
+{
+ nsUrlClassifierUtils* urlUtil = nsUrlClassifierUtils::GetInstance();
+
+ nsTArray<nsCString> listNames;
+ listNames.AppendElement("moztest-phish-proto");
+ listNames.AppendElement("moztest-unwanted-proto");
+
+ nsCString listStates[] = {nsCString("sta\x00te1", 7),
+ nsCString("sta\x00te2", 7)};
+ nsTArray<nsCString> listStateArray;
+ ToBase64EncodedStringArray(listStates, listStateArray);
+
+ nsCString prefixes[] = {nsCString("\x00\x00\x00\x01", 4),
+ nsCString("\x00\x00\x00\x00\x01", 5),
+ nsCString("\x00\xFF\x00\x01", 4),
+ nsCString("\x00\xFF\x00\x01\x11\x23\xAA\xBC", 8),
+ nsCString("\x00\x00\x00\x01\x00\x01\x98", 7)};
+ nsTArray<nsCString> prefixArray;
+ ToBase64EncodedStringArray(prefixes, prefixArray);
+
+ nsCString requestBase64;
+ nsresult rv;
+ rv = urlUtil->MakeFindFullHashRequestV4(listNames, listStateArray,
+ prefixArray, requestBase64);
+ ASSERT_NS_SUCCEEDED(rv);
+
+ // Base64 URL decode first.
+ FallibleTArray<uint8_t> requestBinary;
+ rv = Base64URLDecode(requestBase64, Base64URLDecodePaddingPolicy::Require,
+ requestBinary);
+ ASSERT_NS_SUCCEEDED(rv);
+
+ // Parse the FindFullHash binary and compare with the expected values.
+ FindFullHashesRequest r;
+ ASSERT_TRUE(r.ParseFromArray(&requestBinary[0], requestBinary.Length()));
+
+ // Compare client states.
+ ASSERT_EQ(r.client_states_size(), (int)ArrayLength(listStates));
+ for (int i = 0; i < r.client_states_size(); i++) {
+ auto s = r.client_states(i);
+ ASSERT_TRUE(listStates[i].Equals(nsCString(s.c_str(), s.size())));
+ }
+
+ auto threatInfo = r.threat_info();
+
+ // Compare threat types.
+ ASSERT_EQ(threatInfo.threat_types_size(), (int)ArrayLength(listStates));
+ for (int i = 0; i < threatInfo.threat_types_size(); i++) {
+ uint32_t expectedThreatType;
+ rv =
+ urlUtil->ConvertListNameToThreatType(listNames[i], &expectedThreatType);
+ ASSERT_NS_SUCCEEDED(rv);
+ ASSERT_EQ(threatInfo.threat_types(i), (int)expectedThreatType);
+ }
+
+ // Compare prefixes.
+ ASSERT_EQ(threatInfo.threat_entries_size(), (int)ArrayLength(prefixes));
+ for (int i = 0; i < threatInfo.threat_entries_size(); i++) {
+ auto p = threatInfo.threat_entries(i).hash();
+ ASSERT_TRUE(prefixes[i].Equals(nsCString(p.c_str(), p.size())));
+ }
+}
+
+/////////////////////////////////////////////////////////////
+// Following is to test parsing the gethash response.
+
+namespace {
+
+// safebrowsing::Duration manipulation.
+struct MyDuration {
+ uint32_t mSecs;
+ uint32_t mNanos;
+};
+void PopulateDuration(Duration& aDest, const MyDuration& aSrc) {
+ aDest.set_seconds(aSrc.mSecs);
+ aDest.set_nanos(aSrc.mNanos);
+}
+
+// The expected match data.
+static MyDuration EXPECTED_MIN_WAIT_DURATION = {12, 10};
+static MyDuration EXPECTED_NEG_CACHE_DURATION = {120, 9};
+static const struct ExpectedMatch {
+ nsCString mCompleteHash;
+ ThreatType mThreatType;
+ MyDuration mPerHashCacheDuration;
+} EXPECTED_MATCH[] = {
+ {nsCString("01234567890123456789012345678901"),
+ SOCIAL_ENGINEERING_PUBLIC,
+ {8, 500}},
+ {nsCString("12345678901234567890123456789012"),
+ SOCIAL_ENGINEERING_PUBLIC,
+ {7, 100}},
+ {nsCString("23456789012345678901234567890123"),
+ SOCIAL_ENGINEERING_PUBLIC,
+ {1, 20}},
+};
+
+class MyParseCallback final : public nsIUrlClassifierParseFindFullHashCallback {
+ public:
+ NS_DECL_ISUPPORTS
+
+ explicit MyParseCallback(uint32_t& aCallbackCount)
+ : mCallbackCount(aCallbackCount) {}
+
+ NS_IMETHOD
+ OnCompleteHashFound(const nsACString& aCompleteHash,
+ const nsACString& aTableNames,
+ uint32_t aPerHashCacheDuration) override {
+ Verify(aCompleteHash, aTableNames, aPerHashCacheDuration);
+
+ return NS_OK;
+ }
+
+ NS_IMETHOD
+ OnResponseParsed(uint32_t aMinWaitDuration,
+ uint32_t aNegCacheDuration) override {
+ VerifyDuration(aMinWaitDuration / 1000, EXPECTED_MIN_WAIT_DURATION);
+ VerifyDuration(aNegCacheDuration, EXPECTED_NEG_CACHE_DURATION);
+
+ return NS_OK;
+ }
+
+ private:
+ void Verify(const nsACString& aCompleteHash, const nsACString& aTableNames,
+ uint32_t aPerHashCacheDuration) {
+ auto expected = EXPECTED_MATCH[mCallbackCount];
+
+ ASSERT_TRUE(aCompleteHash.Equals(expected.mCompleteHash));
+
+ // Verify aTableNames
+ nsUrlClassifierUtils* urlUtil = nsUrlClassifierUtils::GetInstance();
+
+ nsCString tableNames;
+ nsresult rv =
+ urlUtil->ConvertThreatTypeToListNames(expected.mThreatType, tableNames);
+ ASSERT_NS_SUCCEEDED(rv);
+ ASSERT_TRUE(aTableNames.Equals(tableNames));
+
+ VerifyDuration(aPerHashCacheDuration, expected.mPerHashCacheDuration);
+
+ mCallbackCount++;
+ }
+
+ void VerifyDuration(uint32_t aToVerify, const MyDuration& aExpected) {
+ ASSERT_TRUE(aToVerify == aExpected.mSecs);
+ }
+
+ ~MyParseCallback() = default;
+
+ uint32_t& mCallbackCount;
+};
+
+NS_IMPL_ISUPPORTS(MyParseCallback, nsIUrlClassifierParseFindFullHashCallback)
+
+} // end of unnamed namespace.
+
+TEST(UrlClassifierFindFullHash, ParseRequest)
+{
+ // Build response.
+ FindFullHashesResponse r;
+
+ // Init response-wise durations.
+ auto minWaitDuration = r.mutable_minimum_wait_duration();
+ PopulateDuration(*minWaitDuration, EXPECTED_MIN_WAIT_DURATION);
+ auto negCacheDuration = r.mutable_negative_cache_duration();
+ PopulateDuration(*negCacheDuration, EXPECTED_NEG_CACHE_DURATION);
+
+ // Init matches.
+ for (uint32_t i = 0; i < ArrayLength(EXPECTED_MATCH); i++) {
+ auto expected = EXPECTED_MATCH[i];
+ auto match = r.mutable_matches()->Add();
+ match->set_threat_type(expected.mThreatType);
+ match->mutable_threat()->set_hash(expected.mCompleteHash.BeginReading(),
+ expected.mCompleteHash.Length());
+ auto perHashCacheDuration = match->mutable_cache_duration();
+ PopulateDuration(*perHashCacheDuration, expected.mPerHashCacheDuration);
+ }
+ std::string s;
+ r.SerializeToString(&s);
+
+ uint32_t callbackCount = 0;
+ nsCOMPtr<nsIUrlClassifierParseFindFullHashCallback> callback =
+ new MyParseCallback(callbackCount);
+
+ nsUrlClassifierUtils* urlUtil = nsUrlClassifierUtils::GetInstance();
+ nsresult rv = urlUtil->ParseFindFullHashResponseV4(
+ nsCString(s.c_str(), s.size()), callback);
+ NS_ENSURE_SUCCESS_VOID(rv);
+
+ ASSERT_EQ(callbackCount, ArrayLength(EXPECTED_MATCH));
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestLookupCacheV4.cpp b/toolkit/components/url-classifier/tests/gtest/TestLookupCacheV4.cpp
new file mode 100644
index 0000000000..e1482c1416
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestLookupCacheV4.cpp
@@ -0,0 +1,152 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "Classifier.h"
+#include "LookupCacheV4.h"
+#include "nsAppDirectoryServiceDefs.h"
+
+#include "Common.h"
+
+#define GTEST_SAFEBROWSING_DIR "safebrowsing"_ns
+
+static void TestHasPrefix(const nsCString& aURL, bool aExpectedHas,
+ bool aExpectedComplete) {
+ _PrefixArray array = {CreatePrefixFromURL("bravo.com/", 32),
+ CreatePrefixFromURL("browsing.com/", 8),
+ CreatePrefixFromURL("gound.com/", 5),
+ CreatePrefixFromURL("small.com/", 4)};
+
+ nsCOMPtr<nsIFile> file;
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
+ file->AppendNative(GTEST_SAFEBROWSING_DIR);
+
+ RunTestInNewThread([&]() -> void {
+ RefPtr<LookupCache> cache = SetupLookupCache<LookupCacheV4>(array, file);
+
+ Completion lookupHash;
+ lookupHash.FromPlaintext(aURL);
+
+ bool has, confirmed;
+ uint32_t matchLength;
+ // Freshness is not used in V4 so we just put dummy values here.
+ TableFreshnessMap dummy;
+ nsresult rv = cache->Has(lookupHash, &has, &matchLength, &confirmed);
+
+ EXPECT_EQ(rv, NS_OK);
+ EXPECT_EQ(has, aExpectedHas);
+ EXPECT_EQ(matchLength == COMPLETE_SIZE, aExpectedComplete);
+ EXPECT_EQ(confirmed, false);
+
+ cache->ClearAll();
+ });
+}
+
+TEST(UrlClassifierLookupCacheV4, HasComplete)
+{ TestHasPrefix("bravo.com/"_ns, true, true); }
+
+TEST(UrlClassifierLookupCacheV4, HasPrefix)
+{ TestHasPrefix("browsing.com/"_ns, true, false); }
+
+TEST(UrlClassifierLookupCacheV4, Nomatch)
+{ TestHasPrefix("nomatch.com/"_ns, false, false); }
+
+// Test an existing .pset should be removed after .vlpset is written
+TEST(UrlClassifierLookupCacheV4, RemoveOldPset)
+{
+ nsCOMPtr<nsIFile> oldPsetFile;
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
+ getter_AddRefs(oldPsetFile));
+ oldPsetFile->AppendNative("safebrowsing"_ns);
+ oldPsetFile->AppendNative(GTEST_TABLE_V4 + ".pset"_ns);
+
+ nsCOMPtr<nsIFile> newPsetFile;
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
+ getter_AddRefs(newPsetFile));
+ newPsetFile->AppendNative("safebrowsing"_ns);
+ newPsetFile->AppendNative(GTEST_TABLE_V4 + ".vlpset"_ns);
+
+ // Create the legacy .pset file
+ nsresult rv = oldPsetFile->Create(nsIFile::NORMAL_FILE_TYPE, 0666);
+ EXPECT_EQ(rv, NS_OK);
+
+ bool exists;
+ rv = oldPsetFile->Exists(&exists);
+ EXPECT_EQ(rv, NS_OK);
+ EXPECT_EQ(exists, true);
+
+ // Setup the data in lookup cache and write its data to disk
+ RefPtr<Classifier> classifier = GetClassifier();
+ _PrefixArray array = {CreatePrefixFromURL("entry.com/", 4)};
+ rv = BuildLookupCache(classifier, GTEST_TABLE_V4, array);
+ EXPECT_EQ(rv, NS_OK);
+
+ RefPtr<LookupCache> cache = classifier->GetLookupCache(GTEST_TABLE_V4, false);
+ rv = cache->WriteFile();
+ EXPECT_EQ(rv, NS_OK);
+
+ // .vlpset should exist while .pset should be removed
+ rv = newPsetFile->Exists(&exists);
+ EXPECT_EQ(rv, NS_OK);
+ EXPECT_EQ(exists, true);
+
+ rv = oldPsetFile->Exists(&exists);
+ EXPECT_EQ(rv, NS_OK);
+ EXPECT_EQ(exists, false);
+
+ newPsetFile->Remove(false);
+}
+
+// Test the legacy load
+TEST(UrlClassifierLookupCacheV4, LoadOldPset)
+{
+ nsCOMPtr<nsIFile> oldPsetFile;
+
+ _PrefixArray array = {CreatePrefixFromURL("entry.com/", 4)};
+ PrefixStringMap map;
+ PrefixArrayToPrefixStringMap(array, map);
+
+ // Prepare .pset file on disk
+ {
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
+ getter_AddRefs(oldPsetFile));
+ oldPsetFile->AppendNative("safebrowsing"_ns);
+ oldPsetFile->AppendNative(GTEST_TABLE_V4 + ".pset"_ns);
+
+ RefPtr<VariableLengthPrefixSet> pset = new VariableLengthPrefixSet;
+ pset->SetPrefixes(map);
+
+ nsCOMPtr<nsIOutputStream> stream;
+ nsresult rv =
+ NS_NewLocalFileOutputStream(getter_AddRefs(stream), oldPsetFile);
+ EXPECT_EQ(rv, NS_OK);
+
+ rv = pset->WritePrefixes(stream);
+ EXPECT_EQ(rv, NS_OK);
+ }
+
+ // Load data from disk
+ RefPtr<Classifier> classifier = GetClassifier();
+ RefPtr<LookupCache> cache = classifier->GetLookupCache(GTEST_TABLE_V4, false);
+
+ RefPtr<LookupCacheV4> cacheV4 = LookupCache::Cast<LookupCacheV4>(cache);
+ CheckContent(cacheV4, array);
+
+ oldPsetFile->Remove(false);
+}
+
+TEST(UrlClassifierLookupCacheV4, BuildAPI)
+{
+ _PrefixArray init = {_Prefix("alph")};
+ RefPtr<LookupCacheV4> cache = SetupLookupCache<LookupCacheV4>(init);
+
+ _PrefixArray update = {_Prefix("beta")};
+ PrefixStringMap map;
+ PrefixArrayToPrefixStringMap(update, map);
+
+ cache->Build(map);
+ EXPECT_TRUE(map.IsEmpty());
+
+ CheckContent(cache, update);
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestPerProviderDirectory.cpp b/toolkit/components/url-classifier/tests/gtest/TestPerProviderDirectory.cpp
new file mode 100644
index 0000000000..a79ab643f4
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestPerProviderDirectory.cpp
@@ -0,0 +1,127 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "HashStore.h"
+#include "LookupCache.h"
+#include "LookupCacheV4.h"
+#include "nsAppDirectoryServiceDefs.h"
+
+#include "Common.h"
+
+namespace mozilla {
+namespace safebrowsing {
+
+class PerProviderDirectoryTestUtils {
+ public:
+ template <typename T>
+ static nsIFile* InspectStoreDirectory(const T& aT) {
+ return aT.mStoreDirectory;
+ }
+};
+
+} // end of namespace safebrowsing
+} // end of namespace mozilla
+
+template <typename T>
+static void VerifyPrivateStorePath(T* target, const nsCString& aTableName,
+ const nsCString& aProvider,
+ const nsCOMPtr<nsIFile>& aRootDir,
+ bool aUsePerProviderStore) {
+ nsString rootStorePath;
+ nsresult rv = aRootDir->GetPath(rootStorePath);
+ EXPECT_EQ(rv, NS_OK);
+
+ nsIFile* privateStoreDirectory =
+ PerProviderDirectoryTestUtils::InspectStoreDirectory(*target);
+
+ nsString privateStorePath;
+ rv = privateStoreDirectory->GetPath(privateStorePath);
+ ASSERT_EQ(rv, NS_OK);
+
+ nsString expectedPrivateStorePath = rootStorePath;
+
+ if (aUsePerProviderStore) {
+ // Use API to append "provider" to the root directoy path
+ nsCOMPtr<nsIFile> expectedPrivateStoreDir;
+ rv = aRootDir->Clone(getter_AddRefs(expectedPrivateStoreDir));
+ ASSERT_EQ(rv, NS_OK);
+
+ expectedPrivateStoreDir->AppendNative(aProvider);
+ rv = expectedPrivateStoreDir->GetPath(expectedPrivateStorePath);
+ ASSERT_EQ(rv, NS_OK);
+ }
+
+ printf("table: %s\nprovider: %s\nroot path: %s\nprivate path: %s\n\n",
+ aTableName.get(), aProvider.get(),
+ NS_ConvertUTF16toUTF8(rootStorePath).get(),
+ NS_ConvertUTF16toUTF8(privateStorePath).get());
+
+ ASSERT_TRUE(privateStorePath == expectedPrivateStorePath);
+}
+
+TEST(UrlClassifierPerProviderDirectory, LookupCache)
+{
+ nsCOMPtr<nsIFile> rootDir;
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(rootDir));
+
+ RunTestInNewThread([&]() -> void {
+ // For V2 tables (NOT ending with '-proto'), root directory should be
+ // used as the private store.
+ {
+ nsAutoCString table("goog-phish-shavar");
+ nsAutoCString provider("google");
+ RefPtr<LookupCacheV2> lc = new LookupCacheV2(table, provider, rootDir);
+ VerifyPrivateStorePath<LookupCacheV2>(lc, table, provider, rootDir,
+ false);
+ }
+
+ // For V4 tables, if provider is found, use per-provider subdirectory;
+ // If not found, use root directory.
+ {
+ nsAutoCString table("goog-noprovider-proto");
+ nsAutoCString provider("");
+ RefPtr<LookupCacheV4> lc = new LookupCacheV4(table, provider, rootDir);
+ VerifyPrivateStorePath<LookupCacheV4>(lc, table, provider, rootDir,
+ false);
+ }
+ {
+ nsAutoCString table("goog-phish-proto");
+ nsAutoCString provider("google4");
+ RefPtr<LookupCacheV4> lc = new LookupCacheV4(table, provider, rootDir);
+ VerifyPrivateStorePath<LookupCacheV4>(lc, table, provider, rootDir, true);
+ }
+ });
+}
+
+TEST(UrlClassifierPerProviderDirectory, HashStore)
+{
+ nsCOMPtr<nsIFile> rootDir;
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(rootDir));
+
+ RunTestInNewThread([&]() -> void {
+ // For V2 tables (NOT ending with '-proto'), root directory should be
+ // used as the private store.
+ {
+ nsAutoCString table("goog-phish-shavar");
+ nsAutoCString provider("google");
+ HashStore hs(table, provider, rootDir);
+ VerifyPrivateStorePath(&hs, table, provider, rootDir, false);
+ }
+ // For V4 tables, if provider is found, use per-provider subdirectory;
+ // If not found, use root directory.
+ {
+ nsAutoCString table("goog-noprovider-proto");
+ nsAutoCString provider("");
+ HashStore hs(table, provider, rootDir);
+ VerifyPrivateStorePath(&hs, table, provider, rootDir, false);
+ }
+ {
+ nsAutoCString table("goog-phish-proto");
+ nsAutoCString provider("google4");
+ HashStore hs(table, provider, rootDir);
+ VerifyPrivateStorePath(&hs, table, provider, rootDir, true);
+ }
+ });
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestPrefixSet.cpp b/toolkit/components/url-classifier/tests/gtest/TestPrefixSet.cpp
new file mode 100644
index 0000000000..6d83cd02f6
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestPrefixSet.cpp
@@ -0,0 +1,87 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/Preferences.h"
+#include "nsString.h"
+#include "nsUrlClassifierPrefixSet.h"
+
+#include "Common.h"
+
+// This function generate N 4 byte prefixes.
+static void RandomPrefixes(uint32_t N, nsTArray<uint32_t>& array) {
+ array.Clear();
+ array.SetCapacity(N);
+
+ for (uint32_t i = 0; i < N; i++) {
+ bool added = false;
+
+ while (!added) {
+ nsAutoCString prefix;
+ char* dst = prefix.BeginWriting();
+ for (uint32_t j = 0; j < 4; j++) {
+ dst[j] = static_cast<char>(rand() % 256);
+ }
+
+ const char* src = prefix.BeginReading();
+ uint32_t data = 0;
+ memcpy(&data, src, sizeof(data));
+
+ if (!array.Contains(data)) {
+ array.AppendElement(data);
+ added = true;
+ }
+ }
+ }
+
+ struct Comparator {
+ bool LessThan(const uint32_t& aA, const uint32_t& aB) const {
+ return aA < aB;
+ }
+
+ bool Equals(const uint32_t& aA, const uint32_t& aB) const {
+ return aA == aB;
+ }
+ };
+
+ array.Sort(Comparator());
+}
+
+void RunTest(uint32_t aTestSize) {
+ RefPtr<nsUrlClassifierPrefixSet> prefixSet = new nsUrlClassifierPrefixSet();
+ nsTArray<uint32_t> array;
+
+ RandomPrefixes(aTestSize, array);
+
+ nsresult rv = prefixSet->SetPrefixes(array.Elements(), array.Length());
+ ASSERT_NS_SUCCEEDED(rv);
+
+ for (uint32_t i = 0; i < array.Length(); i++) {
+ uint32_t value = 0;
+ rv = prefixSet->GetPrefixByIndex(i, &value);
+ ASSERT_NS_SUCCEEDED(rv);
+
+ ASSERT_TRUE(value == array[i]);
+ }
+}
+
+TEST(URLClassifierPrefixSet, GetTargetPrefixWithLargeSet)
+{
+ // Make sure the delta algorithm will be used.
+ static const char prefKey[] = "browser.safebrowsing.prefixset_max_array_size";
+ mozilla::Preferences::SetUint(prefKey, 10000);
+
+ // Ideally, we should test more than 512 * 1024 entries. But, it will make the
+ // test too long. So, we test 100k entries instead.
+ RunTest(100000);
+}
+
+TEST(URLClassifierPrefixSet, GetTargetPrefixWithSmallSet)
+{
+ // Make sure the delta algorithm won't be used.
+ static const char prefKey[] = "browser.safebrowsing.prefixset_max_array_size";
+ mozilla::Preferences::SetUint(prefKey, 10000);
+
+ RunTest(1000);
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestProtocolParser.cpp b/toolkit/components/url-classifier/tests/gtest/TestProtocolParser.cpp
new file mode 100644
index 0000000000..2155140b4d
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestProtocolParser.cpp
@@ -0,0 +1,140 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/EndianUtils.h"
+#include "ProtocolParser.h"
+
+#include "Common.h"
+
+typedef FetchThreatListUpdatesResponse_ListUpdateResponse ListUpdateResponse;
+
+static bool InitUpdateResponse(ListUpdateResponse* aUpdateResponse,
+ ThreatType aThreatType, const nsACString& aState,
+ const nsACString& aChecksum, bool isFullUpdate,
+ const nsTArray<uint32_t>& aFixedLengthPrefixes,
+ bool aDoPrefixEncoding) {
+ aUpdateResponse->set_threat_type(aThreatType);
+ aUpdateResponse->set_new_client_state(aState.BeginReading(), aState.Length());
+ aUpdateResponse->mutable_checksum()->set_sha256(aChecksum.BeginReading(),
+ aChecksum.Length());
+ aUpdateResponse->set_response_type(isFullUpdate
+ ? ListUpdateResponse::FULL_UPDATE
+ : ListUpdateResponse::PARTIAL_UPDATE);
+
+ auto additions = aUpdateResponse->mutable_additions()->Add();
+
+ if (!aDoPrefixEncoding) {
+ additions->set_compression_type(RAW);
+ auto rawHashes = additions->mutable_raw_hashes();
+ rawHashes->set_prefix_size(4);
+ auto prefixes = rawHashes->mutable_raw_hashes();
+ for (auto p : aFixedLengthPrefixes) {
+ char buffer[4];
+ mozilla::NativeEndian::copyAndSwapToBigEndian(buffer, &p, 1);
+ prefixes->append(buffer, 4);
+ }
+ return true;
+ }
+
+ if (1 != aFixedLengthPrefixes.Length()) {
+ printf("This function only supports single value encoding.\n");
+ return false;
+ }
+
+ uint32_t firstValue = aFixedLengthPrefixes[0];
+ additions->set_compression_type(RICE);
+ auto riceHashes = additions->mutable_rice_hashes();
+ riceHashes->set_first_value(firstValue);
+ riceHashes->set_num_entries(0);
+
+ return true;
+}
+
+static void DumpBinary(const nsACString& aBinary) {
+ nsCString s;
+ for (size_t i = 0; i < aBinary.Length(); i++) {
+ s.AppendPrintf("\\x%.2X", (uint8_t)aBinary[i]);
+ }
+ printf("%s\n", s.get());
+}
+
+TEST(UrlClassifierProtocolParser, UpdateWait)
+{
+ // Top level response which contains a list of update response
+ // for different lists.
+ FetchThreatListUpdatesResponse response;
+
+ auto r = response.mutable_list_update_responses()->Add();
+ InitUpdateResponse(r, SOCIAL_ENGINEERING_PUBLIC, nsCString("sta\x00te", 6),
+ nsCString("check\x0sum", 9), true, {0, 1, 2, 3},
+ false /* aDoPrefixEncoding */);
+
+ // Set min wait duration.
+ auto minWaitDuration = response.mutable_minimum_wait_duration();
+ minWaitDuration->set_seconds(8);
+ minWaitDuration->set_nanos(1 * 1000000000);
+
+ std::string s;
+ response.SerializeToString(&s);
+
+ DumpBinary(nsCString(s.c_str(), s.length()));
+
+ ProtocolParser* p = new ProtocolParserProtobuf();
+ p->AppendStream(nsCString(s.c_str(), s.length()));
+ p->End();
+ ASSERT_EQ(p->UpdateWaitSec(), 9u);
+ delete p;
+}
+
+TEST(UrlClassifierProtocolParser, SingleValueEncoding)
+{
+ // Top level response which contains a list of update response
+ // for different lists.
+ FetchThreatListUpdatesResponse response;
+
+ auto r = response.mutable_list_update_responses()->Add();
+
+ const char* expectedPrefix = "\x00\x01\x02\x00";
+ if (!InitUpdateResponse(r, SOCIAL_ENGINEERING_PUBLIC,
+ nsCString("sta\x00te", 6),
+ nsCString("check\x0sum", 9), true,
+ // As per spec, we should interpret the prefix as
+ // uint32 in little endian before encoding.
+ {mozilla::LittleEndian::readUint32(expectedPrefix)},
+ true /* aDoPrefixEncoding */)) {
+ printf("Failed to initialize update response.");
+ ASSERT_TRUE(false);
+ return;
+ }
+
+ // Set min wait duration.
+ auto minWaitDuration = response.mutable_minimum_wait_duration();
+ minWaitDuration->set_seconds(8);
+ minWaitDuration->set_nanos(1 * 1000000000);
+
+ std::string s;
+ response.SerializeToString(&s);
+
+ // Feed data to the protocol parser.
+ ProtocolParser* p = new ProtocolParserProtobuf();
+ p->SetRequestedTables({nsCString("googpub-phish-proto")});
+ p->AppendStream(nsCString(s.c_str(), s.length()));
+ p->End();
+
+ const TableUpdateArray& tus = p->GetTableUpdates();
+ RefPtr<const TableUpdateV4> tuv4 = TableUpdate::Cast<TableUpdateV4>(tus[0]);
+ auto& prefixMap = tuv4->Prefixes();
+ for (const auto& entry : prefixMap) {
+ // This prefix map should contain only a single 4-byte prefixe.
+ ASSERT_EQ(entry.GetKey(), 4u);
+
+ // The fixed-length prefix string from ProtocolParser should
+ // exactly match the expected prefix string.
+ nsCString* prefix = entry.GetWeak();
+ ASSERT_TRUE(prefix->Equals(nsCString(expectedPrefix, 4)));
+ }
+
+ delete p;
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestRiceDeltaDecoder.cpp b/toolkit/components/url-classifier/tests/gtest/TestRiceDeltaDecoder.cpp
new file mode 100644
index 0000000000..9130cfe385
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestRiceDeltaDecoder.cpp
@@ -0,0 +1,173 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/ArrayUtils.h"
+#include "RiceDeltaDecoder.h"
+
+#include "Common.h"
+
+namespace {
+
+struct TestingData {
+ std::vector<uint32_t> mExpectedDecoded;
+ std::vector<uint8_t> mEncoded;
+ uint32_t mRiceParameter;
+};
+
+} // namespace
+
+static bool runOneTest(TestingData& aData) {
+ RiceDeltaDecoder decoder(&aData.mEncoded[0], aData.mEncoded.size());
+
+ std::vector<uint32_t> decoded(aData.mExpectedDecoded.size());
+
+ uint32_t firstValue = aData.mExpectedDecoded[0];
+ bool rv = decoder.Decode(
+ aData.mRiceParameter, firstValue,
+ decoded.size() - 1, // # of entries (first value not included).
+ &decoded[0]);
+
+ return rv && decoded == aData.mExpectedDecoded;
+}
+
+TEST(UrlClassifierRiceDeltaDecoder, SingleEncodedValue)
+{
+ TestingData td = {{99}, {99}, 0};
+
+ ASSERT_TRUE(runOneTest(td));
+}
+
+// In this batch of tests, the encoded data would be like
+// what we originally receive from the network. See comment
+// in |runOneTest| for more detail.
+TEST(UrlClassifierRiceDeltaDecoder, Empty)
+{
+ // The following structure and testing data is copied from Chromium source
+ // code:
+ //
+ // https://chromium.googlesource.com/chromium/src.git/+/950f9975599768b6a08c7146cb4befa161be87aa/components/safe_browsing_db/v4_rice_unittest.cc#75
+ //
+ // and will be translated to our own testing format.
+
+ struct RiceDecodingTestInfo {
+ uint32_t mRiceParameter;
+ std::vector<uint32_t> mDeltas;
+ std::string mEncoded;
+
+ RiceDecodingTestInfo(uint32_t aRiceParameter,
+ const std::vector<uint32_t>& aDeltas,
+ const std::string& aEncoded)
+ : mRiceParameter(aRiceParameter),
+ mDeltas(aDeltas),
+ mEncoded(aEncoded) {}
+ };
+
+ // Copyright 2016 The Chromium Authors. All rights reserved.
+ // Use of this source code is governed by a BSD-style license that can be
+ // found in the media/webrtc/trunk/webrtc/LICENSE.
+
+ // ----- Start of Chromium test code ----
+ const std::vector<RiceDecodingTestInfo> TESTING_DATA_CHROMIUM = {
+ RiceDecodingTestInfo(2, {15, 9}, "\xf7\x2"),
+ RiceDecodingTestInfo(
+ 28, {1777762129, 2093280223, 924369848},
+ "\xbf\xa8\x3f\xfb\xfc\xfb\x5e\x27\xe6\xc3\x1d\xc6\x38"),
+ RiceDecodingTestInfo(
+ 28, {62763050, 1046523781, 192522171, 1800511020, 4442775, 582142548},
+ "\x54\x60\x7b\xe7\x0a\x5f\xc1\xdc\xee\x69\xde"
+ "\xfe\x58\x3c\xa3\xd6\xa5\xf2\x10\x8c\x4a\x59"
+ "\x56\x00"),
+ RiceDecodingTestInfo(
+ 28,
+ {26067715, 344823336, 8420095, 399843890, 95029378, 731622412,
+ 35811335, 1047558127, 1117722715, 78698892},
+ "\x06\x86\x1b\x23\x14\xcb\x46\xf2\xaf\x07\x08\xc9\x88\x54\x1f\x41\x04"
+ "\xd5\x1a\x03\xeb\xe6\x3a\x80\x13\x91\x7b\xbf\x83\xf3\xb7\x85\xf1\x29"
+ "\x18\xb3\x61\x09"),
+ RiceDecodingTestInfo(
+ 27,
+ {225846818, 328287420, 166748623, 29117720, 552397365, 350353215,
+ 558267528, 4738273, 567093445, 28563065, 55077698, 73091685,
+ 339246010, 98242620, 38060941, 63917830, 206319759, 137700744},
+ "\x89\x98\xd8\x75\xbc\x44\x91\xeb\x39\x0c\x3e\x30\x9a\x78\xf3\x6a\xd4"
+ "\xd9\xb1\x9f\xfb\x70\x3e\x44\x3e\xa3\x08\x67\x42\xc2\x2b\x46\x69\x8e"
+ "\x3c\xeb\xd9\x10\x5a\x43\x9a\x32\xa5\x2d\x4e\x77\x0f\x87\x78\x20\xb6"
+ "\xab\x71\x98\x48\x0c\x9e\x9e\xd7\x23\x0c\x13\x43\x2c\xa9\x01"),
+ RiceDecodingTestInfo(
+ 28,
+ {339784008, 263128563, 63871877, 69723256, 826001074, 797300228,
+ 671166008, 207712688},
+ std::string("\x21\xc5\x02\x91\xf9\x82\xd7\x57\xb8\xe9\x3c\xf0\xc8\x4f"
+ "\xe8\x64\x8d\x77\x62\x04\xd6\x85\x3f\x1c\x97\x00\x04\x1b"
+ "\x17\xc6",
+ 30)),
+ RiceDecodingTestInfo(
+ 28,
+ {471820069, 196333855, 855579133, 122737976, 203433838, 85354544,
+ 1307949392, 165938578, 195134475, 553930435, 49231136},
+ "\x95\x9c\x7d\xb0\x8f\xe8\xd9\xbd\xfe\x8c\x7f\x81\x53\x0d\x75\xdc\x4e"
+ "\x40\x18\x0c\x9a\x45\x3d\xa8\xdc\xfa\x26\x59\x40\x9e\x16\x08\x43\x77"
+ "\xc3\x4e\x04\x01\xa4\xe6\x5d\x00"),
+ RiceDecodingTestInfo(
+ 27,
+ {87336845, 129291033, 30906211, 433549264, 30899891, 53207875,
+ 11959529, 354827862, 82919275, 489637251, 53561020, 336722992,
+ 408117728, 204506246, 188216092, 9047110, 479817359, 230317256},
+ "\x1a\x4f\x69\x2a\x63\x9a\xf6\xc6\x2e\xaf\x73\xd0\x6f\xd7\x31\xeb\x77"
+ "\x1d\x43\xe3\x2b\x93\xce\x67\x8b\x59\xf9\x98\xd4\xda\x4f\x3c\x6f\xb0"
+ "\xe8\xa5\x78\x8d\x62\x36\x18\xfe\x08\x1e\x78\xd8\x14\x32\x24\x84\x61"
+ "\x1c\xf3\x37\x63\xc4\xa0\x88\x7b\x74\xcb\x64\xc8\x5c\xba\x05"),
+ RiceDecodingTestInfo(
+ 28,
+ {297968956, 19709657, 259702329, 76998112, 1023176123, 29296013,
+ 1602741145, 393745181, 177326295, 55225536, 75194472},
+ "\xf1\x94\x0a\x87\x6c\x5f\x96\x90\xe3\xab\xf7\xc0\xcb\x2d\xe9\x76\xdb"
+ "\xf8\x59\x63\xc1\x6f\x7c\x99\xe3\x87\x5f\xc7\x04\xde\xb9\x46\x8e\x54"
+ "\xc0\xac\x4a\x03\x0d\x6c\x8f\x00"),
+ RiceDecodingTestInfo(
+ 28,
+ {532220688, 780594691, 436816483, 163436269, 573044456, 1069604,
+ 39629436, 211410997, 227714491, 381562898, 75610008, 196754597,
+ 40310339, 15204118, 99010842},
+ "\x41\x2c\xe4\xfe\x06\xdc\x0d\xbd\x31\xa5\x04\xd5\x6e\xdd\x9b\x43\xb7"
+ "\x3f\x11\x24\x52\x10\x80\x4f\x96\x4b\xd4\x80\x67\xb2\xdd\x52\xc9\x4e"
+ "\x02\xc6\xd7\x60\xde\x06\x92\x52\x1e\xdd\x35\x64\x71\x26\x2c\xfe\xcf"
+ "\x81\x46\xb2\x79\x01"),
+ RiceDecodingTestInfo(
+ 28,
+ {219354713, 389598618, 750263679, 554684211, 87381124, 4523497,
+ 287633354, 801308671, 424169435, 372520475, 277287849},
+ "\xb2\x2c\x26\x3a\xcd\x66\x9c\xdb\x5f\x07\x2e\x6f\xe6\xf9\x21\x10\x52"
+ "\xd5\x94\xf4\x82\x22\x48\xf9\x9d\x24\xf6\xff\x2f\xfc\x6d\x3f\x21\x65"
+ "\x1b\x36\x34\x56\xea\xc4\x21\x00"),
+ };
+
+ // ----- End of Chromium test code ----
+
+ for (auto tdc : TESTING_DATA_CHROMIUM) {
+ // Populate chromium testing data to our native testing data struct.
+ TestingData d;
+
+ d.mRiceParameter = tdc.mRiceParameter; // Populate rice parameter.
+
+ // Populate encoded data from std::string to vector<uint8>.
+ d.mEncoded.resize(tdc.mEncoded.size());
+ memcpy(&d.mEncoded[0], tdc.mEncoded.c_str(), tdc.mEncoded.size());
+
+ // Populate deltas to expected decoded data. The first value would be just
+ // set to an arbitrary value, say 7, to avoid any assumption to the
+ // first value in the implementation.
+ d.mExpectedDecoded.resize(tdc.mDeltas.size() + 1);
+ for (size_t i = 0; i < d.mExpectedDecoded.size(); i++) {
+ if (0 == i) {
+ d.mExpectedDecoded[i] = 7; // "7" is an arbitrary starting value
+ } else {
+ d.mExpectedDecoded[i] = d.mExpectedDecoded[i - 1] + tdc.mDeltas[i - 1];
+ }
+ }
+
+ ASSERT_TRUE(runOneTest(d));
+ }
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestSafeBrowsingProtobuf.cpp b/toolkit/components/url-classifier/tests/gtest/TestSafeBrowsingProtobuf.cpp
new file mode 100644
index 0000000000..aa15344324
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestSafeBrowsingProtobuf.cpp
@@ -0,0 +1,30 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "safebrowsing.pb.h"
+
+#include "Common.h"
+
+TEST(UrlClassifierProtobuf, Empty)
+{
+ using namespace mozilla::safebrowsing;
+
+ const std::string CLIENT_ID = "firefox";
+
+ // Construct a simple update request.
+ FetchThreatListUpdatesRequest r;
+ r.set_allocated_client(new ClientInfo());
+ r.mutable_client()->set_client_id(CLIENT_ID);
+
+ // Then serialize.
+ std::string s;
+ r.SerializeToString(&s);
+
+ // De-serialize.
+ FetchThreatListUpdatesRequest r2;
+ r2.ParseFromString(s);
+
+ ASSERT_EQ(r2.client().client_id(), CLIENT_ID);
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestSafebrowsingHash.cpp b/toolkit/components/url-classifier/tests/gtest/TestSafebrowsingHash.cpp
new file mode 100644
index 0000000000..2fc4c793f7
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestSafebrowsingHash.cpp
@@ -0,0 +1,58 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/EndianUtils.h"
+
+#include "Common.h"
+
+TEST(UrlClassifierHash, ToFromUint32)
+{
+ using namespace mozilla::safebrowsing;
+
+ // typedef SafebrowsingHash<PREFIX_SIZE, PrefixComparator> Prefix;
+ // typedef nsTArray<Prefix> PrefixArray;
+
+ const char PREFIX_RAW[4] = {0x1, 0x2, 0x3, 0x4};
+ uint32_t PREFIX_UINT32;
+ memcpy(&PREFIX_UINT32, PREFIX_RAW, 4);
+
+ Prefix p;
+ p.Assign(nsCString(PREFIX_RAW, 4));
+ ASSERT_EQ(p.ToUint32(), PREFIX_UINT32);
+
+ p.FromUint32(PREFIX_UINT32);
+ ASSERT_EQ(memcmp(PREFIX_RAW, p.buf, 4), 0);
+}
+
+TEST(UrlClassifierHash, Compare)
+{
+ using namespace mozilla;
+ using namespace mozilla::safebrowsing;
+
+ Prefix p1, p2, p3;
+
+ // The order of p1,p2,p3 is "p1 == p3 < p2"
+#if MOZ_LITTLE_ENDIAN()
+ p1.Assign(nsCString("\x01\x00\x00\x00", 4));
+ p2.Assign(nsCString("\x00\x00\x00\x01", 4));
+ p3.Assign(nsCString("\x01\x00\x00\x00", 4));
+#else
+ p1.Assign(nsCString("\x00\x00\x00\x01", 4));
+ p2.Assign(nsCString("\x01\x00\x00\x00", 4));
+ p3.Assign(nsCString("\x00\x00\x00\x01", 4));
+#endif
+
+ // Make sure "p1 == p3 < p2" is true
+ // on both little and big endian machine.
+
+ ASSERT_EQ(p1.Compare(p2), -1);
+ ASSERT_EQ(p1.Compare(p1), 0);
+ ASSERT_EQ(p2.Compare(p1), 1);
+ ASSERT_EQ(p1.Compare(p3), 0);
+
+ ASSERT_TRUE(p1 < p2);
+ ASSERT_TRUE(p1 == p1);
+ ASSERT_TRUE(p1 == p3);
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestTable.cpp b/toolkit/components/url-classifier/tests/gtest/TestTable.cpp
new file mode 100644
index 0000000000..796f40b265
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestTable.cpp
@@ -0,0 +1,59 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsUrlClassifierDBService.h"
+
+#include "Common.h"
+
+static void TestResponseCode(const char* table, nsresult result) {
+ nsCString tableName(table);
+ ASSERT_EQ(TablesToResponse(tableName), result);
+}
+
+TEST(UrlClassifierTable, ResponseCode)
+{
+ // malware URIs.
+ TestResponseCode("goog-malware-shavar", NS_ERROR_MALWARE_URI);
+ TestResponseCode("test-malware-simple", NS_ERROR_MALWARE_URI);
+ TestResponseCode("goog-phish-shavar,test-malware-simple",
+ NS_ERROR_MALWARE_URI);
+ TestResponseCode(
+ "test-malware-simple,mozstd-track-digest256,mozplugin-block-digest256",
+ NS_ERROR_MALWARE_URI);
+
+ // phish URIs.
+ TestResponseCode("goog-phish-shavar", NS_ERROR_PHISHING_URI);
+ TestResponseCode("test-phish-simple", NS_ERROR_PHISHING_URI);
+ TestResponseCode("test-phish-simple,mozplugin-block-digest256",
+ NS_ERROR_PHISHING_URI);
+ TestResponseCode(
+ "mozstd-track-digest256,test-phish-simple,goog-unwanted-shavar",
+ NS_ERROR_PHISHING_URI);
+
+ // unwanted URIs.
+ TestResponseCode("goog-unwanted-shavar", NS_ERROR_UNWANTED_URI);
+ TestResponseCode("test-unwanted-simple", NS_ERROR_UNWANTED_URI);
+ TestResponseCode("mozplugin-unwanted-digest256,mozfull-track-digest256",
+ NS_ERROR_UNWANTED_URI);
+ TestResponseCode(
+ "test-block-simple,mozfull-track-digest256,test-unwanted-simple",
+ NS_ERROR_UNWANTED_URI);
+
+ // track URIs.
+ TestResponseCode("test-track-simple", NS_ERROR_TRACKING_URI);
+ TestResponseCode("mozstd-track-digest256", NS_ERROR_TRACKING_URI);
+ TestResponseCode("test-block-simple,mozstd-track-digest256",
+ NS_ERROR_TRACKING_URI);
+
+ // block URIs
+ TestResponseCode("test-block-simple", NS_ERROR_BLOCKED_URI);
+ TestResponseCode("mozplugin-block-digest256", NS_ERROR_BLOCKED_URI);
+ TestResponseCode("mozplugin2-block-digest256", NS_ERROR_BLOCKED_URI);
+
+ TestResponseCode("test-trackwhite-simple", NS_OK);
+ TestResponseCode("mozstd-trackwhite-digest256", NS_OK);
+ TestResponseCode("goog-badbinurl-shavar", NS_OK);
+ TestResponseCode("goog-downloadwhite-digest256", NS_OK);
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestURLsAndHashing.cpp b/toolkit/components/url-classifier/tests/gtest/TestURLsAndHashing.cpp
new file mode 100644
index 0000000000..0563c6a776
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestURLsAndHashing.cpp
@@ -0,0 +1,71 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "LookupCache.h"
+
+#include "Common.h"
+
+static void VerifyFragments(const nsACString& aURL,
+ const nsTArray<nsCString>& aExpected) {
+ nsTArray<nsCString> fragments;
+ nsresult rv = LookupCache::GetLookupFragments(aURL, &fragments);
+ ASSERT_EQ(rv, NS_OK) << "GetLookupFragments should not fail";
+
+ ASSERT_EQ(aExpected.Length(), fragments.Length())
+ << "Fragments generated from " << aURL.BeginReading()
+ << " are not the same as expected";
+
+ for (const auto& fragment : fragments) {
+ ASSERT_TRUE(aExpected.Contains(fragment))
+ << "Fragments generated from " << aURL.BeginReading()
+ << " are not the same as expected";
+ }
+}
+
+// This testcase references SafeBrowsing spec:
+// https://developers.google.com/safe-browsing/v4/urls-hashing#suffixprefix-expressions
+TEST(URLsAndHashing, FragmentURLWithQuery)
+{
+ const nsLiteralCString url("a.b.c/1/2.html?param=1");
+ nsTArray<nsCString> expect = {
+ "a.b.c/1/2.html?param=1"_ns,
+ "a.b.c/1/2.html"_ns,
+ "a.b.c/"_ns,
+ "a.b.c/1/"_ns,
+ "b.c/1/2.html?param=1"_ns,
+ "b.c/1/2.html"_ns,
+ "b.c/"_ns,
+ "b.c/1/"_ns,
+ };
+
+ VerifyFragments(url, expect);
+}
+
+// This testcase references SafeBrowsing spec:
+// https://developers.google.com/safe-browsing/v4/urls-hashing#suffixprefix-expressions
+TEST(URLsAndHashing, FragmentURLWithoutQuery)
+{
+ const nsLiteralCString url("a.b.c.d.e.f.g/1.html");
+ nsTArray<nsCString> expect = {
+ "a.b.c.d.e.f.g/1.html"_ns, "a.b.c.d.e.f.g/"_ns,
+ "c.d.e.f.g/1.html"_ns, "c.d.e.f.g/"_ns,
+ "d.e.f.g/1.html"_ns, "d.e.f.g/"_ns,
+ "e.f.g/1.html"_ns, "e.f.g/"_ns,
+ "f.g/1.html"_ns, "f.g/"_ns,
+ };
+
+ VerifyFragments(url, expect);
+}
+
+TEST(URLsAndHashing, FragmentURLEndWithoutPath)
+{
+ const nsLiteralCString url("1.2.3.4/?query=string");
+ nsTArray<nsCString> expect = {
+ "1.2.3.4/?query=string"_ns,
+ "1.2.3.4/"_ns,
+ };
+
+ VerifyFragments(url, expect);
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestUrlClassifierTableUpdateV4.cpp b/toolkit/components/url-classifier/tests/gtest/TestUrlClassifierTableUpdateV4.cpp
new file mode 100644
index 0000000000..d4dec867bd
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestUrlClassifierTableUpdateV4.cpp
@@ -0,0 +1,785 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "Classifier.h"
+#include "HashStore.h"
+#include "mozilla/Components.h"
+#include "mozilla/Unused.h"
+#include "nsAppDirectoryServiceDefs.h"
+#include "nsIFile.h"
+#include "nsThreadUtils.h"
+#include "string.h"
+#include "LookupCacheV4.h"
+#include "nsUrlClassifierUtils.h"
+
+#include "Common.h"
+
+#define GTEST_SAFEBROWSING_DIR "safebrowsing"_ns
+#define GTEST_TABLE "gtest-malware-proto"_ns
+#define GTEST_PREFIXFILE "gtest-malware-proto.vlpset"_ns
+
+// This function removes common elements of inArray and outArray from
+// outArray. This is used by partial update testcase to ensure partial update
+// data won't contain prefixes we already have.
+static void RemoveIntersection(const _PrefixArray& inArray,
+ _PrefixArray& outArray) {
+ for (uint32_t i = 0; i < inArray.Length(); i++) {
+ int32_t idx = outArray.BinaryIndexOf(inArray[i]);
+ if (idx >= 0) {
+ outArray.RemoveElementAt(idx);
+ }
+ }
+}
+
+// This fucntion removes elements from outArray by index specified in
+// removal array.
+static void RemoveElements(const nsTArray<uint32_t>& removal,
+ _PrefixArray& outArray) {
+ for (int32_t i = removal.Length() - 1; i >= 0; i--) {
+ outArray.RemoveElementAt(removal[i]);
+ }
+}
+
+static void MergeAndSortArray(const _PrefixArray& array1,
+ const _PrefixArray& array2,
+ _PrefixArray& output) {
+ output.Clear();
+ output.AppendElements(array1);
+ output.AppendElements(array2);
+ output.Sort();
+}
+
+static void CalculateSHA256(_PrefixArray& prefixArray, nsCString& sha256) {
+ prefixArray.Sort();
+
+ nsresult rv;
+ nsCOMPtr<nsICryptoHash> cryptoHash =
+ do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
+
+ cryptoHash->Init(nsICryptoHash::SHA256);
+ for (uint32_t i = 0; i < prefixArray.Length(); i++) {
+ const _Prefix& prefix = prefixArray[i];
+ cryptoHash->Update(
+ reinterpret_cast<uint8_t*>(const_cast<char*>(prefix.get())),
+ prefix.Length());
+ }
+ cryptoHash->Finish(false, sha256);
+}
+
+// N: Number of prefixes, MIN/MAX: minimum/maximum prefix size
+// This function will append generated prefixes to outArray.
+static void CreateRandomSortedPrefixArray(uint32_t N, uint32_t MIN,
+ uint32_t MAX,
+ _PrefixArray& outArray) {
+ outArray.SetCapacity(outArray.Length() + N);
+
+ const uint32_t range = (MAX - MIN + 1);
+
+ for (uint32_t i = 0; i < N; i++) {
+ uint32_t prefixSize = (rand() % range) + MIN;
+ _Prefix prefix;
+ prefix.SetLength(prefixSize);
+
+ while (true) {
+ char* dst = prefix.BeginWriting();
+ for (uint32_t j = 0; j < prefixSize; j++) {
+ dst[j] = rand() % 256;
+ }
+
+ if (!outArray.Contains(prefix)) {
+ outArray.AppendElement(prefix);
+ break;
+ }
+ }
+ }
+
+ outArray.Sort();
+}
+
+// N: Number of removal indices, MAX: maximum index
+static void CreateRandomRemovalIndices(uint32_t N, uint32_t MAX,
+ nsTArray<uint32_t>& outArray) {
+ for (uint32_t i = 0; i < N; i++) {
+ uint32_t idx = rand() % MAX;
+ if (!outArray.Contains(idx)) {
+ outArray.InsertElementSorted(idx);
+ }
+ }
+}
+
+// Function to generate TableUpdateV4.
+static void GenerateUpdateData(bool fullUpdate, PrefixStringMap& add,
+ nsTArray<uint32_t>* removal, nsCString* sha256,
+ TableUpdateArray& tableUpdates) {
+ RefPtr<TableUpdateV4> tableUpdate = new TableUpdateV4(GTEST_TABLE);
+ tableUpdate->SetFullUpdate(fullUpdate);
+
+ for (const auto& entry : add) {
+ nsCString* pstring = entry.GetWeak();
+ tableUpdate->NewPrefixes(entry.GetKey(), *pstring);
+ }
+
+ if (removal) {
+ tableUpdate->NewRemovalIndices(removal->Elements(), removal->Length());
+ }
+
+ if (sha256) {
+ std::string stdSHA256;
+ stdSHA256.assign(const_cast<char*>(sha256->BeginReading()),
+ sha256->Length());
+
+ tableUpdate->SetSHA256(stdSHA256);
+ }
+
+ tableUpdates.AppendElement(tableUpdate);
+}
+
+static void VerifyPrefixSet(PrefixStringMap& expected) {
+ // Verify the prefix set is written to disk.
+ nsCOMPtr<nsIFile> file;
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
+ file->AppendNative(GTEST_SAFEBROWSING_DIR);
+
+ RefPtr<LookupCacheV4> lookup =
+ new LookupCacheV4(GTEST_TABLE, "test"_ns, file);
+ lookup->Init();
+
+ file->AppendNative(GTEST_PREFIXFILE);
+ lookup->LoadFromFile(file);
+
+ PrefixStringMap prefixesInFile;
+ lookup->GetPrefixes(prefixesInFile);
+
+ for (const auto& entry : expected) {
+ nsCString* expectedPrefix = entry.GetWeak();
+ nsCString* resultPrefix = prefixesInFile.Get(entry.GetKey());
+
+ ASSERT_TRUE(*resultPrefix == *expectedPrefix);
+ }
+}
+
+static void Clear() {
+ nsCOMPtr<nsIFile> file;
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
+
+ RefPtr<Classifier> classifier = new Classifier();
+ classifier->Open(*file);
+ classifier->Reset();
+}
+
+static void testUpdateFail(TableUpdateArray& tableUpdates) {
+ nsresult rv = SyncApplyUpdates(tableUpdates);
+ ASSERT_NS_FAILED(rv);
+}
+
+static void testUpdate(TableUpdateArray& tableUpdates,
+ PrefixStringMap& expected) {
+ // Force nsUrlClassifierUtils loading on main thread
+ // because nsIUrlClassifierDBService will not run in advance
+ // in gtest.
+ nsUrlClassifierUtils::GetInstance();
+
+ nsresult rv = SyncApplyUpdates(tableUpdates);
+ ASSERT_TRUE(rv == NS_OK);
+ VerifyPrefixSet(expected);
+}
+
+static void testFullUpdate(PrefixStringMap& add, nsCString* sha256) {
+ TableUpdateArray tableUpdates;
+
+ GenerateUpdateData(true, add, nullptr, sha256, tableUpdates);
+
+ testUpdate(tableUpdates, add);
+}
+
+static void testPartialUpdate(PrefixStringMap& add, nsTArray<uint32_t>* removal,
+ nsCString* sha256, PrefixStringMap& expected) {
+ TableUpdateArray tableUpdates;
+ GenerateUpdateData(false, add, removal, sha256, tableUpdates);
+
+ testUpdate(tableUpdates, expected);
+}
+
+static void testOpenLookupCache() {
+ nsCOMPtr<nsIFile> file;
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
+ file->AppendNative(GTEST_SAFEBROWSING_DIR);
+
+ RunTestInNewThread([&]() -> void {
+ RefPtr<LookupCacheV4> cache =
+ new LookupCacheV4(nsCString(GTEST_TABLE), ""_ns, file);
+ nsresult rv = cache->Init();
+ ASSERT_EQ(rv, NS_OK);
+
+ rv = cache->Open();
+ ASSERT_EQ(rv, NS_OK);
+ });
+}
+
+// Tests start from here.
+TEST(UrlClassifierTableUpdateV4, FixLengthPSetFullUpdate)
+{
+ srand(time(NULL));
+
+ _PrefixArray array;
+ PrefixStringMap map;
+ nsCString sha256;
+
+ CreateRandomSortedPrefixArray(5000, 4, 4, array);
+ PrefixArrayToPrefixStringMap(array, map);
+ CalculateSHA256(array, sha256);
+
+ testFullUpdate(map, &sha256);
+
+ Clear();
+}
+
+TEST(UrlClassifierTableUpdateV4, VariableLengthPSetFullUpdate)
+{
+ _PrefixArray array;
+ PrefixStringMap map;
+ nsCString sha256;
+
+ CreateRandomSortedPrefixArray(5000, 5, 32, array);
+ PrefixArrayToPrefixStringMap(array, map);
+ CalculateSHA256(array, sha256);
+
+ testFullUpdate(map, &sha256);
+
+ Clear();
+}
+
+// This test contain both variable length prefix set and fixed-length prefix set
+TEST(UrlClassifierTableUpdateV4, MixedPSetFullUpdate)
+{
+ _PrefixArray array;
+ PrefixStringMap map;
+ nsCString sha256;
+
+ CreateRandomSortedPrefixArray(5000, 4, 4, array);
+ CreateRandomSortedPrefixArray(1000, 5, 32, array);
+ PrefixArrayToPrefixStringMap(array, map);
+ CalculateSHA256(array, sha256);
+
+ testFullUpdate(map, &sha256);
+
+ Clear();
+}
+
+TEST(UrlClassifierTableUpdateV4, PartialUpdateWithRemoval)
+{
+ _PrefixArray fArray;
+
+ // Apply a full update first.
+ {
+ PrefixStringMap fMap;
+ nsCString sha256;
+
+ CreateRandomSortedPrefixArray(10000, 4, 4, fArray);
+ CreateRandomSortedPrefixArray(2000, 5, 32, fArray);
+ PrefixArrayToPrefixStringMap(fArray, fMap);
+ CalculateSHA256(fArray, sha256);
+
+ testFullUpdate(fMap, &sha256);
+ }
+
+ // Apply a partial update with removal.
+ {
+ _PrefixArray pArray, mergedArray;
+ PrefixStringMap pMap, mergedMap;
+ nsCString sha256;
+
+ CreateRandomSortedPrefixArray(5000, 4, 4, pArray);
+ CreateRandomSortedPrefixArray(1000, 5, 32, pArray);
+ RemoveIntersection(fArray, pArray);
+ PrefixArrayToPrefixStringMap(pArray, pMap);
+
+ // Remove 1/5 of elements of original prefix set.
+ nsTArray<uint32_t> removal;
+ CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal);
+ RemoveElements(removal, fArray);
+
+ // Calculate the expected prefix map.
+ MergeAndSortArray(fArray, pArray, mergedArray);
+ PrefixArrayToPrefixStringMap(mergedArray, mergedMap);
+ CalculateSHA256(mergedArray, sha256);
+
+ testPartialUpdate(pMap, &removal, &sha256, mergedMap);
+ }
+
+ Clear();
+}
+
+TEST(UrlClassifierTableUpdateV4, PartialUpdateWithoutRemoval)
+{
+ _PrefixArray fArray;
+
+ // Apply a full update first.
+ {
+ PrefixStringMap fMap;
+ nsCString sha256;
+
+ CreateRandomSortedPrefixArray(10000, 4, 4, fArray);
+ CreateRandomSortedPrefixArray(2000, 5, 32, fArray);
+ PrefixArrayToPrefixStringMap(fArray, fMap);
+ CalculateSHA256(fArray, sha256);
+
+ testFullUpdate(fMap, &sha256);
+ }
+
+ // Apply a partial update without removal
+ {
+ _PrefixArray pArray, mergedArray;
+ PrefixStringMap pMap, mergedMap;
+ nsCString sha256;
+
+ CreateRandomSortedPrefixArray(5000, 4, 4, pArray);
+ CreateRandomSortedPrefixArray(1000, 5, 32, pArray);
+ RemoveIntersection(fArray, pArray);
+ PrefixArrayToPrefixStringMap(pArray, pMap);
+
+ // Calculate the expected prefix map.
+ MergeAndSortArray(fArray, pArray, mergedArray);
+ PrefixArrayToPrefixStringMap(mergedArray, mergedMap);
+ CalculateSHA256(mergedArray, sha256);
+
+ testPartialUpdate(pMap, nullptr, &sha256, mergedMap);
+ }
+
+ Clear();
+}
+
+// Expect failure because partial update contains prefix already
+// in old prefix set.
+TEST(UrlClassifierTableUpdateV4, PartialUpdatePrefixAlreadyExist)
+{
+ _PrefixArray fArray;
+
+ // Apply a full update fist.
+ {
+ PrefixStringMap fMap;
+ nsCString sha256;
+
+ CreateRandomSortedPrefixArray(1000, 4, 32, fArray);
+ PrefixArrayToPrefixStringMap(fArray, fMap);
+ CalculateSHA256(fArray, sha256);
+
+ testFullUpdate(fMap, &sha256);
+ }
+
+ // Apply a partial update which contains a prefix in previous full update.
+ // This should cause an update error.
+ {
+ _PrefixArray pArray;
+ PrefixStringMap pMap;
+ TableUpdateArray tableUpdates;
+
+ // Pick one prefix from full update prefix and add it to partial update.
+ // This should result a failure when call ApplyUpdates.
+ pArray.AppendElement(fArray[rand() % fArray.Length()]);
+ CreateRandomSortedPrefixArray(200, 4, 32, pArray);
+ PrefixArrayToPrefixStringMap(pArray, pMap);
+
+ GenerateUpdateData(false, pMap, nullptr, nullptr, tableUpdates);
+ testUpdateFail(tableUpdates);
+ }
+
+ Clear();
+}
+
+// Test apply partial update directly without applying an full update first.
+TEST(UrlClassifierTableUpdateV4, OnlyPartialUpdate)
+{
+ _PrefixArray pArray;
+ PrefixStringMap pMap;
+ nsCString sha256;
+
+ CreateRandomSortedPrefixArray(5000, 4, 4, pArray);
+ CreateRandomSortedPrefixArray(1000, 5, 32, pArray);
+ PrefixArrayToPrefixStringMap(pArray, pMap);
+ CalculateSHA256(pArray, sha256);
+
+ testPartialUpdate(pMap, nullptr, &sha256, pMap);
+
+ Clear();
+}
+
+// Test partial update without any ADD prefixes, only removalIndices.
+TEST(UrlClassifierTableUpdateV4, PartialUpdateOnlyRemoval)
+{
+ _PrefixArray fArray;
+
+ // Apply a full update first.
+ {
+ PrefixStringMap fMap;
+ nsCString sha256;
+
+ CreateRandomSortedPrefixArray(5000, 4, 4, fArray);
+ CreateRandomSortedPrefixArray(1000, 5, 32, fArray);
+ PrefixArrayToPrefixStringMap(fArray, fMap);
+ CalculateSHA256(fArray, sha256);
+
+ testFullUpdate(fMap, &sha256);
+ }
+
+ // Apply a partial update without add prefix, only contain removal indices.
+ {
+ _PrefixArray pArray;
+ PrefixStringMap pMap, mergedMap;
+ nsCString sha256;
+
+ // Remove 1/5 of elements of original prefix set.
+ nsTArray<uint32_t> removal;
+ CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal);
+ RemoveElements(removal, fArray);
+
+ PrefixArrayToPrefixStringMap(fArray, mergedMap);
+ CalculateSHA256(fArray, sha256);
+
+ testPartialUpdate(pMap, &removal, &sha256, mergedMap);
+ }
+
+ Clear();
+}
+
+// Test one tableupdate array contains full update and multiple partial updates.
+TEST(UrlClassifierTableUpdateV4, MultipleTableUpdates)
+{
+ _PrefixArray fArray, pArray, mergedArray;
+ PrefixStringMap fMap, pMap, mergedMap;
+ nsCString sha256;
+
+ TableUpdateArray tableUpdates;
+
+ // Generate first full udpate
+ CreateRandomSortedPrefixArray(10000, 4, 4, fArray);
+ CreateRandomSortedPrefixArray(2000, 5, 32, fArray);
+ PrefixArrayToPrefixStringMap(fArray, fMap);
+ CalculateSHA256(fArray, sha256);
+
+ GenerateUpdateData(true, fMap, nullptr, &sha256, tableUpdates);
+
+ // Generate second partial update
+ CreateRandomSortedPrefixArray(3000, 4, 4, pArray);
+ CreateRandomSortedPrefixArray(1000, 5, 32, pArray);
+ RemoveIntersection(fArray, pArray);
+ PrefixArrayToPrefixStringMap(pArray, pMap);
+
+ MergeAndSortArray(fArray, pArray, mergedArray);
+ CalculateSHA256(mergedArray, sha256);
+
+ GenerateUpdateData(false, pMap, nullptr, &sha256, tableUpdates);
+
+ // Generate thrid partial update
+ fArray.AppendElements(pArray);
+ fArray.Sort();
+ pArray.Clear();
+ CreateRandomSortedPrefixArray(3000, 4, 4, pArray);
+ CreateRandomSortedPrefixArray(1000, 5, 32, pArray);
+ RemoveIntersection(fArray, pArray);
+ PrefixArrayToPrefixStringMap(pArray, pMap);
+
+ // Remove 1/5 of elements of original prefix set.
+ nsTArray<uint32_t> removal;
+ CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal);
+ RemoveElements(removal, fArray);
+
+ MergeAndSortArray(fArray, pArray, mergedArray);
+ PrefixArrayToPrefixStringMap(mergedArray, mergedMap);
+ CalculateSHA256(mergedArray, sha256);
+
+ GenerateUpdateData(false, pMap, &removal, &sha256, tableUpdates);
+
+ testUpdate(tableUpdates, mergedMap);
+
+ Clear();
+}
+
+// Test apply full update first, and then apply multiple partial updates
+// in one tableupdate array.
+TEST(UrlClassifierTableUpdateV4, MultiplePartialUpdateTableUpdates)
+{
+ _PrefixArray fArray;
+
+ // Apply a full update first
+ {
+ PrefixStringMap fMap;
+ nsCString sha256;
+
+ // Generate first full udpate
+ CreateRandomSortedPrefixArray(10000, 4, 4, fArray);
+ CreateRandomSortedPrefixArray(3000, 5, 32, fArray);
+ PrefixArrayToPrefixStringMap(fArray, fMap);
+ CalculateSHA256(fArray, sha256);
+
+ testFullUpdate(fMap, &sha256);
+ }
+
+ // Apply multiple partial updates in one table update
+ {
+ _PrefixArray pArray, mergedArray;
+ PrefixStringMap pMap, mergedMap;
+ nsCString sha256;
+ nsTArray<uint32_t> removal;
+ TableUpdateArray tableUpdates;
+
+ // Generate first partial update
+ CreateRandomSortedPrefixArray(3000, 4, 4, pArray);
+ CreateRandomSortedPrefixArray(1000, 5, 32, pArray);
+ RemoveIntersection(fArray, pArray);
+ PrefixArrayToPrefixStringMap(pArray, pMap);
+
+ // Remove 1/5 of elements of original prefix set.
+ CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal);
+ RemoveElements(removal, fArray);
+
+ MergeAndSortArray(fArray, pArray, mergedArray);
+ CalculateSHA256(mergedArray, sha256);
+
+ GenerateUpdateData(false, pMap, &removal, &sha256, tableUpdates);
+
+ fArray.AppendElements(pArray);
+ fArray.Sort();
+ pArray.Clear();
+ removal.Clear();
+
+ // Generate second partial update.
+ CreateRandomSortedPrefixArray(2000, 4, 4, pArray);
+ CreateRandomSortedPrefixArray(1000, 5, 32, pArray);
+ RemoveIntersection(fArray, pArray);
+ PrefixArrayToPrefixStringMap(pArray, pMap);
+
+ // Remove 1/5 of elements of original prefix set.
+ CreateRandomRemovalIndices(fArray.Length() / 5, fArray.Length(), removal);
+ RemoveElements(removal, fArray);
+
+ MergeAndSortArray(fArray, pArray, mergedArray);
+ PrefixArrayToPrefixStringMap(mergedArray, mergedMap);
+ CalculateSHA256(mergedArray, sha256);
+
+ GenerateUpdateData(false, pMap, &removal, &sha256, tableUpdates);
+
+ testUpdate(tableUpdates, mergedMap);
+ }
+
+ Clear();
+}
+
+// Test removal indices are larger than the original prefix set.
+TEST(UrlClassifierTableUpdateV4, RemovalIndexTooLarge)
+{
+ _PrefixArray fArray;
+
+ // Apply a full update first
+ {
+ PrefixStringMap fMap;
+ nsCString sha256;
+
+ CreateRandomSortedPrefixArray(1000, 4, 32, fArray);
+ PrefixArrayToPrefixStringMap(fArray, fMap);
+ CalculateSHA256(fArray, sha256);
+
+ testFullUpdate(fMap, &sha256);
+ }
+
+ // Apply a partial update with removal indice array larger than
+ // old prefix set(fArray). This should cause an error.
+ {
+ _PrefixArray pArray;
+ PrefixStringMap pMap;
+ nsTArray<uint32_t> removal;
+ TableUpdateArray tableUpdates;
+
+ CreateRandomSortedPrefixArray(200, 4, 32, pArray);
+ RemoveIntersection(fArray, pArray);
+ PrefixArrayToPrefixStringMap(pArray, pMap);
+
+ for (uint32_t i = 0; i < fArray.Length() + 1; i++) {
+ removal.AppendElement(i);
+ }
+
+ GenerateUpdateData(false, pMap, &removal, nullptr, tableUpdates);
+ testUpdateFail(tableUpdates);
+ }
+
+ Clear();
+}
+
+TEST(UrlClassifierTableUpdateV4, ChecksumMismatch)
+{
+ // Apply a full update first
+ {
+ _PrefixArray fArray;
+ PrefixStringMap fMap;
+ nsCString sha256;
+
+ CreateRandomSortedPrefixArray(1000, 4, 32, fArray);
+ PrefixArrayToPrefixStringMap(fArray, fMap);
+ CalculateSHA256(fArray, sha256);
+
+ testFullUpdate(fMap, &sha256);
+ }
+
+ // Apply a partial update with incorrect sha256
+ {
+ _PrefixArray pArray;
+ PrefixStringMap pMap;
+ nsCString sha256;
+ TableUpdateArray tableUpdates;
+
+ CreateRandomSortedPrefixArray(200, 4, 32, pArray);
+ PrefixArrayToPrefixStringMap(pArray, pMap);
+
+ // sha256 should be calculated with both old prefix set and add prefix
+ // set, here we only calculate sha256 with add prefix set to check if
+ // applyUpdate will return failure.
+ CalculateSHA256(pArray, sha256);
+
+ GenerateUpdateData(false, pMap, nullptr, &sha256, tableUpdates);
+ testUpdateFail(tableUpdates);
+ }
+
+ Clear();
+}
+
+TEST(UrlClassifierTableUpdateV4, ApplyUpdateThenLoad)
+{
+ // Apply update with sha256
+ {
+ _PrefixArray fArray;
+ PrefixStringMap fMap;
+ nsCString sha256;
+
+ CreateRandomSortedPrefixArray(1000, 4, 32, fArray);
+ PrefixArrayToPrefixStringMap(fArray, fMap);
+ CalculateSHA256(fArray, sha256);
+
+ testFullUpdate(fMap, &sha256);
+
+ // Open lookup cache will load prefix set and verify the sha256
+ testOpenLookupCache();
+ }
+
+ Clear();
+
+ // Apply update without sha256
+ {
+ _PrefixArray fArray;
+ PrefixStringMap fMap;
+
+ CreateRandomSortedPrefixArray(1000, 4, 32, fArray);
+ PrefixArrayToPrefixStringMap(fArray, fMap);
+
+ testFullUpdate(fMap, nullptr);
+
+ testOpenLookupCache();
+ }
+
+ Clear();
+}
+
+// This test is used to avoid an eror from nsICryptoHash
+TEST(UrlClassifierTableUpdateV4, ApplyUpdateWithFixedChecksum)
+{
+ _PrefixArray fArray = {_Prefix("enus"),
+ _Prefix("apollo"),
+ _Prefix("mars"),
+ _Prefix("Hecatonchires cyclopes"),
+ _Prefix("vesta"),
+ _Prefix("neptunus"),
+ _Prefix("jupiter"),
+ _Prefix("diana"),
+ _Prefix("minerva"),
+ _Prefix("ceres"),
+ _Prefix("Aidos,Adephagia,Adikia,Aletheia"),
+ _Prefix("hecatonchires"),
+ _Prefix("alcyoneus"),
+ _Prefix("hades"),
+ _Prefix("vulcanus"),
+ _Prefix("juno"),
+ _Prefix("mercury"),
+ _Prefix("Stheno, Euryale and Medusa")};
+ fArray.Sort();
+
+ PrefixStringMap fMap;
+ PrefixArrayToPrefixStringMap(fArray, fMap);
+
+ nsCString sha256(
+ "\xae\x18\x94\xd7\xd0\x83\x5f\xc1"
+ "\x58\x59\x5c\x2c\x72\xb9\x6e\x5e"
+ "\xf4\xe8\x0a\x6b\xff\x5e\x6b\x81"
+ "\x65\x34\x06\x16\x06\x59\xa0\x67");
+
+ testFullUpdate(fMap, &sha256);
+
+ // Open lookup cache will load prefix set and verify the sha256
+ testOpenLookupCache();
+
+ Clear();
+}
+
+// This test ensure that an empty update works correctly. Empty update
+// should be skipped by CheckValidUpdate in Classifier::UpdateTableV4.
+TEST(UrlClassifierTableUpdateV4, EmptyUpdate)
+{
+ PrefixStringMap emptyAddition;
+ nsTArray<uint32_t> emptyRemoval;
+
+ _PrefixArray array;
+ PrefixStringMap map;
+ nsCString sha256;
+
+ CalculateSHA256(array, sha256);
+
+ // Test apply empty full/partial update before we already
+ // have data in DB.
+ testFullUpdate(emptyAddition, &sha256);
+ testPartialUpdate(emptyAddition, &emptyRemoval, &sha256, map);
+
+ // Apply an full update.
+ CreateRandomSortedPrefixArray(100, 4, 4, array);
+ CreateRandomSortedPrefixArray(10, 5, 32, array);
+ PrefixArrayToPrefixStringMap(array, map);
+ CalculateSHA256(array, sha256);
+
+ testFullUpdate(map, &sha256);
+
+ // Test apply empty full/partial update when we already
+ // have data in DB
+ testPartialUpdate(emptyAddition, &emptyRemoval, &sha256, map);
+ testFullUpdate(emptyAddition, &sha256);
+
+ Clear();
+}
+
+// This test ensure applying an empty update directly through update algorithm
+// should be correct.
+TEST(UrlClassifierTableUpdateV4, EmptyUpdate2)
+{
+ // Setup LookupCache with initial data
+ _PrefixArray array;
+ CreateRandomSortedPrefixArray(100, 4, 4, array);
+ CreateRandomSortedPrefixArray(10, 5, 32, array);
+ RefPtr<LookupCacheV4> cache = SetupLookupCache<LookupCacheV4>(array);
+
+ // Setup TableUpdate object with only sha256 from previous update(initial
+ // data).
+ nsCString sha256;
+ CalculateSHA256(array, sha256);
+ std::string stdSHA256;
+ stdSHA256.assign(const_cast<char*>(sha256.BeginReading()), sha256.Length());
+
+ RefPtr<TableUpdateV4> tableUpdate = new TableUpdateV4(GTEST_TABLE);
+ tableUpdate->SetSHA256(stdSHA256);
+
+ // Apply update directly through LookupCache interface
+ PrefixStringMap input, output;
+ PrefixArrayToPrefixStringMap(array, input);
+ nsresult rv = cache->ApplyUpdate(tableUpdate.get(), input, output);
+
+ ASSERT_TRUE(rv == NS_OK);
+
+ Clear();
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestUrlClassifierUtils.cpp b/toolkit/components/url-classifier/tests/gtest/TestUrlClassifierUtils.cpp
new file mode 100644
index 0000000000..354a47f07d
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestUrlClassifierUtils.cpp
@@ -0,0 +1,254 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <ctype.h>
+#include <stdio.h>
+
+#include <mozilla/RefPtr.h>
+#include "nsEscape.h"
+#include "nsString.h"
+#include "nsUrlClassifierUtils.h"
+#include "stdlib.h"
+
+#include "Common.h"
+
+static char int_to_hex_digit(int32_t i) {
+ NS_ASSERTION((i >= 0) && (i <= 15), "int too big in int_to_hex_digit");
+ return static_cast<char>(((i < 10) ? (i + '0') : ((i - 10) + 'A')));
+}
+
+static void CheckEquals(nsCString& expected, nsCString& actual) {
+ ASSERT_TRUE((expected).Equals((actual)))
+ << "Expected:" << expected.get() << ", Actual:" << actual.get();
+}
+
+static void TestUnescapeHelper(const char* in, const char* expected) {
+ nsCString out, strIn(in), strExp(expected);
+
+ NS_UnescapeURL(strIn.get(), strIn.Length(), esc_AlwaysCopy, out);
+ CheckEquals(strExp, out);
+}
+
+static void TestEncodeHelper(const char* in, const char* expected) {
+ nsCString out, strIn(in), strExp(expected);
+ nsUrlClassifierUtils::GetInstance()->SpecialEncode(strIn, true, out);
+ CheckEquals(strExp, out);
+}
+
+static void TestCanonicalizeHelper(const char* in, const char* expected) {
+ nsCString out, strIn(in), strExp(expected);
+ nsUrlClassifierUtils::GetInstance()->CanonicalizePath(strIn, out);
+ CheckEquals(strExp, out);
+}
+
+static void TestCanonicalNumHelper(const char* in, uint32_t bytes,
+ bool allowOctal, const char* expected) {
+ nsCString out, strIn(in), strExp(expected);
+ nsUrlClassifierUtils::GetInstance()->CanonicalNum(strIn, bytes, allowOctal,
+ out);
+ CheckEquals(strExp, out);
+}
+
+void TestHostnameHelper(const char* in, const char* expected) {
+ nsCString out, strIn(in), strExp(expected);
+ nsUrlClassifierUtils::GetInstance()->CanonicalizeHostname(strIn, out);
+ CheckEquals(strExp, out);
+}
+
+// Make sure Unescape from nsEncode.h's unescape does what the server does.
+TEST(UrlClassifierUtils, Unescape)
+{
+ // test empty string
+ TestUnescapeHelper("\0", "\0");
+
+ // Test docoding of all characters.
+ nsCString allCharsEncoded, allCharsEncodedLowercase, allCharsAsString;
+ for (int32_t i = 1; i < 256; ++i) {
+ allCharsEncoded.Append('%');
+ allCharsEncoded.Append(int_to_hex_digit(i / 16));
+ allCharsEncoded.Append((int_to_hex_digit(i % 16)));
+
+ allCharsEncodedLowercase.Append('%');
+ allCharsEncodedLowercase.Append(tolower(int_to_hex_digit(i / 16)));
+ allCharsEncodedLowercase.Append(tolower(int_to_hex_digit(i % 16)));
+
+ allCharsAsString.Append(static_cast<char>(i));
+ }
+
+ nsCString out;
+ NS_UnescapeURL(allCharsEncoded.get(), allCharsEncoded.Length(),
+ esc_AlwaysCopy, out);
+
+ CheckEquals(allCharsAsString, out);
+
+ out.Truncate();
+ NS_UnescapeURL(allCharsEncodedLowercase.get(),
+ allCharsEncodedLowercase.Length(), esc_AlwaysCopy, out);
+ CheckEquals(allCharsAsString, out);
+
+ // Test %-related edge cases
+ TestUnescapeHelper("%", "%");
+ TestUnescapeHelper("%xx", "%xx");
+ TestUnescapeHelper("%%", "%%");
+ TestUnescapeHelper("%%%", "%%%");
+ TestUnescapeHelper("%%%%", "%%%%");
+ TestUnescapeHelper("%1", "%1");
+ TestUnescapeHelper("%1z", "%1z");
+ TestUnescapeHelper("a%1z", "a%1z");
+ TestUnescapeHelper("abc%d%e%fg%hij%klmno%", "abc%d%e%fg%hij%klmno%");
+
+ // A few more tests
+ TestUnescapeHelper("%25", "%");
+ TestUnescapeHelper("%25%32%35", "%25");
+}
+
+TEST(UrlClassifierUtils, Enc)
+{
+ // Test empty string
+ TestEncodeHelper("", "");
+
+ // Test that all characters we shouldn't encode ([33-34],36,[38,126]) are not.
+ nsCString noenc;
+ for (int32_t i = 33; i < 127; i++) {
+ if (i != 35 && i != 37) { // skip %
+ noenc.Append(static_cast<char>(i));
+ }
+ }
+ nsCString out;
+ nsUrlClassifierUtils::GetInstance()->SpecialEncode(noenc, false, out);
+ CheckEquals(noenc, out);
+
+ // Test that all the chars that we should encode [0,32],35, 37,[127,255] are
+ nsCString yesAsString, yesExpectedString;
+ for (int32_t i = 1; i < 256; i++) {
+ if (i < 33 || i == 35 || i == 37 || i > 126) {
+ yesAsString.Append(static_cast<char>(i));
+ yesExpectedString.Append('%');
+ yesExpectedString.Append(int_to_hex_digit(i / 16));
+ yesExpectedString.Append(int_to_hex_digit(i % 16));
+ }
+ }
+
+ out.Truncate();
+ nsUrlClassifierUtils::GetInstance()->SpecialEncode(yesAsString, false, out);
+ CheckEquals(yesExpectedString, out);
+
+ TestEncodeHelper("blah//blah", "blah/blah");
+}
+
+TEST(UrlClassifierUtils, Canonicalize)
+{
+ // Test repeated %-decoding. Note: %25 --> %, %32 --> 2, %35 --> 5
+ TestCanonicalizeHelper("%25", "%25");
+ TestCanonicalizeHelper("%25%32%35", "%25");
+ TestCanonicalizeHelper("asdf%25%32%35asd", "asdf%25asd");
+ TestCanonicalizeHelper("%%%25%32%35asd%%", "%25%25%25asd%25%25");
+ TestCanonicalizeHelper("%25%32%35%25%32%35%25%32%35", "%25%25%25");
+ TestCanonicalizeHelper("%25", "%25");
+ TestCanonicalizeHelper(
+ "%257Ea%2521b%2540c%2523d%2524e%25f%255E00%252611%252A22%252833%252944_"
+ "55%252B",
+ "~a!b@c%23d$e%25f^00&11*22(33)44_55+");
+
+ TestCanonicalizeHelper("", "");
+ TestCanonicalizeHelper(
+ "%31%36%38%2e%31%38%38%2e%39%39%2e%32%36/%2E%73%65%63%75%72%65/"
+ "%77%77%77%2E%65%62%61%79%2E%63%6F%6D/",
+ "168.188.99.26/.secure/www.ebay.com/");
+ TestCanonicalizeHelper(
+ "195.127.0.11/uploads/%20%20%20%20/.verify/"
+ ".eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/",
+ "195.127.0.11/uploads/%20%20%20%20/.verify/"
+ ".eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/");
+ // Added in bug 489455. %00 should no longer be changed to %01.
+ TestCanonicalizeHelper("%00", "%00");
+}
+
+void TestParseIPAddressHelper(const char* in, const char* expected) {
+ nsCString out, strIn(in), strExp(expected);
+ nsUrlClassifierUtils::GetInstance()->ParseIPAddress(strIn, out);
+ CheckEquals(strExp, out);
+}
+
+TEST(UrlClassifierUtils, ParseIPAddress)
+{
+ TestParseIPAddressHelper("123.123.0.0.1", "");
+ TestParseIPAddressHelper("255.0.0.1", "255.0.0.1");
+ TestParseIPAddressHelper("12.0x12.01234", "12.18.2.156");
+ TestParseIPAddressHelper("276.2.3", "20.2.0.3");
+ TestParseIPAddressHelper("012.034.01.055", "10.28.1.45");
+ TestParseIPAddressHelper("0x12.0x43.0x44.0x01", "18.67.68.1");
+ TestParseIPAddressHelper("167838211", "10.1.2.3");
+ TestParseIPAddressHelper("3279880203", "195.127.0.11");
+ TestParseIPAddressHelper("0x12434401", "18.67.68.1");
+ TestParseIPAddressHelper("413960661", "24.172.137.213");
+ TestParseIPAddressHelper("03053104725", "24.172.137.213");
+ TestParseIPAddressHelper("030.0254.0x89d5", "24.172.137.213");
+ TestParseIPAddressHelper("1.234.4.0377", "1.234.4.255");
+ TestParseIPAddressHelper("1.2.3.00x0", "");
+ TestParseIPAddressHelper("10.192.95.89 xy", "10.192.95.89");
+ TestParseIPAddressHelper("10.192.95.89 xyz", "");
+ TestParseIPAddressHelper("1.2.3.0x0", "1.2.3.0");
+ TestParseIPAddressHelper("1.2.3.4", "1.2.3.4");
+}
+
+TEST(UrlClassifierUtils, CanonicalNum)
+{
+ TestCanonicalNumHelper("", 1, true, "");
+ TestCanonicalNumHelper("10", 0, true, "");
+ TestCanonicalNumHelper("45", 1, true, "45");
+ TestCanonicalNumHelper("0x10", 1, true, "16");
+ TestCanonicalNumHelper("367", 2, true, "1.111");
+ TestCanonicalNumHelper("012345", 3, true, "0.20.229");
+ TestCanonicalNumHelper("0173", 1, true, "123");
+ TestCanonicalNumHelper("09", 1, false, "9");
+ TestCanonicalNumHelper("0x120x34", 2, true, "");
+ TestCanonicalNumHelper("0x12fc", 2, true, "18.252");
+ TestCanonicalNumHelper("3279880203", 4, true, "195.127.0.11");
+ TestCanonicalNumHelper("0x0000059", 1, true, "89");
+ TestCanonicalNumHelper("0x00000059", 1, true, "89");
+ TestCanonicalNumHelper("0x0000067", 1, true, "103");
+}
+
+TEST(UrlClassifierUtils, Hostname)
+{
+ TestHostnameHelper("abcd123;[]", "abcd123;[]");
+ TestHostnameHelper("abc.123", "abc.123");
+ TestHostnameHelper("abc..123", "abc.123");
+ TestHostnameHelper("trailing.", "trailing");
+ TestHostnameHelper("i love trailing dots....", "i%20love%20trailing%20dots");
+ TestHostnameHelper(".leading", "leading");
+ TestHostnameHelper("..leading", "leading");
+ TestHostnameHelper(".dots.", "dots");
+ TestHostnameHelper(".both.", "both");
+ TestHostnameHelper(".both..", "both");
+ TestHostnameHelper("..both.", "both");
+ TestHostnameHelper("..both..", "both");
+ TestHostnameHelper("..a.b.c.d..", "a.b.c.d");
+ TestHostnameHelper("..127.0.0.1..", "127.0.0.1");
+ TestHostnameHelper("AB CD 12354", "ab%20cd%2012354");
+ TestHostnameHelper("\1\2\3\4\112\177", "%01%02%03%04j%7F");
+ TestHostnameHelper("<>.AS/-+", "<>.as/-+");
+ // Added in bug 489455. %00 should no longer be changed to %01.
+ TestHostnameHelper("%00", "%00");
+}
+
+TEST(UrlClassifierUtils, LongHostname)
+{
+ static const int kTestSize = 1024 * 150;
+ char* str = static_cast<char*>(malloc(kTestSize + 1));
+ memset(str, 'x', kTestSize);
+ str[kTestSize] = '\0';
+
+ nsAutoCString out;
+ nsDependentCString in(str);
+ PRIntervalTime clockStart = PR_IntervalNow();
+ nsUrlClassifierUtils::GetInstance()->CanonicalizeHostname(in, out);
+ PRIntervalTime clockEnd = PR_IntervalNow();
+
+ CheckEquals(in, out);
+
+ printf("CanonicalizeHostname on long string (%dms)\n",
+ PR_IntervalToMilliseconds(clockEnd - clockStart));
+}
diff --git a/toolkit/components/url-classifier/tests/gtest/TestVariableLengthPrefixSet.cpp b/toolkit/components/url-classifier/tests/gtest/TestVariableLengthPrefixSet.cpp
new file mode 100644
index 0000000000..6eb36a12d2
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/TestVariableLengthPrefixSet.cpp
@@ -0,0 +1,486 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "LookupCacheV4.h"
+#include "mozilla/Preferences.h"
+#include <mozilla/RefPtr.h>
+#include "nsAppDirectoryServiceDefs.h"
+#include "nsClassHashtable.h"
+#include "nsString.h"
+#include "VariableLengthPrefixSet.h"
+
+#include "Common.h"
+
+// Create fullhash by appending random characters.
+static nsCString CreateFullHash(const nsACString& in) {
+ nsCString out(in);
+ out.SetLength(32);
+ for (size_t i = in.Length(); i < 32; i++) {
+ out.SetCharAt(char(rand() % 256), i);
+ }
+
+ return out;
+}
+
+// This function generate N prefixes with size between MIN and MAX.
+// The output array will not be cleared, random result will append to it
+static void RandomPrefixes(uint32_t N, uint32_t MIN, uint32_t MAX,
+ _PrefixArray& array) {
+ array.SetCapacity(array.Length() + N);
+
+ uint32_t range = (MAX - MIN + 1);
+
+ for (uint32_t i = 0; i < N; i++) {
+ uint32_t prefixSize = (rand() % range) + MIN;
+ _Prefix prefix;
+ prefix.SetLength(prefixSize);
+
+ bool added = false;
+ while (!added) {
+ char* dst = prefix.BeginWriting();
+ for (uint32_t j = 0; j < prefixSize; j++) {
+ dst[j] = rand() % 256;
+ }
+
+ if (!array.Contains(prefix)) {
+ array.AppendElement(prefix);
+ added = true;
+ }
+ }
+ }
+}
+
+// This test loops through all the prefixes and converts each prefix to
+// fullhash by appending random characters, each converted fullhash
+// should at least match its original length in the prefixSet.
+static void DoExpectedLookup(LookupCacheV4* cache, _PrefixArray& array) {
+ uint32_t matchLength = 0;
+ for (uint32_t i = 0; i < array.Length(); i++) {
+ const nsCString& prefix = array[i];
+ Completion complete;
+ complete.Assign(CreateFullHash(prefix));
+
+ // Find match for prefix-generated full hash
+ bool has, confirmed;
+ cache->Has(complete, &has, &matchLength, &confirmed);
+ MOZ_ASSERT(matchLength != 0);
+
+ if (matchLength != prefix.Length()) {
+ // Return match size is not the same as prefix size.
+ // In this case it could be because the generated fullhash match other
+ // prefixes, check if this prefix exist.
+ bool found = false;
+
+ for (uint32_t j = 0; j < array.Length(); j++) {
+ if (array[j].Length() != matchLength) {
+ continue;
+ }
+
+ if (0 == memcmp(complete.buf, array[j].BeginReading(), matchLength)) {
+ found = true;
+ break;
+ }
+ }
+ ASSERT_TRUE(found);
+ }
+ }
+}
+
+static void DoRandomLookup(LookupCacheV4* cache, uint32_t N,
+ _PrefixArray& array) {
+ for (uint32_t i = 0; i < N; i++) {
+ // Random 32-bytes test fullhash
+ char buf[32];
+ for (uint32_t j = 0; j < 32; j++) {
+ buf[j] = (char)(rand() % 256);
+ }
+
+ // Get the expected result.
+ nsTArray<uint32_t> expected;
+ for (uint32_t j = 0; j < array.Length(); j++) {
+ const nsACString& str = array[j];
+ if (0 == memcmp(buf, str.BeginReading(), str.Length())) {
+ expected.AppendElement(str.Length());
+ }
+ }
+
+ Completion complete;
+ complete.Assign(nsDependentCSubstring(buf, 32));
+ bool has, confirmed;
+ uint32_t matchLength = 0;
+ cache->Has(complete, &has, &matchLength, &confirmed);
+
+ ASSERT_TRUE(expected.IsEmpty() ? !matchLength
+ : expected.Contains(matchLength));
+ }
+}
+
+static already_AddRefed<LookupCacheV4> SetupLookupCache(
+ const nsACString& aName) {
+ nsCOMPtr<nsIFile> rootDir;
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(rootDir));
+
+ nsAutoCString provider("test");
+ RefPtr<LookupCacheV4> lookup = new LookupCacheV4(aName, provider, rootDir);
+ lookup->Init();
+
+ return lookup.forget();
+}
+
+class UrlClassifierPrefixSetTest : public ::testing::TestWithParam<uint32_t> {
+ protected:
+ void SetUp() override {
+ // max_array_size to 0 means we are testing delta algorithm here.
+ static const char prefKey[] =
+ "browser.safebrowsing.prefixset.max_array_size";
+ mozilla::Preferences::SetUint(prefKey, GetParam());
+
+ mCache = SetupLookupCache("test"_ns);
+ }
+
+ void TearDown() override {
+ mCache = nullptr;
+ mArray.Clear();
+ mMap.Clear();
+ }
+
+ nsresult SetupPrefixes(_PrefixArray&& aArray) {
+ mArray = std::move(aArray);
+ PrefixArrayToPrefixStringMap(mArray, mMap);
+ return mCache->Build(mMap);
+ }
+
+ void SetupPrefixesAndVerify(_PrefixArray& aArray) {
+ mArray = aArray.Clone();
+ PrefixArrayToPrefixStringMap(mArray, mMap);
+
+ ASSERT_NS_SUCCEEDED(mCache->Build(mMap));
+ Verify();
+ }
+
+ void SetupPrefixesAndVerify(_PrefixArray&& aArray) {
+ nsresult rv = SetupPrefixes(std::move(aArray));
+ ASSERT_NS_SUCCEEDED(rv);
+ Verify();
+ }
+
+ void SetupRandomPrefixesAndVerify(uint32_t N, uint32_t MIN, uint32_t MAX) {
+ srand(time(nullptr));
+ RandomPrefixes(N, MIN, MAX, mArray);
+ PrefixArrayToPrefixStringMap(mArray, mMap);
+
+ ASSERT_NS_SUCCEEDED(mCache->Build(mMap));
+ Verify();
+ }
+
+ void Verify() {
+ DoExpectedLookup(mCache, mArray);
+ DoRandomLookup(mCache, 1000, mArray);
+ CheckContent(mCache, mArray);
+ }
+
+ RefPtr<LookupCacheV4> mCache;
+ _PrefixArray mArray;
+ PrefixStringMap mMap;
+};
+
+// Test setting prefix set with only 4-bytes prefixes
+TEST_P(UrlClassifierPrefixSetTest, FixedLengthSet) {
+ SetupPrefixesAndVerify({
+ _Prefix("alph"),
+ _Prefix("brav"),
+ _Prefix("char"),
+ _Prefix("delt"),
+ _Prefix("echo"),
+ _Prefix("foxt"),
+ });
+}
+
+TEST_P(UrlClassifierPrefixSetTest, FixedLengthRandomSet) {
+ SetupRandomPrefixesAndVerify(1500, 4, 4);
+}
+
+TEST_P(UrlClassifierPrefixSetTest, FixedLengthRandomLargeSet) {
+ SetupRandomPrefixesAndVerify(15000, 4, 4);
+}
+
+TEST_P(UrlClassifierPrefixSetTest, FixedLengthTinySet) {
+ SetupPrefixesAndVerify({
+ _Prefix("tiny"),
+ });
+}
+
+// Test setting prefix set with only 5~32 bytes prefixes
+TEST_P(UrlClassifierPrefixSetTest, VariableLengthSet) {
+ SetupPrefixesAndVerify(
+ {_Prefix("bravo"), _Prefix("charlie"), _Prefix("delta"),
+ _Prefix("EchoEchoEchoEchoEcho"), _Prefix("foxtrot"),
+ _Prefix("GolfGolfGolfGolfGolfGolfGolfGolf"), _Prefix("hotel"),
+ _Prefix("november"), _Prefix("oscar"), _Prefix("quebec"),
+ _Prefix("romeo"), _Prefix("sierrasierrasierrasierrasierra"),
+ _Prefix("Tango"), _Prefix("whiskey"), _Prefix("yankee"),
+ _Prefix("ZuluZuluZuluZulu")});
+}
+
+TEST_P(UrlClassifierPrefixSetTest, VariableLengthRandomSet) {
+ SetupRandomPrefixesAndVerify(1500, 5, 32);
+}
+
+// Test setting prefix set with both 4-bytes prefixes and 5~32 bytes prefixes
+TEST_P(UrlClassifierPrefixSetTest, MixedPrefixSet) {
+ SetupPrefixesAndVerify(
+ {_Prefix("enus"), _Prefix("apollo"), _Prefix("mars"),
+ _Prefix("Hecatonchires cyclopes"), _Prefix("vesta"), _Prefix("neptunus"),
+ _Prefix("jupiter"), _Prefix("diana"), _Prefix("minerva"),
+ _Prefix("ceres"), _Prefix("Aidos,Adephagia,Adikia,Aletheia"),
+ _Prefix("hecatonchires"), _Prefix("alcyoneus"), _Prefix("hades"),
+ _Prefix("vulcanus"), _Prefix("juno"), _Prefix("mercury"),
+ _Prefix("Stheno, Euryale and Medusa")});
+}
+
+TEST_P(UrlClassifierPrefixSetTest, MixedRandomPrefixSet) {
+ SetupRandomPrefixesAndVerify(1500, 4, 32);
+}
+
+// Test resetting prefix set
+TEST_P(UrlClassifierPrefixSetTest, ResetPrefix) {
+ // Base prefix set
+ _PrefixArray oldArray = {
+ _Prefix("Iceland"), _Prefix("Peru"), _Prefix("Mexico"),
+ _Prefix("Australia"), _Prefix("Japan"), _Prefix("Egypt"),
+ _Prefix("America"), _Prefix("Finland"), _Prefix("Germany"),
+ _Prefix("Italy"), _Prefix("France"), _Prefix("Taiwan"),
+ };
+ SetupPrefixesAndVerify(oldArray);
+
+ // New prefix set
+ _PrefixArray newArray = {
+ _Prefix("Pikachu"), _Prefix("Bulbasaur"), _Prefix("Charmander"),
+ _Prefix("Blastoise"), _Prefix("Pidgey"), _Prefix("Mewtwo"),
+ _Prefix("Jigglypuff"), _Prefix("Persian"), _Prefix("Tentacool"),
+ _Prefix("Onix"), _Prefix("Eevee"), _Prefix("Jynx"),
+ };
+ SetupPrefixesAndVerify(newArray);
+
+ // Should not match any of the first prefix set
+ uint32_t matchLength = 0;
+ for (uint32_t i = 0; i < oldArray.Length(); i++) {
+ Completion complete;
+ complete.Assign(CreateFullHash(oldArray[i]));
+
+ // Find match for prefix-generated full hash
+ bool has, confirmed;
+ mCache->Has(complete, &has, &matchLength, &confirmed);
+
+ ASSERT_TRUE(matchLength == 0);
+ }
+}
+
+// Test only set one 4-bytes prefix and one full-length prefix
+TEST_P(UrlClassifierPrefixSetTest, TinyPrefixSet) {
+ SetupPrefixesAndVerify({
+ _Prefix("AAAA"),
+ _Prefix("11112222333344445555666677778888"),
+ });
+}
+
+// Test empty prefix set and IsEmpty function
+TEST_P(UrlClassifierPrefixSetTest, EmptyFixedPrefixSet) {
+ ASSERT_TRUE(mCache->IsEmpty());
+
+ SetupPrefixesAndVerify({});
+
+ // Insert an 4-bytes prefix, then IsEmpty should return false
+ SetupPrefixesAndVerify({_Prefix("test")});
+
+ ASSERT_TRUE(!mCache->IsEmpty());
+}
+
+TEST_P(UrlClassifierPrefixSetTest, EmptyVariableLengthPrefixSet) {
+ ASSERT_TRUE(mCache->IsEmpty());
+
+ SetupPrefixesAndVerify({});
+
+ // Insert an 5~32 bytes prefix, then IsEmpty should return false
+ SetupPrefixesAndVerify({_Prefix("test variable length")});
+
+ ASSERT_TRUE(!mCache->IsEmpty());
+}
+
+// Test prefix size should only between 4~32 bytes
+TEST_P(UrlClassifierPrefixSetTest, MinMaxPrefixSet) {
+ // Test prefix set between 4-32 bytes, should success
+ SetupPrefixesAndVerify({_Prefix("1234"), _Prefix("ABCDEFGHIJKKMNOP"),
+ _Prefix("1aaa2bbb3ccc4ddd5eee6fff7ggg8hhh")});
+
+ // Prefix size less than 4-bytes should fail
+ nsresult rv = SetupPrefixes({_Prefix("123")});
+ ASSERT_NS_FAILED(rv);
+
+ // Prefix size greater than 32-bytes should fail
+ rv = SetupPrefixes({_Prefix("1aaa2bbb3ccc4ddd5eee6fff7ggg8hhh9")});
+ ASSERT_NS_FAILED(rv);
+}
+
+// Test save then load prefix set with only 4-bytes prefixes
+TEST_P(UrlClassifierPrefixSetTest, LoadSaveFixedLengthPrefixSet) {
+ nsCOMPtr<nsIFile> file;
+ _PrefixArray array;
+ PrefixStringMap map;
+
+ // Save
+ {
+ RefPtr<LookupCacheV4> save = SetupLookupCache("test-save"_ns);
+
+ RandomPrefixes(10000, 4, 4, array);
+
+ PrefixArrayToPrefixStringMap(array, map);
+ save->Build(map);
+
+ DoExpectedLookup(save, array);
+ DoRandomLookup(save, 1000, array);
+ CheckContent(save, array);
+
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
+ file->Append(u"test.vlpset"_ns);
+ save->StoreToFile(file);
+ }
+
+ // Load
+ {
+ RefPtr<LookupCacheV4> load = SetupLookupCache("test-load"_ns);
+ load->LoadFromFile(file);
+
+ DoExpectedLookup(load, array);
+ DoRandomLookup(load, 1000, array);
+ CheckContent(load, array);
+ }
+
+ file->Remove(false);
+}
+
+// Test save then load prefix set with only 5~32 bytes prefixes
+TEST_P(UrlClassifierPrefixSetTest, LoadSaveVariableLengthPrefixSet) {
+ nsCOMPtr<nsIFile> file;
+ _PrefixArray array;
+ PrefixStringMap map;
+
+ // Save
+ {
+ RefPtr<LookupCacheV4> save = SetupLookupCache("test-save"_ns);
+
+ RandomPrefixes(10000, 5, 32, array);
+
+ PrefixArrayToPrefixStringMap(array, map);
+ save->Build(map);
+
+ DoExpectedLookup(save, array);
+ DoRandomLookup(save, 1000, array);
+ CheckContent(save, array);
+
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
+ file->Append(u"test.vlpset"_ns);
+ save->StoreToFile(file);
+ }
+
+ // Load
+ {
+ RefPtr<LookupCacheV4> load = SetupLookupCache("test-load"_ns);
+ load->LoadFromFile(file);
+
+ DoExpectedLookup(load, array);
+ DoRandomLookup(load, 1000, array);
+ CheckContent(load, array);
+ }
+
+ file->Remove(false);
+}
+
+// Test save then load prefix with both 4 bytes prefixes and 5~32 bytes prefixes
+TEST_P(UrlClassifierPrefixSetTest, LoadSavePrefixSet) {
+ nsCOMPtr<nsIFile> file;
+ _PrefixArray array;
+ PrefixStringMap map;
+
+ // Save
+ {
+ RefPtr<LookupCacheV4> save = SetupLookupCache("test-save"_ns);
+
+ // Try to simulate the real case that most prefixes are 4bytes
+ RandomPrefixes(20000, 4, 4, array);
+ RandomPrefixes(1000, 5, 32, array);
+
+ PrefixArrayToPrefixStringMap(array, map);
+ save->Build(map);
+
+ DoExpectedLookup(save, array);
+ DoRandomLookup(save, 1000, array);
+ CheckContent(save, array);
+
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
+ file->Append(u"test.vlpset"_ns);
+ save->StoreToFile(file);
+ }
+
+ // Load
+ {
+ RefPtr<LookupCacheV4> load = SetupLookupCache("test-load"_ns);
+ load->LoadFromFile(file);
+
+ DoExpectedLookup(load, array);
+ DoRandomLookup(load, 1000, array);
+ CheckContent(load, array);
+ }
+
+ file->Remove(false);
+}
+
+// This is for fixed-length prefixset
+TEST_P(UrlClassifierPrefixSetTest, LoadSaveNoDelta) {
+ nsCOMPtr<nsIFile> file;
+ _PrefixArray array;
+ PrefixStringMap map;
+
+ for (uint32_t i = 0; i < 100; i++) {
+ // construct a tree without deltas by making the distance
+ // between entries larger than 16 bits
+ uint32_t v = ((1 << 16) + 1) * i;
+ nsCString* ele = array.AppendElement();
+ ele->AppendASCII(reinterpret_cast<const char*>(&v), 4);
+ }
+
+ // Save
+ {
+ RefPtr<LookupCacheV4> save = SetupLookupCache("test-save"_ns);
+
+ PrefixArrayToPrefixStringMap(array, map);
+ save->Build(map);
+
+ DoExpectedLookup(save, array);
+ DoRandomLookup(save, 1000, array);
+ CheckContent(save, array);
+
+ NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
+ file->Append(u"test.vlpset"_ns);
+ save->StoreToFile(file);
+ }
+
+ // Load
+ {
+ RefPtr<LookupCacheV4> load = SetupLookupCache("test-load"_ns);
+ load->LoadFromFile(file);
+
+ DoExpectedLookup(load, array);
+ DoRandomLookup(load, 1000, array);
+ CheckContent(load, array);
+ }
+
+ file->Remove(false);
+}
+
+// To run the same test for different configurations of
+// "browser_safebrowsing_prefixset_max_array_size"
+INSTANTIATE_TEST_SUITE_P(UrlClassifierPrefixSetTest, UrlClassifierPrefixSetTest,
+ ::testing::Values(0, UINT32_MAX));
diff --git a/toolkit/components/url-classifier/tests/gtest/moz.build b/toolkit/components/url-classifier/tests/gtest/moz.build
new file mode 100644
index 0000000000..070012de77
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/gtest/moz.build
@@ -0,0 +1,40 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+LOCAL_INCLUDES += [
+ "../..",
+]
+
+DEFINES["GOOGLE_PROTOBUF_NO_RTTI"] = True
+DEFINES["GOOGLE_PROTOBUF_NO_STATIC_INITIALIZER"] = True
+
+UNIFIED_SOURCES += [
+ "Common.cpp",
+ "TestCaching.cpp",
+ "TestChunkSet.cpp",
+ "TestClassifier.cpp",
+ "TestFailUpdate.cpp",
+ "TestFindFullHash.cpp",
+ "TestLookupCacheV4.cpp",
+ "TestPerProviderDirectory.cpp",
+ "TestPrefixSet.cpp",
+ "TestProtocolParser.cpp",
+ "TestRiceDeltaDecoder.cpp",
+ "TestSafebrowsingHash.cpp",
+ "TestSafeBrowsingProtobuf.cpp",
+ "TestTable.cpp",
+ "TestUrlClassifierTableUpdateV4.cpp",
+ "TestUrlClassifierUtils.cpp",
+ "TestURLsAndHashing.cpp",
+ "TestVariableLengthPrefixSet.cpp",
+]
+
+# Required to have the same MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
+# as non-testing code.
+if CONFIG["NIGHTLY_BUILD"] or CONFIG["MOZ_DEBUG"]:
+ DEFINES["MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES"] = True
+
+FINAL_LIBRARY = "xul-gtest"