/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "chromium/safebrowsing.pb.h" #include "nsEscape.h" #include "nsString.h" #include "nsIURI.h" #include "nsIURIMutator.h" #include "nsIURL.h" #include "nsIXULRuntime.h" #include "nsUrlClassifierUtils.h" #include "nsTArray.h" #include "nsReadableUtils.h" #include "plbase64.h" #include "nsPrintfCString.h" #include "mozilla/ClearOnShutdown.h" #include "mozilla/Sprintf.h" #include "mozilla/StaticPtr.h" #include "mozilla/Mutex.h" #include "nsIRedirectHistoryEntry.h" #include "nsIHttpChannelInternal.h" #include "mozIThirdPartyUtil.h" #include "nsIDocShell.h" #include "mozilla/TextUtils.h" #include "mozilla/Preferences.h" #include "mozilla/Services.h" #include "mozilla/Telemetry.h" #include "nsNetUtil.h" #include "nsIHttpChannel.h" #include "nsIObserverService.h" #include "nsIPrefBranch.h" #include "nsIPrefService.h" #include "nsPIDOMWindow.h" #include "nsServiceManagerUtils.h" #include "nsThreadManager.h" #include "nsTHashSet.h" #include "Classifier.h" #include "Entries.h" #include "prprf.h" #include "prtime.h" #define DEFAULT_PROTOCOL_VERSION "2.2" using namespace mozilla; using namespace mozilla::safebrowsing; static mozilla::StaticRefPtr gUrlClassifierUtils; static char int_to_hex_digit(int32_t i) { NS_ASSERTION((i >= 0) && (i <= 15), "int too big in int_to_hex_digit"); return static_cast(((i < 10) ? (i + '0') : ((i - 10) + 'A'))); } static bool IsDecimal(const nsACString& num) { for (uint32_t i = 0; i < num.Length(); i++) { if (!mozilla::IsAsciiDigit(num[i])) { return false; } } return true; } static bool IsHex(const nsACString& num) { if (num.Length() < 3) { return false; } if (num[0] != '0' || !(num[1] == 'x' || num[1] == 'X')) { return false; } for (uint32_t i = 2; i < num.Length(); i++) { if (!mozilla::IsAsciiHexDigit(num[i])) { return false; } } return true; } static bool IsOctal(const nsACString& num) { if (num.Length() < 2) { return false; } if (num[0] != '0') { return false; } for (uint32_t i = 1; i < num.Length(); i++) { if (!mozilla::IsAsciiDigit(num[i]) || num[i] == '8' || num[i] == '9') { return false; } } return true; } ///////////////////////////////////////////////////////////////// // SafeBrowsing V4 related utits. namespace mozilla { namespace safebrowsing { static PlatformType GetPlatformType() { #if defined(ANDROID) return ANDROID_PLATFORM; #elif defined(XP_MACOSX) return OSX_PLATFORM; #elif defined(XP_LINUX) return LINUX_PLATFORM; #elif defined(XP_WIN) return WINDOWS_PLATFORM; #else // Default to Linux for other platforms (see bug 1362501). return LINUX_PLATFORM; #endif } typedef FetchThreatListUpdatesRequest_ListUpdateRequest ListUpdateRequest; typedef FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints Constraints; static void InitListUpdateRequest(ThreatType aThreatType, const nsCString& aStateBase64, ListUpdateRequest* aListUpdateRequest) { aListUpdateRequest->set_threat_type(aThreatType); PlatformType platform = GetPlatformType(); #if defined(ANDROID) // Temporary hack to fix bug 1441345. if ((aThreatType == SOCIAL_ENGINEERING_PUBLIC) || (aThreatType == SOCIAL_ENGINEERING)) { platform = LINUX_PLATFORM; } #endif aListUpdateRequest->set_platform_type(platform); aListUpdateRequest->set_threat_entry_type(URL); Constraints* contraints = new Constraints(); contraints->add_supported_compressions(RICE); aListUpdateRequest->set_allocated_constraints(contraints); // Only set non-empty state. if (!aStateBase64.IsEmpty()) { nsCString stateBinary; nsresult rv = Base64Decode(aStateBase64, stateBinary); if (NS_SUCCEEDED(rv)) { aListUpdateRequest->set_state(stateBinary.get(), stateBinary.Length()); } } } static ClientInfo* CreateClientInfo() { ClientInfo* c = new ClientInfo(); nsCOMPtr prefBranch = do_GetService(NS_PREFSERVICE_CONTRACTID); nsAutoCString clientId; nsresult rv = prefBranch->GetCharPref("browser.safebrowsing.id", clientId); if (NS_FAILED(rv)) { clientId = "Firefox"; // Use "Firefox" as fallback. } c->set_client_id(clientId.get()); return c; } static bool IsAllowedOnCurrentPlatform(uint32_t aThreatType) { PlatformType platform = GetPlatformType(); switch (aThreatType) { case POTENTIALLY_HARMFUL_APPLICATION: // Bug 1388582 - Google server would respond 404 error if the request // contains PHA on non-mobile platform. return ANDROID_PLATFORM == platform; case MALICIOUS_BINARY: case CSD_DOWNLOAD_WHITELIST: // Bug 1392204 - 'goog-downloadwhite-proto' and 'goog-badbinurl-proto' // are not available on android. return ANDROID_PLATFORM != platform; } // We allow every threat type not listed in the switch cases. return true; } } // end of namespace safebrowsing. } // end of namespace mozilla. // static already_AddRefed nsUrlClassifierUtils::GetXPCOMSingleton() { if (gUrlClassifierUtils) { return do_AddRef(gUrlClassifierUtils); } RefPtr utils = new nsUrlClassifierUtils(); if (NS_WARN_IF(NS_FAILED(utils->Init()))) { return nullptr; } // Note: This is cleared in the nsUrlClassifierUtils destructor. gUrlClassifierUtils = utils.get(); ClearOnShutdown(&gUrlClassifierUtils); return utils.forget(); } // static nsUrlClassifierUtils* nsUrlClassifierUtils::GetInstance() { if (!gUrlClassifierUtils) { RefPtr utils = GetXPCOMSingleton(); } return gUrlClassifierUtils; } nsUrlClassifierUtils::nsUrlClassifierUtils() : mProviderDictLock("nsUrlClassifierUtils.mProviderDictLock") {} nsUrlClassifierUtils::~nsUrlClassifierUtils() { if (gUrlClassifierUtils) { MOZ_ASSERT(gUrlClassifierUtils == this); gUrlClassifierUtils = nullptr; } } nsresult nsUrlClassifierUtils::Init() { // nsIUrlClassifierUtils is a thread-safe service so it's // allowed to use on non-main threads. However, building // the provider dictionary must be on the main thread. // We forcefully load nsUrlClassifierUtils in // nsUrlClassifierDBService::Init() to ensure we must // now be on the main thread. nsresult rv = ReadProvidersFromPrefs(mProviderDict); NS_ENSURE_SUCCESS(rv, rv); // Add an observer for shutdown nsCOMPtr observerService = mozilla::services::GetObserverService(); if (!observerService) return NS_ERROR_FAILURE; observerService->AddObserver(this, "xpcom-shutdown-threads", false); mozilla::Preferences::AddStrongObserver(this, "browser.safebrowsing"); return NS_OK; } NS_IMPL_ISUPPORTS(nsUrlClassifierUtils, nsIUrlClassifierUtils, nsIObserver) ///////////////////////////////////////////////////////////////////////////// // nsIUrlClassifierUtils NS_IMETHODIMP nsUrlClassifierUtils::GetKeyForURI(nsIURI* uri, nsACString& _retval) { nsCOMPtr innerURI = NS_GetInnermostURI(uri); if (!innerURI) innerURI = uri; nsAutoCString host; innerURI->GetAsciiHost(host); if (host.IsEmpty()) { return NS_ERROR_MALFORMED_URI; } nsresult rv = CanonicalizeHostname(host, _retval); NS_ENSURE_SUCCESS(rv, rv); nsAutoCString path; rv = innerURI->GetPathQueryRef(path); NS_ENSURE_SUCCESS(rv, rv); // Strip fragment and query because canonicalization only applies to path int32_t ref = path.FindChar('#'); if (ref != kNotFound) { path.SetLength(ref); } int32_t query = path.FindChar('?'); if (query != kNotFound) { path.SetLength(query); } nsAutoCString temp; rv = CanonicalizePath(path, temp); NS_ENSURE_SUCCESS(rv, rv); _retval.Append(temp); if (query != kNotFound) { nsAutoCString query; rv = innerURI->GetQuery(query); NS_ENSURE_SUCCESS(rv, rv); _retval.AppendPrintf("?%s", query.get()); } return NS_OK; } // We use "goog-*-proto" as the list name for v4, where "proto" indicates // it's updated (as well as hash completion) via protobuf. // // In the mozilla official build, we are allowed to use the // private phishing list (goog-phish-proto). See Bug 1288840. static const struct { const char* mListName; uint32_t mThreatType; } THREAT_TYPE_CONV_TABLE[] = { {"goog-malware-proto", MALWARE_THREAT}, // 1 {"googpub-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2 {"goog-unwanted-proto", UNWANTED_SOFTWARE}, // 3 {"goog-harmful-proto", POTENTIALLY_HARMFUL_APPLICATION}, // 4 {"goog-phish-proto", SOCIAL_ENGINEERING}, // 5 // For application reputation {"goog-badbinurl-proto", MALICIOUS_BINARY}, // 7 {"goog-downloadwhite-proto", CSD_DOWNLOAD_WHITELIST}, // 9 // For testing purpose. {"moztest-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2 {"test-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2 {"moztest-unwanted-proto", UNWANTED_SOFTWARE}, // 3 {"test-unwanted-proto", UNWANTED_SOFTWARE}, // 3 }; NS_IMETHODIMP nsUrlClassifierUtils::ConvertThreatTypeToListNames(uint32_t aThreatType, nsACString& aListNames) { for (uint32_t i = 0; i < ArrayLength(THREAT_TYPE_CONV_TABLE); i++) { if (aThreatType == THREAT_TYPE_CONV_TABLE[i].mThreatType) { if (!aListNames.IsEmpty()) { aListNames.AppendLiteral(","); } aListNames += THREAT_TYPE_CONV_TABLE[i].mListName; } } return aListNames.IsEmpty() ? NS_ERROR_FAILURE : NS_OK; } NS_IMETHODIMP nsUrlClassifierUtils::ConvertListNameToThreatType(const nsACString& aListName, uint32_t* aThreatType) { for (uint32_t i = 0; i < ArrayLength(THREAT_TYPE_CONV_TABLE); i++) { if (aListName.EqualsASCII(THREAT_TYPE_CONV_TABLE[i].mListName)) { *aThreatType = THREAT_TYPE_CONV_TABLE[i].mThreatType; return NS_OK; } } return NS_ERROR_FAILURE; } NS_IMETHODIMP nsUrlClassifierUtils::GetProvider(const nsACString& aTableName, nsACString& aProvider) { MutexAutoLock lock(mProviderDictLock); nsCString* provider = nullptr; if (IsTestTable(aTableName)) { aProvider = nsLiteralCString(TESTING_TABLE_PROVIDER_NAME); } else if (mProviderDict.Get(aTableName, &provider)) { aProvider = provider ? *provider : ""_ns; } else { aProvider.Truncate(); } return NS_OK; } NS_IMETHODIMP nsUrlClassifierUtils::GetTelemetryProvider(const nsACString& aTableName, nsACString& aProvider) { GetProvider(aTableName, aProvider); // Exceptionlist known providers to avoid reporting on private ones. // An empty provider is treated as "other" if (!"mozilla"_ns.Equals(aProvider) && !"google"_ns.Equals(aProvider) && !"google4"_ns.Equals(aProvider) && !"baidu"_ns.Equals(aProvider) && !"mozcn"_ns.Equals(aProvider) && !"yandex"_ns.Equals(aProvider) && !nsLiteralCString(TESTING_TABLE_PROVIDER_NAME).Equals(aProvider)) { aProvider.AssignLiteral("other"); } return NS_OK; } NS_IMETHODIMP nsUrlClassifierUtils::GetProtocolVersion(const nsACString& aProvider, nsACString& aVersion) { nsCOMPtr prefBranch = do_GetService(NS_PREFSERVICE_CONTRACTID); if (prefBranch) { nsPrintfCString prefName("browser.safebrowsing.provider.%s.pver", nsCString(aProvider).get()); nsAutoCString version; nsresult rv = prefBranch->GetCharPref(prefName.get(), version); aVersion = NS_SUCCEEDED(rv) ? version.get() : DEFAULT_PROTOCOL_VERSION; } else { aVersion = DEFAULT_PROTOCOL_VERSION; } return NS_OK; } NS_IMETHODIMP nsUrlClassifierUtils::MakeUpdateRequestV4( const nsTArray& aListNames, const nsTArray& aStatesBase64, nsACString& aRequest) { using namespace mozilla::safebrowsing; if (aListNames.Length() != aStatesBase64.Length()) { return NS_ERROR_INVALID_ARG; } FetchThreatListUpdatesRequest r; r.set_allocated_client(CreateClientInfo()); for (uint32_t i = 0; i < aListNames.Length(); i++) { uint32_t threatType; nsresult rv = ConvertListNameToThreatType(aListNames[i], &threatType); if (NS_FAILED(rv)) { continue; // Unknown list name. } if (!IsAllowedOnCurrentPlatform(threatType)) { NS_WARNING( nsPrintfCString( "Threat type %d (%s) is unsupported on current platform: %d", threatType, aListNames[i].get(), GetPlatformType()) .get()); continue; // Some threat types are not available on some platforms. } auto lur = r.mutable_list_update_requests()->Add(); InitListUpdateRequest(static_cast(threatType), aStatesBase64[i], lur); } // Then serialize. std::string s; r.SerializeToString(&s); nsCString out; nsresult rv = Base64URLEncode(s.size(), (const uint8_t*)s.c_str(), Base64URLEncodePaddingPolicy::Include, out); NS_ENSURE_SUCCESS(rv, rv); aRequest = out; return NS_OK; } NS_IMETHODIMP nsUrlClassifierUtils::MakeFindFullHashRequestV4( const nsTArray& aListNames, const nsTArray& aListStatesBase64, const nsTArray& aPrefixesBase64, nsACString& aRequest) { if (aListNames.Length() != aListStatesBase64.Length()) { return NS_ERROR_INVALID_ARG; } FindFullHashesRequest r; r.set_allocated_client(CreateClientInfo()); nsresult rv; //------------------------------------------------------------------- // Set up FindFullHashesRequest.threat_info. auto threatInfo = r.mutable_threat_info(); PlatformType platform = GetPlatformType(); // 1) Set threat types. for (uint32_t i = 0; i < aListNames.Length(); i++) { // Add threat types. uint32_t threatType; rv = ConvertListNameToThreatType(aListNames[i], &threatType); NS_ENSURE_SUCCESS(rv, rv); if (!IsAllowedOnCurrentPlatform(threatType)) { NS_WARNING( nsPrintfCString( "Threat type %d (%s) is unsupported on current platform: %d", threatType, aListNames[i].get(), GetPlatformType()) .get()); continue; } threatInfo->add_threat_types((ThreatType)threatType); #if defined(ANDROID) // Temporary hack to fix bug 1441345. if (((ThreatType)threatType == SOCIAL_ENGINEERING_PUBLIC) || ((ThreatType)threatType == SOCIAL_ENGINEERING)) { platform = LINUX_PLATFORM; } #endif // Add client states for index 'i' only when the threat type is available // on current platform. nsCString stateBinary; rv = Base64Decode(aListStatesBase64[i], stateBinary); NS_ENSURE_SUCCESS(rv, rv); r.add_client_states(stateBinary.get(), stateBinary.Length()); } // 2) Set platform type. threatInfo->add_platform_types(platform); // 3) Set threat entry type. threatInfo->add_threat_entry_types(URL); // 4) Set threat entries. for (const nsCString& prefix : aPrefixesBase64) { nsCString prefixBinary; rv = Base64Decode(prefix, prefixBinary); threatInfo->add_threat_entries()->set_hash(prefixBinary.get(), prefixBinary.Length()); } //------------------------------------------------------------------- // Then serialize. std::string s; r.SerializeToString(&s); nsCString out; rv = Base64URLEncode(s.size(), (const uint8_t*)s.c_str(), Base64URLEncodePaddingPolicy::Include, out); NS_ENSURE_SUCCESS(rv, rv); aRequest = out; return NS_OK; } // Remove ref, query, userpass, anypart which may contain sensitive data static nsresult GetSpecWithoutSensitiveData(nsIURI* aUri, nsACString& aSpec) { if (NS_WARN_IF(!aUri)) { return NS_ERROR_INVALID_ARG; } nsresult rv; nsCOMPtr url(do_QueryInterface(aUri)); if (url) { nsCOMPtr clone; rv = NS_MutateURI(url) .SetQuery(""_ns) .SetRef(""_ns) .SetUserPass(""_ns) .Finalize(clone); NS_ENSURE_SUCCESS(rv, rv); rv = clone->GetAsciiSpec(aSpec); NS_ENSURE_SUCCESS(rv, rv); } return NS_OK; } static nsresult AddThreatSourceFromChannel(ThreatHit& aHit, nsIChannel* aChannel, ThreatHit_ThreatSourceType aType) { if (NS_WARN_IF(!aChannel)) { return NS_ERROR_INVALID_ARG; } nsresult rv; auto matchingSource = aHit.add_resources(); matchingSource->set_type(aType); nsCOMPtr uri; rv = NS_GetFinalChannelURI(aChannel, getter_AddRefs(uri)); NS_ENSURE_SUCCESS(rv, rv); nsCString spec; rv = GetSpecWithoutSensitiveData(uri, spec); NS_ENSURE_SUCCESS(rv, rv); matchingSource->set_url(spec.get()); nsCOMPtr httpChannel = do_QueryInterface(aChannel); if (httpChannel) { nsCOMPtr referrerInfo = httpChannel->GetReferrerInfo(); if (referrerInfo) { nsAutoCString referrerSpec; nsCOMPtr referrer = referrerInfo->GetComputedReferrer(); if (referrer) { rv = GetSpecWithoutSensitiveData(referrer, referrerSpec); NS_ENSURE_SUCCESS(rv, rv); matchingSource->set_referrer(referrerSpec.get()); } } } nsCOMPtr httpChannelInternal = do_QueryInterface(aChannel); if (httpChannelInternal) { nsCString remoteIp; rv = httpChannelInternal->GetRemoteAddress(remoteIp); if (NS_SUCCEEDED(rv) && !remoteIp.IsEmpty()) { matchingSource->set_remote_ip(remoteIp.get()); } } return NS_OK; } static nsresult AddThreatSourceFromRedirectEntry( ThreatHit& aHit, nsIRedirectHistoryEntry* aRedirectEntry, ThreatHit_ThreatSourceType aType) { if (NS_WARN_IF(!aRedirectEntry)) { return NS_ERROR_INVALID_ARG; } nsresult rv; nsCOMPtr principal; rv = aRedirectEntry->GetPrincipal(getter_AddRefs(principal)); NS_ENSURE_SUCCESS(rv, rv); nsCString spec; rv = principal->GetExposableSpec(spec); NS_ENSURE_SUCCESS(rv, rv); auto source = aHit.add_resources(); source->set_url(spec.get()); source->set_type(aType); nsCOMPtr referrer; rv = aRedirectEntry->GetReferrerURI(getter_AddRefs(referrer)); if (NS_SUCCEEDED(rv) && referrer) { nsCString referrerSpec; rv = GetSpecWithoutSensitiveData(referrer, referrerSpec); NS_ENSURE_SUCCESS(rv, rv); source->set_referrer(referrerSpec.get()); } nsCString remoteIp; rv = aRedirectEntry->GetRemoteAddress(remoteIp); if (NS_SUCCEEDED(rv) && !remoteIp.IsEmpty()) { source->set_remote_ip(remoteIp.get()); } return NS_OK; } // Add top level tab url and redirect threatsources to threatHit message static nsresult AddTabThreatSources(ThreatHit& aHit, nsIChannel* aChannel) { if (NS_WARN_IF(!aChannel)) { return NS_ERROR_INVALID_ARG; } nsresult rv; nsCOMPtr win; nsCOMPtr thirdPartyUtil = do_GetService(THIRDPARTYUTIL_CONTRACTID, &rv); NS_ENSURE_SUCCESS(rv, rv); rv = thirdPartyUtil->GetTopWindowForChannel(aChannel, nullptr, getter_AddRefs(win)); NS_ENSURE_SUCCESS(rv, rv); auto* pwin = nsPIDOMWindowOuter::From(win); nsCOMPtr docShell = pwin->GetDocShell(); if (!docShell) { return NS_OK; } nsCOMPtr topChannel; docShell->GetCurrentDocumentChannel(getter_AddRefs(topChannel)); if (!topChannel) { return NS_OK; } nsCOMPtr uri; rv = aChannel->GetURI(getter_AddRefs(uri)); NS_ENSURE_SUCCESS(rv, rv); nsCOMPtr topUri; rv = topChannel->GetURI(getter_AddRefs(topUri)); NS_ENSURE_SUCCESS(rv, rv); bool isTopUri = false; rv = topUri->Equals(uri, &isTopUri); if (NS_SUCCEEDED(rv) && !isTopUri) { nsCOMPtr loadInfo = aChannel->LoadInfo(); if (loadInfo->RedirectChain().Length()) { AddThreatSourceFromRedirectEntry(aHit, loadInfo->RedirectChain()[0], ThreatHit_ThreatSourceType_TAB_RESOURCE); } } // Set top level tab_url threat source rv = AddThreatSourceFromChannel(aHit, topChannel, ThreatHit_ThreatSourceType_TAB_URL); Unused << NS_WARN_IF(NS_FAILED(rv)); // Set tab_redirect threat sources if there's any nsCOMPtr topLoadInfo = topChannel->LoadInfo(); nsIRedirectHistoryEntry* redirectEntry; size_t length = topLoadInfo->RedirectChain().Length(); for (size_t i = 0; i < length; i++) { redirectEntry = topLoadInfo->RedirectChain()[i]; AddThreatSourceFromRedirectEntry(aHit, redirectEntry, ThreatHit_ThreatSourceType_TAB_REDIRECT); } return NS_OK; } NS_IMETHODIMP nsUrlClassifierUtils::MakeThreatHitReport(nsIChannel* aChannel, const nsACString& aListName, const nsACString& aHashBase64, nsACString& aRequest) { if (NS_WARN_IF(aListName.IsEmpty()) || NS_WARN_IF(aHashBase64.IsEmpty()) || NS_WARN_IF(!aChannel)) { return NS_ERROR_INVALID_ARG; } ThreatHit hit; nsresult rv; uint32_t threatType; rv = ConvertListNameToThreatType(aListName, &threatType); NS_ENSURE_SUCCESS(rv, rv); hit.set_threat_type(static_cast(threatType)); hit.set_platform_type(GetPlatformType()); nsCString hash; rv = Base64Decode(aHashBase64, hash); if (NS_FAILED(rv) || hash.Length() != COMPLETE_SIZE) { return NS_ERROR_FAILURE; } auto threatEntry = hit.mutable_entry(); threatEntry->set_hash(hash.get(), hash.Length()); // Set matching source rv = AddThreatSourceFromChannel(hit, aChannel, ThreatHit_ThreatSourceType_MATCHING_URL); Unused << NS_WARN_IF(NS_FAILED(rv)); // Set tab url, tab resource url and redirect sources rv = AddTabThreatSources(hit, aChannel); Unused << NS_WARN_IF(NS_FAILED(rv)); hit.set_allocated_client_info(CreateClientInfo()); std::string s; hit.SerializeToString(&s); nsCString out; rv = Base64URLEncode(s.size(), reinterpret_cast(s.c_str()), Base64URLEncodePaddingPolicy::Include, out); NS_ENSURE_SUCCESS(rv, rv); aRequest = out; return NS_OK; } static uint32_t DurationToMs(const Duration& aDuration) { // Seconds precision is good enough. Ignore nanoseconds like Chrome does. return aDuration.seconds() * 1000; } NS_IMETHODIMP nsUrlClassifierUtils::ParseFindFullHashResponseV4( const nsACString& aResponse, nsIUrlClassifierParseFindFullHashCallback* aCallback) { enum CompletionErrorType { SUCCESS = 0, PARSING_FAILURE = 1, UNKNOWN_THREAT_TYPE = 2, }; FindFullHashesResponse r; if (!r.ParseFromArray(aResponse.BeginReading(), aResponse.Length())) { NS_WARNING("Invalid response"); Telemetry::Accumulate(Telemetry::URLCLASSIFIER_COMPLETION_ERROR, PARSING_FAILURE); return NS_ERROR_FAILURE; } bool hasUnknownThreatType = false; for (auto& m : r.matches()) { nsCString tableNames; nsresult rv = ConvertThreatTypeToListNames(m.threat_type(), tableNames); if (NS_FAILED(rv)) { hasUnknownThreatType = true; continue; // Ignore un-convertable threat type. } auto& hash = m.threat().hash(); auto cacheDurationSec = m.cache_duration().seconds(); aCallback->OnCompleteHashFound( nsDependentCString(hash.c_str(), hash.length()), tableNames, cacheDurationSec); } auto minWaitDuration = DurationToMs(r.minimum_wait_duration()); auto negCacheDurationSec = r.negative_cache_duration().seconds(); aCallback->OnResponseParsed(minWaitDuration, negCacheDurationSec); Telemetry::Accumulate(Telemetry::URLCLASSIFIER_COMPLETION_ERROR, hasUnknownThreatType ? UNKNOWN_THREAT_TYPE : SUCCESS); return NS_OK; } ////////////////////////////////////////////////////////// // nsIObserver NS_IMETHODIMP nsUrlClassifierUtils::Observe(nsISupports* aSubject, const char* aTopic, const char16_t* aData) { if (0 == strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) { MutexAutoLock lock(mProviderDictLock); return ReadProvidersFromPrefs(mProviderDict); } if (0 == strcmp(aTopic, "xpcom-shutdown-threads")) { nsCOMPtr prefs = do_GetService(NS_PREFSERVICE_CONTRACTID); NS_ENSURE_TRUE(prefs, NS_ERROR_FAILURE); return prefs->RemoveObserver("browser.safebrowsing", this); } return NS_ERROR_UNEXPECTED; } ///////////////////////////////////////////////////////////////////////////// // non-interface methods nsresult nsUrlClassifierUtils::ReadProvidersFromPrefs(ProviderDictType& aDict) { MOZ_ASSERT(NS_IsMainThread(), "ReadProvidersFromPrefs must be on main thread"); nsCOMPtr prefs = do_GetService(NS_PREFSERVICE_CONTRACTID); NS_ENSURE_TRUE(prefs, NS_ERROR_FAILURE); nsCOMPtr prefBranch; nsresult rv = prefs->GetBranch("browser.safebrowsing.provider.", getter_AddRefs(prefBranch)); NS_ENSURE_SUCCESS(rv, rv); // We've got a pref branch for "browser.safebrowsing.provider.". // Enumerate all children prefs and parse providers. nsTArray childArray; rv = prefBranch->GetChildList("", childArray); NS_ENSURE_SUCCESS(rv, rv); // Collect providers from childArray. nsTHashSet providers; for (auto& child : childArray) { auto dotPos = child.FindChar('.'); if (dotPos < 0) { continue; } nsDependentCSubstring provider = Substring(child, 0, dotPos); providers.Insert(provider); } // Now we have all providers. Check which one owns |aTableName|. // e.g. The owning lists of provider "google" is defined in // "browser.safebrowsing.provider.google.lists". for (const auto& provider : providers) { nsPrintfCString owninListsPref("%s.lists", nsPromiseFlatCString{provider}.get()); nsAutoCString owningLists; nsresult rv = prefBranch->GetCharPref(owninListsPref.get(), owningLists); if (NS_FAILED(rv)) { continue; } // We've got the owning lists (represented as string) of |provider|. // Build the dictionary for the owning list and the current provider. nsTArray tables; Classifier::SplitTables(owningLists, tables); for (auto tableName : tables) { aDict.InsertOrUpdate(tableName, MakeUnique(provider)); } } return NS_OK; } nsresult nsUrlClassifierUtils::CanonicalizeHostname(const nsACString& hostname, nsACString& _retval) { nsAutoCString unescaped; if (!NS_UnescapeURL(PromiseFlatCString(hostname).get(), PromiseFlatCString(hostname).Length(), 0, unescaped)) { unescaped.Assign(hostname); } nsAutoCString cleaned; CleanupHostname(unescaped, cleaned); nsAutoCString temp; ParseIPAddress(cleaned, temp); if (!temp.IsEmpty()) { cleaned.Assign(temp); } ToLowerCase(cleaned); SpecialEncode(cleaned, false, _retval); return NS_OK; } nsresult nsUrlClassifierUtils::CanonicalizePath(const nsACString& path, nsACString& _retval) { _retval.Truncate(); nsAutoCString decodedPath(path); nsAutoCString temp; while (NS_UnescapeURL(decodedPath.get(), decodedPath.Length(), 0, temp)) { decodedPath.Assign(temp); temp.Truncate(); } SpecialEncode(decodedPath, true, _retval); // XXX: lowercase the path? return NS_OK; } void nsUrlClassifierUtils::CleanupHostname(const nsACString& hostname, nsACString& _retval) { _retval.Truncate(); const char* curChar = hostname.BeginReading(); const char* end = hostname.EndReading(); char lastChar = '\0'; while (curChar != end) { unsigned char c = static_cast(*curChar); if (c == '.' && (lastChar == '\0' || lastChar == '.')) { // skip } else { _retval.Append(*curChar); } lastChar = c; ++curChar; } // cut off trailing dots while (_retval.Length() > 0 && _retval[_retval.Length() - 1] == '.') { _retval.SetLength(_retval.Length() - 1); } } void nsUrlClassifierUtils::ParseIPAddress(const nsACString& host, nsACString& _retval) { _retval.Truncate(); nsACString::const_iterator iter, end; host.BeginReading(iter); host.EndReading(end); if (host.Length() <= 15) { // The Windows resolver allows a 4-part dotted decimal IP address to // have a space followed by any old rubbish, so long as the total length // of the string doesn't get above 15 characters. So, "10.192.95.89 xy" // is resolved to 10.192.95.89. // If the string length is greater than 15 characters, e.g. // "10.192.95.89 xy.wildcard.example.com", it will be resolved through // DNS. if (FindCharInReadable(' ', iter, end)) { end = iter; } } for (host.BeginReading(iter); iter != end; iter++) { if (!(mozilla::IsAsciiHexDigit(*iter) || *iter == 'x' || *iter == 'X' || *iter == '.')) { // not an IP return; } } host.BeginReading(iter); nsTArray parts; ParseString(PromiseFlatCString(Substring(iter, end)), '.', parts); if (parts.Length() > 4) { return; } // If any potentially-octal numbers (start with 0 but not hex) have // non-octal digits, no part of the ip can be in octal // XXX: this came from the old javascript implementation, is it really // supposed to be like this? bool allowOctal = true; uint32_t i; for (i = 0; i < parts.Length(); i++) { const nsCString& part = parts[i]; if (part[0] == '0') { for (uint32_t j = 1; j < part.Length(); j++) { if (part[j] == 'x') { break; } if (part[j] == '8' || part[j] == '9') { allowOctal = false; break; } } } } for (i = 0; i < parts.Length(); i++) { nsAutoCString canonical; if (i == parts.Length() - 1) { CanonicalNum(parts[i], 5 - parts.Length(), allowOctal, canonical); } else { CanonicalNum(parts[i], 1, allowOctal, canonical); } if (canonical.IsEmpty()) { _retval.Truncate(); return; } if (_retval.IsEmpty()) { _retval.Assign(canonical); } else { _retval.Append('.'); _retval.Append(canonical); } } } void nsUrlClassifierUtils::CanonicalNum(const nsACString& num, uint32_t bytes, bool allowOctal, nsACString& _retval) { _retval.Truncate(); if (num.Length() < 1) { return; } uint32_t val; if (allowOctal && IsOctal(num)) { if (PR_sscanf(PromiseFlatCString(num).get(), "%o", &val) != 1) { return; } } else if (IsDecimal(num)) { if (PR_sscanf(PromiseFlatCString(num).get(), "%u", &val) != 1) { return; } } else if (IsHex(num)) { if (PR_sscanf(PromiseFlatCString(num).get(), num[1] == 'X' ? "0X%x" : "0x%x", &val) != 1) { return; } } else { return; } while (bytes--) { char buf[20]; SprintfLiteral(buf, "%u", val & 0xff); if (_retval.IsEmpty()) { _retval.Assign(buf); } else { _retval = nsDependentCString(buf) + "."_ns + _retval; } val >>= 8; } } // This function will encode all "special" characters in typical url // encoding, that is %hh where h is a valid hex digit. It will also fold // any duplicated slashes. bool nsUrlClassifierUtils::SpecialEncode(const nsACString& url, bool foldSlashes, nsACString& _retval) { bool changed = false; const char* curChar = url.BeginReading(); const char* end = url.EndReading(); unsigned char lastChar = '\0'; while (curChar != end) { unsigned char c = static_cast(*curChar); if (ShouldURLEscape(c)) { _retval.Append('%'); _retval.Append(int_to_hex_digit(c / 16)); _retval.Append(int_to_hex_digit(c % 16)); changed = true; } else if (foldSlashes && (c == '/' && lastChar == '/')) { // skip } else { _retval.Append(*curChar); } lastChar = c; curChar++; } return changed; } bool nsUrlClassifierUtils::ShouldURLEscape(const unsigned char c) const { return c <= 32 || c == '%' || c == '#' || c >= 127; } // moztest- tables are built-in created in LookupCache, they contain hardcoded // url entries in it. moztest tables don't support updates. // static bool nsUrlClassifierUtils::IsMozTestTable(const nsACString& aTableName) { return StringBeginsWith(aTableName, "moztest-"_ns); } // test- tables are used by testcases and can add custom test entries // through update API. // static bool nsUrlClassifierUtils::IsTestTable(const nsACString& aTableName) { return IsMozTestTable(aTableName) || StringBeginsWith(aTableName, "test"_ns); } bool nsUrlClassifierUtils::IsInSafeMode() { static Maybe sIsInSafeMode; if (!sIsInSafeMode.isSome()) { nsCOMPtr appInfo = do_GetService("@mozilla.org/xre/runtime;1"); if (appInfo) { bool inSafeMode = false; appInfo->GetInSafeMode(&inSafeMode); sIsInSafeMode.emplace(inSafeMode); } } return sIsInSafeMode.value(); }