diff options
Diffstat (limited to 'toolkit/components/url-classifier/nsUrlClassifierUtils.cpp')
| -rw-r--r-- | toolkit/components/url-classifier/nsUrlClassifierUtils.cpp | 1125 |
1 files changed, 1125 insertions, 0 deletions
diff --git a/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp b/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp new file mode 100644 index 0000000000..da5e9cd451 --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierUtils.cpp @@ -0,0 +1,1125 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "chromium/safebrowsing.pb.h" +#include "nsEscape.h" +#include "nsString.h" +#include "nsIURI.h" +#include "nsIURIMutator.h" +#include "nsIURL.h" +#include "nsIXULRuntime.h" +#include "nsUrlClassifierUtils.h" +#include "nsTArray.h" +#include "nsReadableUtils.h" +#include "plbase64.h" +#include "nsPrintfCString.h" +#include "mozilla/ClearOnShutdown.h" +#include "mozilla/Sprintf.h" +#include "mozilla/StaticPtr.h" +#include "mozilla/Mutex.h" +#include "nsIRedirectHistoryEntry.h" +#include "nsIHttpChannelInternal.h" +#include "mozIThirdPartyUtil.h" +#include "nsIDocShell.h" +#include "mozilla/TextUtils.h" +#include "mozilla/Preferences.h" +#include "mozilla/Services.h" +#include "mozilla/Telemetry.h" +#include "nsNetUtil.h" +#include "nsIHttpChannel.h" +#include "nsIObserverService.h" +#include "nsIPrefBranch.h" +#include "nsIPrefService.h" +#include "nsPIDOMWindow.h" +#include "nsServiceManagerUtils.h" +#include "nsThreadManager.h" +#include "nsTHashSet.h" +#include "Classifier.h" +#include "Entries.h" +#include "prprf.h" +#include "prtime.h" + +#define DEFAULT_PROTOCOL_VERSION "2.2" + +using namespace mozilla; +using namespace mozilla::safebrowsing; + +static mozilla::StaticRefPtr<nsUrlClassifierUtils> gUrlClassifierUtils; + +static char int_to_hex_digit(int32_t i) { + NS_ASSERTION((i >= 0) && (i <= 15), "int too big in int_to_hex_digit"); + return static_cast<char>(((i < 10) ? (i + '0') : ((i - 10) + 'A'))); +} + +static bool IsDecimal(const nsACString& num) { + for (uint32_t i = 0; i < num.Length(); i++) { + if (!mozilla::IsAsciiDigit(num[i])) { + return false; + } + } + + return true; +} + +static bool IsHex(const nsACString& num) { + if (num.Length() < 3) { + return false; + } + + if (num[0] != '0' || !(num[1] == 'x' || num[1] == 'X')) { + return false; + } + + for (uint32_t i = 2; i < num.Length(); i++) { + if (!mozilla::IsAsciiHexDigit(num[i])) { + return false; + } + } + + return true; +} + +static bool IsOctal(const nsACString& num) { + if (num.Length() < 2) { + return false; + } + + if (num[0] != '0') { + return false; + } + + for (uint32_t i = 1; i < num.Length(); i++) { + if (!mozilla::IsAsciiDigit(num[i]) || num[i] == '8' || num[i] == '9') { + return false; + } + } + + return true; +} + +///////////////////////////////////////////////////////////////// +// SafeBrowsing V4 related utits. + +namespace mozilla { +namespace safebrowsing { + +static PlatformType GetPlatformType() { +#if defined(ANDROID) + return ANDROID_PLATFORM; +#elif defined(XP_MACOSX) + return OSX_PLATFORM; +#elif defined(XP_LINUX) + return LINUX_PLATFORM; +#elif defined(XP_WIN) + return WINDOWS_PLATFORM; +#else + // Default to Linux for other platforms (see bug 1362501). + return LINUX_PLATFORM; +#endif +} + +typedef FetchThreatListUpdatesRequest_ListUpdateRequest ListUpdateRequest; +typedef FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints Constraints; + +static void InitListUpdateRequest(ThreatType aThreatType, + const nsCString& aStateBase64, + ListUpdateRequest* aListUpdateRequest) { + aListUpdateRequest->set_threat_type(aThreatType); + PlatformType platform = GetPlatformType(); +#if defined(ANDROID) + // Temporary hack to fix bug 1441345. + if ((aThreatType == SOCIAL_ENGINEERING_PUBLIC) || + (aThreatType == SOCIAL_ENGINEERING)) { + platform = LINUX_PLATFORM; + } +#endif + aListUpdateRequest->set_platform_type(platform); + aListUpdateRequest->set_threat_entry_type(URL); + + Constraints* contraints = new Constraints(); + contraints->add_supported_compressions(RICE); + aListUpdateRequest->set_allocated_constraints(contraints); + + // Only set non-empty state. + if (!aStateBase64.IsEmpty()) { + nsCString stateBinary; + nsresult rv = Base64Decode(aStateBase64, stateBinary); + if (NS_SUCCEEDED(rv)) { + aListUpdateRequest->set_state(stateBinary.get(), stateBinary.Length()); + } + } +} + +static ClientInfo* CreateClientInfo() { + ClientInfo* c = new ClientInfo(); + + nsCOMPtr<nsIPrefBranch> prefBranch = do_GetService(NS_PREFSERVICE_CONTRACTID); + + nsAutoCString clientId; + nsresult rv = prefBranch->GetCharPref("browser.safebrowsing.id", clientId); + + if (NS_FAILED(rv)) { + clientId = "Firefox"; // Use "Firefox" as fallback. + } + + c->set_client_id(clientId.get()); + + return c; +} + +static bool IsAllowedOnCurrentPlatform(uint32_t aThreatType) { + PlatformType platform = GetPlatformType(); + + switch (aThreatType) { + case POTENTIALLY_HARMFUL_APPLICATION: + // Bug 1388582 - Google server would respond 404 error if the request + // contains PHA on non-mobile platform. + return ANDROID_PLATFORM == platform; + case MALICIOUS_BINARY: + case CSD_DOWNLOAD_WHITELIST: + // Bug 1392204 - 'goog-downloadwhite-proto' and 'goog-badbinurl-proto' + // are not available on android. + return ANDROID_PLATFORM != platform; + } + // We allow every threat type not listed in the switch cases. + return true; +} + +} // end of namespace safebrowsing. +} // end of namespace mozilla. + +// static +already_AddRefed<nsUrlClassifierUtils> +nsUrlClassifierUtils::GetXPCOMSingleton() { + if (gUrlClassifierUtils) { + return do_AddRef(gUrlClassifierUtils); + } + + RefPtr<nsUrlClassifierUtils> utils = new nsUrlClassifierUtils(); + if (NS_WARN_IF(NS_FAILED(utils->Init()))) { + return nullptr; + } + + // Note: This is cleared in the nsUrlClassifierUtils destructor. + gUrlClassifierUtils = utils.get(); + ClearOnShutdown(&gUrlClassifierUtils); + return utils.forget(); +} + +// static +nsUrlClassifierUtils* nsUrlClassifierUtils::GetInstance() { + if (!gUrlClassifierUtils) { + RefPtr<nsUrlClassifierUtils> utils = GetXPCOMSingleton(); + } + + return gUrlClassifierUtils; +} + +nsUrlClassifierUtils::nsUrlClassifierUtils() + : mProviderDictLock("nsUrlClassifierUtils.mProviderDictLock") {} + +nsUrlClassifierUtils::~nsUrlClassifierUtils() { + if (gUrlClassifierUtils) { + MOZ_ASSERT(gUrlClassifierUtils == this); + gUrlClassifierUtils = nullptr; + } +} + +nsresult nsUrlClassifierUtils::Init() { + // nsIUrlClassifierUtils is a thread-safe service so it's + // allowed to use on non-main threads. However, building + // the provider dictionary must be on the main thread. + // We forcefully load nsUrlClassifierUtils in + // nsUrlClassifierDBService::Init() to ensure we must + // now be on the main thread. + nsresult rv = ReadProvidersFromPrefs(mProviderDict); + NS_ENSURE_SUCCESS(rv, rv); + + // Add an observer for shutdown + nsCOMPtr<nsIObserverService> observerService = + mozilla::services::GetObserverService(); + if (!observerService) return NS_ERROR_FAILURE; + + observerService->AddObserver(this, "xpcom-shutdown-threads", false); + mozilla::Preferences::AddStrongObserver(this, "browser.safebrowsing"); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(nsUrlClassifierUtils, nsIUrlClassifierUtils, nsIObserver) + +///////////////////////////////////////////////////////////////////////////// +// nsIUrlClassifierUtils + +NS_IMETHODIMP +nsUrlClassifierUtils::GetKeyForURI(nsIURI* uri, nsACString& _retval) { + nsCOMPtr<nsIURI> innerURI = NS_GetInnermostURI(uri); + if (!innerURI) innerURI = uri; + + nsAutoCString host; + innerURI->GetAsciiHost(host); + + if (host.IsEmpty()) { + return NS_ERROR_MALFORMED_URI; + } + + nsresult rv = CanonicalizeHostname(host, _retval); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString path; + rv = innerURI->GetPathQueryRef(path); + NS_ENSURE_SUCCESS(rv, rv); + + // Strip fragment and query because canonicalization only applies to path + int32_t ref = path.FindChar('#'); + if (ref != kNotFound) { + path.SetLength(ref); + } + + int32_t query = path.FindChar('?'); + if (query != kNotFound) { + path.SetLength(query); + } + + nsAutoCString temp; + rv = CanonicalizePath(path, temp); + NS_ENSURE_SUCCESS(rv, rv); + + _retval.Append(temp); + + if (query != kNotFound) { + nsAutoCString query; + rv = innerURI->GetQuery(query); + NS_ENSURE_SUCCESS(rv, rv); + + _retval.AppendPrintf("?%s", query.get()); + } + + return NS_OK; +} + +// We use "goog-*-proto" as the list name for v4, where "proto" indicates +// it's updated (as well as hash completion) via protobuf. +// +// In the mozilla official build, we are allowed to use the +// private phishing list (goog-phish-proto). See Bug 1288840. +static const struct { + const char* mListName; + uint32_t mThreatType; +} THREAT_TYPE_CONV_TABLE[] = { + {"goog-malware-proto", MALWARE_THREAT}, // 1 + {"googpub-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2 + {"goog-unwanted-proto", UNWANTED_SOFTWARE}, // 3 + {"goog-harmful-proto", POTENTIALLY_HARMFUL_APPLICATION}, // 4 + {"goog-phish-proto", SOCIAL_ENGINEERING}, // 5 + + // For application reputation + {"goog-badbinurl-proto", MALICIOUS_BINARY}, // 7 + {"goog-downloadwhite-proto", CSD_DOWNLOAD_WHITELIST}, // 9 + + // For testing purpose. + {"moztest-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2 + {"test-phish-proto", SOCIAL_ENGINEERING_PUBLIC}, // 2 + {"moztest-unwanted-proto", UNWANTED_SOFTWARE}, // 3 + {"test-unwanted-proto", UNWANTED_SOFTWARE}, // 3 +}; + +NS_IMETHODIMP +nsUrlClassifierUtils::ConvertThreatTypeToListNames(uint32_t aThreatType, + nsACString& aListNames) { + for (uint32_t i = 0; i < ArrayLength(THREAT_TYPE_CONV_TABLE); i++) { + if (aThreatType == THREAT_TYPE_CONV_TABLE[i].mThreatType) { + if (!aListNames.IsEmpty()) { + aListNames.AppendLiteral(","); + } + aListNames += THREAT_TYPE_CONV_TABLE[i].mListName; + } + } + + return aListNames.IsEmpty() ? NS_ERROR_FAILURE : NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierUtils::ConvertListNameToThreatType(const nsACString& aListName, + uint32_t* aThreatType) { + for (uint32_t i = 0; i < ArrayLength(THREAT_TYPE_CONV_TABLE); i++) { + if (aListName.EqualsASCII(THREAT_TYPE_CONV_TABLE[i].mListName)) { + *aThreatType = THREAT_TYPE_CONV_TABLE[i].mThreatType; + return NS_OK; + } + } + + return NS_ERROR_FAILURE; +} + +NS_IMETHODIMP +nsUrlClassifierUtils::GetProvider(const nsACString& aTableName, + nsACString& aProvider) { + MutexAutoLock lock(mProviderDictLock); + nsCString* provider = nullptr; + + if (IsTestTable(aTableName)) { + aProvider = nsLiteralCString(TESTING_TABLE_PROVIDER_NAME); + } else if (mProviderDict.Get(aTableName, &provider)) { + aProvider = provider ? *provider : ""_ns; + } else { + aProvider.Truncate(); + } + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierUtils::GetTelemetryProvider(const nsACString& aTableName, + nsACString& aProvider) { + GetProvider(aTableName, aProvider); + // Exceptionlist known providers to avoid reporting on private ones. + // An empty provider is treated as "other" + if (!"mozilla"_ns.Equals(aProvider) && !"google"_ns.Equals(aProvider) && + !"google4"_ns.Equals(aProvider) && !"baidu"_ns.Equals(aProvider) && + !"mozcn"_ns.Equals(aProvider) && !"yandex"_ns.Equals(aProvider) && + !nsLiteralCString(TESTING_TABLE_PROVIDER_NAME).Equals(aProvider)) { + aProvider.AssignLiteral("other"); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierUtils::GetProtocolVersion(const nsACString& aProvider, + nsACString& aVersion) { + nsCOMPtr<nsIPrefBranch> prefBranch = do_GetService(NS_PREFSERVICE_CONTRACTID); + if (prefBranch) { + nsPrintfCString prefName("browser.safebrowsing.provider.%s.pver", + nsCString(aProvider).get()); + nsAutoCString version; + nsresult rv = prefBranch->GetCharPref(prefName.get(), version); + + aVersion = NS_SUCCEEDED(rv) ? version.get() : DEFAULT_PROTOCOL_VERSION; + } else { + aVersion = DEFAULT_PROTOCOL_VERSION; + } + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierUtils::MakeUpdateRequestV4( + const nsTArray<nsCString>& aListNames, + const nsTArray<nsCString>& aStatesBase64, nsACString& aRequest) { + using namespace mozilla::safebrowsing; + + if (aListNames.Length() != aStatesBase64.Length()) { + return NS_ERROR_INVALID_ARG; + } + + FetchThreatListUpdatesRequest r; + r.set_allocated_client(CreateClientInfo()); + + for (uint32_t i = 0; i < aListNames.Length(); i++) { + uint32_t threatType; + nsresult rv = ConvertListNameToThreatType(aListNames[i], &threatType); + if (NS_FAILED(rv)) { + continue; // Unknown list name. + } + if (!IsAllowedOnCurrentPlatform(threatType)) { + NS_WARNING( + nsPrintfCString( + "Threat type %d (%s) is unsupported on current platform: %d", + threatType, aListNames[i].get(), GetPlatformType()) + .get()); + continue; // Some threat types are not available on some platforms. + } + auto lur = r.mutable_list_update_requests()->Add(); + InitListUpdateRequest(static_cast<ThreatType>(threatType), aStatesBase64[i], + lur); + } + + // Then serialize. + std::string s; + r.SerializeToString(&s); + + nsCString out; + nsresult rv = Base64URLEncode(s.size(), (const uint8_t*)s.c_str(), + Base64URLEncodePaddingPolicy::Include, out); + NS_ENSURE_SUCCESS(rv, rv); + + aRequest = out; + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierUtils::MakeFindFullHashRequestV4( + const nsTArray<nsCString>& aListNames, + const nsTArray<nsCString>& aListStatesBase64, + const nsTArray<nsCString>& aPrefixesBase64, nsACString& aRequest) { + if (aListNames.Length() != aListStatesBase64.Length()) { + return NS_ERROR_INVALID_ARG; + } + + FindFullHashesRequest r; + r.set_allocated_client(CreateClientInfo()); + + nsresult rv; + + //------------------------------------------------------------------- + // Set up FindFullHashesRequest.threat_info. + auto threatInfo = r.mutable_threat_info(); + + PlatformType platform = GetPlatformType(); + + // 1) Set threat types. + for (uint32_t i = 0; i < aListNames.Length(); i++) { + // Add threat types. + uint32_t threatType; + rv = ConvertListNameToThreatType(aListNames[i], &threatType); + NS_ENSURE_SUCCESS(rv, rv); + if (!IsAllowedOnCurrentPlatform(threatType)) { + NS_WARNING( + nsPrintfCString( + "Threat type %d (%s) is unsupported on current platform: %d", + threatType, aListNames[i].get(), GetPlatformType()) + .get()); + continue; + } + threatInfo->add_threat_types((ThreatType)threatType); + +#if defined(ANDROID) + // Temporary hack to fix bug 1441345. + if (((ThreatType)threatType == SOCIAL_ENGINEERING_PUBLIC) || + ((ThreatType)threatType == SOCIAL_ENGINEERING)) { + platform = LINUX_PLATFORM; + } +#endif + + // Add client states for index 'i' only when the threat type is available + // on current platform. + nsCString stateBinary; + rv = Base64Decode(aListStatesBase64[i], stateBinary); + NS_ENSURE_SUCCESS(rv, rv); + r.add_client_states(stateBinary.get(), stateBinary.Length()); + } + + // 2) Set platform type. + threatInfo->add_platform_types(platform); + + // 3) Set threat entry type. + threatInfo->add_threat_entry_types(URL); + + // 4) Set threat entries. + for (const nsCString& prefix : aPrefixesBase64) { + nsCString prefixBinary; + rv = Base64Decode(prefix, prefixBinary); + threatInfo->add_threat_entries()->set_hash(prefixBinary.get(), + prefixBinary.Length()); + } + //------------------------------------------------------------------- + + // Then serialize. + std::string s; + r.SerializeToString(&s); + + nsCString out; + rv = Base64URLEncode(s.size(), (const uint8_t*)s.c_str(), + Base64URLEncodePaddingPolicy::Include, out); + NS_ENSURE_SUCCESS(rv, rv); + + aRequest = out; + + return NS_OK; +} + +// Remove ref, query, userpass, anypart which may contain sensitive data +static nsresult GetSpecWithoutSensitiveData(nsIURI* aUri, nsACString& aSpec) { + if (NS_WARN_IF(!aUri)) { + return NS_ERROR_INVALID_ARG; + } + + nsresult rv; + nsCOMPtr<nsIURL> url(do_QueryInterface(aUri)); + if (url) { + nsCOMPtr<nsIURI> clone; + rv = NS_MutateURI(url) + .SetQuery(""_ns) + .SetRef(""_ns) + .SetUserPass(""_ns) + .Finalize(clone); + NS_ENSURE_SUCCESS(rv, rv); + rv = clone->GetAsciiSpec(aSpec); + NS_ENSURE_SUCCESS(rv, rv); + } + return NS_OK; +} + +static nsresult AddThreatSourceFromChannel(ThreatHit& aHit, + nsIChannel* aChannel, + ThreatHit_ThreatSourceType aType) { + if (NS_WARN_IF(!aChannel)) { + return NS_ERROR_INVALID_ARG; + } + + nsresult rv; + + auto matchingSource = aHit.add_resources(); + matchingSource->set_type(aType); + + nsCOMPtr<nsIURI> uri; + rv = NS_GetFinalChannelURI(aChannel, getter_AddRefs(uri)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCString spec; + rv = GetSpecWithoutSensitiveData(uri, spec); + NS_ENSURE_SUCCESS(rv, rv); + matchingSource->set_url(spec.get()); + + nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(aChannel); + if (httpChannel) { + nsCOMPtr<nsIReferrerInfo> referrerInfo = httpChannel->GetReferrerInfo(); + if (referrerInfo) { + nsAutoCString referrerSpec; + nsCOMPtr<nsIURI> referrer = referrerInfo->GetComputedReferrer(); + if (referrer) { + rv = GetSpecWithoutSensitiveData(referrer, referrerSpec); + NS_ENSURE_SUCCESS(rv, rv); + matchingSource->set_referrer(referrerSpec.get()); + } + } + } + + nsCOMPtr<nsIHttpChannelInternal> httpChannelInternal = + do_QueryInterface(aChannel); + if (httpChannelInternal) { + nsCString remoteIp; + rv = httpChannelInternal->GetRemoteAddress(remoteIp); + if (NS_SUCCEEDED(rv) && !remoteIp.IsEmpty()) { + matchingSource->set_remote_ip(remoteIp.get()); + } + } + return NS_OK; +} +static nsresult AddThreatSourceFromRedirectEntry( + ThreatHit& aHit, nsIRedirectHistoryEntry* aRedirectEntry, + ThreatHit_ThreatSourceType aType) { + if (NS_WARN_IF(!aRedirectEntry)) { + return NS_ERROR_INVALID_ARG; + } + + nsresult rv; + + nsCOMPtr<nsIPrincipal> principal; + rv = aRedirectEntry->GetPrincipal(getter_AddRefs(principal)); + NS_ENSURE_SUCCESS(rv, rv); + nsCString spec; + rv = principal->GetExposableSpec(spec); + NS_ENSURE_SUCCESS(rv, rv); + auto source = aHit.add_resources(); + source->set_url(spec.get()); + source->set_type(aType); + + nsCOMPtr<nsIURI> referrer; + rv = aRedirectEntry->GetReferrerURI(getter_AddRefs(referrer)); + if (NS_SUCCEEDED(rv) && referrer) { + nsCString referrerSpec; + rv = GetSpecWithoutSensitiveData(referrer, referrerSpec); + NS_ENSURE_SUCCESS(rv, rv); + source->set_referrer(referrerSpec.get()); + } + + nsCString remoteIp; + rv = aRedirectEntry->GetRemoteAddress(remoteIp); + if (NS_SUCCEEDED(rv) && !remoteIp.IsEmpty()) { + source->set_remote_ip(remoteIp.get()); + } + return NS_OK; +} + +// Add top level tab url and redirect threatsources to threatHit message +static nsresult AddTabThreatSources(ThreatHit& aHit, nsIChannel* aChannel) { + if (NS_WARN_IF(!aChannel)) { + return NS_ERROR_INVALID_ARG; + } + + nsresult rv; + nsCOMPtr<mozIDOMWindowProxy> win; + nsCOMPtr<mozIThirdPartyUtil> thirdPartyUtil = + do_GetService(THIRDPARTYUTIL_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = thirdPartyUtil->GetTopWindowForChannel(aChannel, nullptr, + getter_AddRefs(win)); + NS_ENSURE_SUCCESS(rv, rv); + + auto* pwin = nsPIDOMWindowOuter::From(win); + nsCOMPtr<nsIDocShell> docShell = pwin->GetDocShell(); + if (!docShell) { + return NS_OK; + } + + nsCOMPtr<nsIChannel> topChannel; + docShell->GetCurrentDocumentChannel(getter_AddRefs(topChannel)); + if (!topChannel) { + return NS_OK; + } + + nsCOMPtr<nsIURI> uri; + rv = aChannel->GetURI(getter_AddRefs(uri)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIURI> topUri; + rv = topChannel->GetURI(getter_AddRefs(topUri)); + NS_ENSURE_SUCCESS(rv, rv); + + bool isTopUri = false; + rv = topUri->Equals(uri, &isTopUri); + if (NS_SUCCEEDED(rv) && !isTopUri) { + nsCOMPtr<nsILoadInfo> loadInfo = aChannel->LoadInfo(); + if (loadInfo->RedirectChain().Length()) { + AddThreatSourceFromRedirectEntry(aHit, loadInfo->RedirectChain()[0], + ThreatHit_ThreatSourceType_TAB_RESOURCE); + } + } + + // Set top level tab_url threat source + rv = AddThreatSourceFromChannel(aHit, topChannel, + ThreatHit_ThreatSourceType_TAB_URL); + Unused << NS_WARN_IF(NS_FAILED(rv)); + + // Set tab_redirect threat sources if there's any + nsCOMPtr<nsILoadInfo> topLoadInfo = topChannel->LoadInfo(); + nsIRedirectHistoryEntry* redirectEntry; + size_t length = topLoadInfo->RedirectChain().Length(); + for (size_t i = 0; i < length; i++) { + redirectEntry = topLoadInfo->RedirectChain()[i]; + AddThreatSourceFromRedirectEntry(aHit, redirectEntry, + ThreatHit_ThreatSourceType_TAB_REDIRECT); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierUtils::MakeThreatHitReport(nsIChannel* aChannel, + const nsACString& aListName, + const nsACString& aHashBase64, + nsACString& aRequest) { + if (NS_WARN_IF(aListName.IsEmpty()) || NS_WARN_IF(aHashBase64.IsEmpty()) || + NS_WARN_IF(!aChannel)) { + return NS_ERROR_INVALID_ARG; + } + + ThreatHit hit; + nsresult rv; + + uint32_t threatType; + rv = ConvertListNameToThreatType(aListName, &threatType); + NS_ENSURE_SUCCESS(rv, rv); + hit.set_threat_type(static_cast<ThreatType>(threatType)); + + hit.set_platform_type(GetPlatformType()); + + nsCString hash; + rv = Base64Decode(aHashBase64, hash); + if (NS_FAILED(rv) || hash.Length() != COMPLETE_SIZE) { + return NS_ERROR_FAILURE; + } + + auto threatEntry = hit.mutable_entry(); + threatEntry->set_hash(hash.get(), hash.Length()); + + // Set matching source + rv = AddThreatSourceFromChannel(hit, aChannel, + ThreatHit_ThreatSourceType_MATCHING_URL); + Unused << NS_WARN_IF(NS_FAILED(rv)); + // Set tab url, tab resource url and redirect sources + rv = AddTabThreatSources(hit, aChannel); + Unused << NS_WARN_IF(NS_FAILED(rv)); + + hit.set_allocated_client_info(CreateClientInfo()); + + std::string s; + hit.SerializeToString(&s); + + nsCString out; + rv = Base64URLEncode(s.size(), reinterpret_cast<const uint8_t*>(s.c_str()), + Base64URLEncodePaddingPolicy::Include, out); + NS_ENSURE_SUCCESS(rv, rv); + + aRequest = out; + + return NS_OK; +} + +static uint32_t DurationToMs(const Duration& aDuration) { + // Seconds precision is good enough. Ignore nanoseconds like Chrome does. + return aDuration.seconds() * 1000; +} + +NS_IMETHODIMP +nsUrlClassifierUtils::ParseFindFullHashResponseV4( + const nsACString& aResponse, + nsIUrlClassifierParseFindFullHashCallback* aCallback) { + enum CompletionErrorType { + SUCCESS = 0, + PARSING_FAILURE = 1, + UNKNOWN_THREAT_TYPE = 2, + }; + + FindFullHashesResponse r; + if (!r.ParseFromArray(aResponse.BeginReading(), aResponse.Length())) { + NS_WARNING("Invalid response"); + Telemetry::Accumulate(Telemetry::URLCLASSIFIER_COMPLETION_ERROR, + PARSING_FAILURE); + return NS_ERROR_FAILURE; + } + + bool hasUnknownThreatType = false; + + for (auto& m : r.matches()) { + nsCString tableNames; + nsresult rv = ConvertThreatTypeToListNames(m.threat_type(), tableNames); + if (NS_FAILED(rv)) { + hasUnknownThreatType = true; + continue; // Ignore un-convertable threat type. + } + auto& hash = m.threat().hash(); + auto cacheDurationSec = m.cache_duration().seconds(); + aCallback->OnCompleteHashFound( + nsDependentCString(hash.c_str(), hash.length()), tableNames, + cacheDurationSec); + } + + auto minWaitDuration = DurationToMs(r.minimum_wait_duration()); + auto negCacheDurationSec = r.negative_cache_duration().seconds(); + + aCallback->OnResponseParsed(minWaitDuration, negCacheDurationSec); + + Telemetry::Accumulate(Telemetry::URLCLASSIFIER_COMPLETION_ERROR, + hasUnknownThreatType ? UNKNOWN_THREAT_TYPE : SUCCESS); + return NS_OK; +} + +////////////////////////////////////////////////////////// +// nsIObserver + +NS_IMETHODIMP +nsUrlClassifierUtils::Observe(nsISupports* aSubject, const char* aTopic, + const char16_t* aData) { + if (0 == strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) { + MutexAutoLock lock(mProviderDictLock); + return ReadProvidersFromPrefs(mProviderDict); + } + + if (0 == strcmp(aTopic, "xpcom-shutdown-threads")) { + nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID); + NS_ENSURE_TRUE(prefs, NS_ERROR_FAILURE); + return prefs->RemoveObserver("browser.safebrowsing", this); + } + + return NS_ERROR_UNEXPECTED; +} + +///////////////////////////////////////////////////////////////////////////// +// non-interface methods + +nsresult nsUrlClassifierUtils::ReadProvidersFromPrefs(ProviderDictType& aDict) { + MOZ_ASSERT(NS_IsMainThread(), + "ReadProvidersFromPrefs must be on main thread"); + + nsCOMPtr<nsIPrefService> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID); + NS_ENSURE_TRUE(prefs, NS_ERROR_FAILURE); + nsCOMPtr<nsIPrefBranch> prefBranch; + nsresult rv = prefs->GetBranch("browser.safebrowsing.provider.", + getter_AddRefs(prefBranch)); + NS_ENSURE_SUCCESS(rv, rv); + + // We've got a pref branch for "browser.safebrowsing.provider.". + // Enumerate all children prefs and parse providers. + nsTArray<nsCString> childArray; + rv = prefBranch->GetChildList("", childArray); + NS_ENSURE_SUCCESS(rv, rv); + + // Collect providers from childArray. + nsTHashSet<nsCString> providers; + for (auto& child : childArray) { + auto dotPos = child.FindChar('.'); + if (dotPos < 0) { + continue; + } + + nsDependentCSubstring provider = Substring(child, 0, dotPos); + + providers.Insert(provider); + } + + // Now we have all providers. Check which one owns |aTableName|. + // e.g. The owning lists of provider "google" is defined in + // "browser.safebrowsing.provider.google.lists". + for (const auto& provider : providers) { + nsPrintfCString owninListsPref("%s.lists", + nsPromiseFlatCString{provider}.get()); + + nsAutoCString owningLists; + nsresult rv = prefBranch->GetCharPref(owninListsPref.get(), owningLists); + if (NS_FAILED(rv)) { + continue; + } + + // We've got the owning lists (represented as string) of |provider|. + // Build the dictionary for the owning list and the current provider. + nsTArray<nsCString> tables; + Classifier::SplitTables(owningLists, tables); + for (auto tableName : tables) { + aDict.InsertOrUpdate(tableName, MakeUnique<nsCString>(provider)); + } + } + + return NS_OK; +} + +nsresult nsUrlClassifierUtils::CanonicalizeHostname(const nsACString& hostname, + nsACString& _retval) { + nsAutoCString unescaped; + if (!NS_UnescapeURL(PromiseFlatCString(hostname).get(), + PromiseFlatCString(hostname).Length(), 0, unescaped)) { + unescaped.Assign(hostname); + } + + nsAutoCString cleaned; + CleanupHostname(unescaped, cleaned); + + nsAutoCString temp; + ParseIPAddress(cleaned, temp); + if (!temp.IsEmpty()) { + cleaned.Assign(temp); + } + + ToLowerCase(cleaned); + SpecialEncode(cleaned, false, _retval); + + return NS_OK; +} + +nsresult nsUrlClassifierUtils::CanonicalizePath(const nsACString& path, + nsACString& _retval) { + _retval.Truncate(); + + nsAutoCString decodedPath(path); + nsAutoCString temp; + while (NS_UnescapeURL(decodedPath.get(), decodedPath.Length(), 0, temp)) { + decodedPath.Assign(temp); + temp.Truncate(); + } + + SpecialEncode(decodedPath, true, _retval); + // XXX: lowercase the path? + + return NS_OK; +} + +void nsUrlClassifierUtils::CleanupHostname(const nsACString& hostname, + nsACString& _retval) { + _retval.Truncate(); + + const char* curChar = hostname.BeginReading(); + const char* end = hostname.EndReading(); + char lastChar = '\0'; + while (curChar != end) { + unsigned char c = static_cast<unsigned char>(*curChar); + if (c == '.' && (lastChar == '\0' || lastChar == '.')) { + // skip + } else { + _retval.Append(*curChar); + } + lastChar = c; + ++curChar; + } + + // cut off trailing dots + while (_retval.Length() > 0 && _retval[_retval.Length() - 1] == '.') { + _retval.SetLength(_retval.Length() - 1); + } +} + +void nsUrlClassifierUtils::ParseIPAddress(const nsACString& host, + nsACString& _retval) { + _retval.Truncate(); + nsACString::const_iterator iter, end; + host.BeginReading(iter); + host.EndReading(end); + + if (host.Length() <= 15) { + // The Windows resolver allows a 4-part dotted decimal IP address to + // have a space followed by any old rubbish, so long as the total length + // of the string doesn't get above 15 characters. So, "10.192.95.89 xy" + // is resolved to 10.192.95.89. + // If the string length is greater than 15 characters, e.g. + // "10.192.95.89 xy.wildcard.example.com", it will be resolved through + // DNS. + + if (FindCharInReadable(' ', iter, end)) { + end = iter; + } + } + + for (host.BeginReading(iter); iter != end; iter++) { + if (!(mozilla::IsAsciiHexDigit(*iter) || *iter == 'x' || *iter == 'X' || + *iter == '.')) { + // not an IP + return; + } + } + + host.BeginReading(iter); + nsTArray<nsCString> parts; + ParseString(PromiseFlatCString(Substring(iter, end)), '.', parts); + if (parts.Length() > 4) { + return; + } + + // If any potentially-octal numbers (start with 0 but not hex) have + // non-octal digits, no part of the ip can be in octal + // XXX: this came from the old javascript implementation, is it really + // supposed to be like this? + bool allowOctal = true; + uint32_t i; + + for (i = 0; i < parts.Length(); i++) { + const nsCString& part = parts[i]; + if (part[0] == '0') { + for (uint32_t j = 1; j < part.Length(); j++) { + if (part[j] == 'x') { + break; + } + if (part[j] == '8' || part[j] == '9') { + allowOctal = false; + break; + } + } + } + } + + for (i = 0; i < parts.Length(); i++) { + nsAutoCString canonical; + + if (i == parts.Length() - 1) { + CanonicalNum(parts[i], 5 - parts.Length(), allowOctal, canonical); + } else { + CanonicalNum(parts[i], 1, allowOctal, canonical); + } + + if (canonical.IsEmpty()) { + _retval.Truncate(); + return; + } + + if (_retval.IsEmpty()) { + _retval.Assign(canonical); + } else { + _retval.Append('.'); + _retval.Append(canonical); + } + } +} + +void nsUrlClassifierUtils::CanonicalNum(const nsACString& num, uint32_t bytes, + bool allowOctal, nsACString& _retval) { + _retval.Truncate(); + + if (num.Length() < 1) { + return; + } + + uint32_t val; + if (allowOctal && IsOctal(num)) { + if (PR_sscanf(PromiseFlatCString(num).get(), "%o", &val) != 1) { + return; + } + } else if (IsDecimal(num)) { + if (PR_sscanf(PromiseFlatCString(num).get(), "%u", &val) != 1) { + return; + } + } else if (IsHex(num)) { + if (PR_sscanf(PromiseFlatCString(num).get(), + num[1] == 'X' ? "0X%x" : "0x%x", &val) != 1) { + return; + } + } else { + return; + } + + while (bytes--) { + char buf[20]; + SprintfLiteral(buf, "%u", val & 0xff); + if (_retval.IsEmpty()) { + _retval.Assign(buf); + } else { + _retval = nsDependentCString(buf) + "."_ns + _retval; + } + val >>= 8; + } +} + +// This function will encode all "special" characters in typical url +// encoding, that is %hh where h is a valid hex digit. It will also fold +// any duplicated slashes. +bool nsUrlClassifierUtils::SpecialEncode(const nsACString& url, + bool foldSlashes, + nsACString& _retval) { + bool changed = false; + const char* curChar = url.BeginReading(); + const char* end = url.EndReading(); + + unsigned char lastChar = '\0'; + while (curChar != end) { + unsigned char c = static_cast<unsigned char>(*curChar); + if (ShouldURLEscape(c)) { + _retval.Append('%'); + _retval.Append(int_to_hex_digit(c / 16)); + _retval.Append(int_to_hex_digit(c % 16)); + + changed = true; + } else if (foldSlashes && (c == '/' && lastChar == '/')) { + // skip + } else { + _retval.Append(*curChar); + } + lastChar = c; + curChar++; + } + return changed; +} + +bool nsUrlClassifierUtils::ShouldURLEscape(const unsigned char c) const { + return c <= 32 || c == '%' || c == '#' || c >= 127; +} + +// moztest- tables are built-in created in LookupCache, they contain hardcoded +// url entries in it. moztest tables don't support updates. +// static +bool nsUrlClassifierUtils::IsMozTestTable(const nsACString& aTableName) { + return StringBeginsWith(aTableName, "moztest-"_ns); +} + +// test- tables are used by testcases and can add custom test entries +// through update API. +// static +bool nsUrlClassifierUtils::IsTestTable(const nsACString& aTableName) { + return IsMozTestTable(aTableName) || StringBeginsWith(aTableName, "test"_ns); +} + +bool nsUrlClassifierUtils::IsInSafeMode() { + static Maybe<bool> sIsInSafeMode; + + if (!sIsInSafeMode.isSome()) { + nsCOMPtr<nsIXULRuntime> appInfo = + do_GetService("@mozilla.org/xre/runtime;1"); + if (appInfo) { + bool inSafeMode = false; + appInfo->GetInSafeMode(&inSafeMode); + sIsInSafeMode.emplace(inSafeMode); + } + } + + return sIsInSafeMode.value(); +} |