path: root/parser/html/nsHtml5StreamParser.cpp

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=2 et tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "nsHtml5StreamParser.h"

#include <stdlib.h>
#include <string.h>
#include <algorithm>
#include <new>
#include <type_traits>
#include <utility>
#include "ErrorList.h"
#include "GeckoProfiler.h"
#include "js/GCAPI.h"
#include "mozilla/ArrayIterator.h"
#include "mozilla/Buffer.h"
#include "mozilla/CheckedInt.h"
#include "mozilla/DebugOnly.h"
#include "mozilla/Encoding.h"
#include "mozilla/EncodingDetector.h"
#include "mozilla/Likely.h"
#include "mozilla/Maybe.h"
#include "mozilla/SchedulerGroup.h"
#include "mozilla/ScopeExit.h"
#include "mozilla/Services.h"
#include "mozilla/StaticPrefs_html5.h"
#include "mozilla/StaticPrefs_intl.h"
#include "mozilla/TaskCategory.h"
#include "mozilla/TextUtils.h"
#include "mozilla/Tuple.h"
#include "mozilla/UniquePtrExtensions.h"
#include "mozilla/Unused.h"
#include "mozilla/dom/BindingDeclarations.h"
#include "mozilla/dom/BrowsingContext.h"
#include "mozilla/dom/DebuggerUtilsBinding.h"
#include "mozilla/dom/DocGroup.h"
#include "mozilla/dom/Document.h"
#include "mozilla/mozalloc.h"
#include "mozilla/Vector.h"
#include "nsContentSink.h"
#include "nsContentUtils.h"
#include "nsCycleCollectionTraversalCallback.h"
#include "nsHtml5AtomTable.h"
#include "nsHtml5ByteReadable.h"
#include "nsHtml5Highlighter.h"
#include "nsHtml5Module.h"
#include "nsHtml5OwningUTF16Buffer.h"
#include "nsHtml5Parser.h"
#include "nsHtml5Speculation.h"
#include "nsHtml5StreamParserPtr.h"
#include "nsHtml5Tokenizer.h"
#include "nsHtml5TreeBuilder.h"
#include "nsHtml5TreeOpExecutor.h"
#include "nsHtml5TreeOpStage.h"
#include "nsIChannel.h"
#include "nsIContentSink.h"
#include "nsID.h"
#include "nsIDTD.h"
#include "nsIDocShell.h"
#include "nsIEventTarget.h"
#include "nsIHttpChannel.h"
#include "nsIInputStream.h"
#include "nsINestedURI.h"
#include "nsIObserverService.h"
#include "nsIRequest.h"
#include "nsIRunnable.h"
#include "nsIScriptError.h"
#include "nsIThread.h"
#include "nsIThreadRetargetableRequest.h"
#include "nsIThreadRetargetableStreamListener.h"
#include "nsITimer.h"
#include "nsIURI.h"
#include "nsJSEnvironment.h"
#include "nsLiteralString.h"
#include "nsNetUtil.h"
#include "nsString.h"
#include "nsTPromiseFlatString.h"
#include "nsThreadUtils.h"
#include "nsXULAppAPI.h"

extern "C" {
// Defined in intl/encoding_glue/src/lib.rs
const mozilla::Encoding* xmldecl_parse(const uint8_t* buf, size_t buf_len);
};

using namespace mozilla;
using namespace mozilla::dom;

/*
 * Note that nsHtml5StreamParser implements cycle collecting AddRef and
 * Release. Therefore, nsHtml5StreamParser must never be refcounted from
 * the parser thread!
 *
 * To work around this limitation, runnables posted by the main thread to the
 * parser thread hold their reference to the stream parser in an
 * nsHtml5StreamParserPtr. Upon creation, nsHtml5StreamParserPtr addrefs the
 * object it holds
 * just like a regular nsRefPtr. This is OK, since the creation of the
 * runnable and the nsHtml5StreamParserPtr happens on the main thread.
 *
 * When the runnable is done on the parser thread, the destructor of
 * nsHtml5StreamParserPtr runs there. It doesn't call Release on the held object
 * directly. Instead, it posts another runnable back to the main thread where
 * that runnable calls Release on the wrapped object.
 *
 * When posting runnables in the other direction, the runnables have to be
 * created on the main thread when nsHtml5StreamParser is instantiated and
 * held for the lifetime of the nsHtml5StreamParser. This works, because the
 * same runnabled can be dispatched multiple times and currently runnables
 * posted from the parser thread to main thread don't need to wrap any
 * runnable-specific data. (In the other direction, the runnables most notably
 * wrap the byte data of the stream.)
 */
NS_IMPL_CYCLE_COLLECTING_ADDREF(nsHtml5StreamParser)
NS_IMPL_CYCLE_COLLECTING_RELEASE(nsHtml5StreamParser)

NS_INTERFACE_TABLE_HEAD(nsHtml5StreamParser)
  NS_INTERFACE_TABLE(nsHtml5StreamParser, nsISupports)
  NS_INTERFACE_TABLE_TO_MAP_SEGUE_CYCLE_COLLECTION(nsHtml5StreamParser)
NS_INTERFACE_MAP_END

NS_IMPL_CYCLE_COLLECTION_CLASS(nsHtml5StreamParser)

NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(nsHtml5StreamParser)
  tmp->DropTimer();
  NS_IMPL_CYCLE_COLLECTION_UNLINK(mRequest)
  NS_IMPL_CYCLE_COLLECTION_UNLINK(mOwner)
  tmp->mExecutorFlusher = nullptr;
  tmp->mLoadFlusher = nullptr;
  tmp->mExecutor = nullptr;
NS_IMPL_CYCLE_COLLECTION_UNLINK_END

NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(nsHtml5StreamParser)
  NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mRequest)
  NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOwner)
  // hack: count the strongly owned edge wrapped in the runnable
  if (tmp->mExecutorFlusher) {
    NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(cb, "mExecutorFlusher->mExecutor");
    cb.NoteXPCOMChild(static_cast<nsIContentSink*>(tmp->mExecutor));
  }
  // hack: count the strongly owned edge wrapped in the runnable
  if (tmp->mLoadFlusher) {
    NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(cb, "mLoadFlusher->mExecutor");
    cb.NoteXPCOMChild(static_cast<nsIContentSink*>(tmp->mExecutor));
  }
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END

class nsHtml5ExecutorFlusher : public Runnable {
 private:
  RefPtr<nsHtml5TreeOpExecutor> mExecutor;

 public:
  explicit nsHtml5ExecutorFlusher(nsHtml5TreeOpExecutor* aExecutor)
      : Runnable("nsHtml5ExecutorFlusher"), mExecutor(aExecutor) {}
  NS_IMETHOD Run() override {
    if (!mExecutor->isInList()) {
      Document* doc = mExecutor->GetDocument();
      if (XRE_IsContentProcess() &&
          nsContentUtils::
              HighPriorityEventPendingForTopLevelDocumentBeforeContentfulPaint(
                  doc)) {
        // Possible early paint pending, reuse the runnable and try to
        // call RunFlushLoop later.
        nsCOMPtr<nsIRunnable> flusher = this;
        if (NS_SUCCEEDED(
                doc->Dispatch(TaskCategory::Network, flusher.forget()))) {
          PROFILER_MARKER_UNTYPED("HighPrio blocking parser flushing(1)", DOM);
          return NS_OK;
        }
      }
      mExecutor->RunFlushLoop();
    }
    return NS_OK;
  }
};

class nsHtml5LoadFlusher : public Runnable {
 private:
  RefPtr<nsHtml5TreeOpExecutor> mExecutor;

 public:
  explicit nsHtml5LoadFlusher(nsHtml5TreeOpExecutor* aExecutor)
      : Runnable("nsHtml5LoadFlusher"), mExecutor(aExecutor) {}
  NS_IMETHOD Run() override {
    mExecutor->FlushSpeculativeLoads();
    return NS_OK;
  }
};

nsHtml5StreamParser::nsHtml5StreamParser(nsHtml5TreeOpExecutor* aExecutor,
                                         nsHtml5Parser* aOwner,
                                         eParserMode aMode)
    : mBomState(eBomState::BOM_SNIFFING_NOT_STARTED),
      mCharsetSource(kCharsetUninitialized),
      mEncodingSwitchSource(kCharsetUninitialized),
      mEncoding(X_USER_DEFINED_ENCODING),  // Obviously bogus value to notice if
                                           // not updated
      mNeedsEncodingSwitchTo(nullptr),
      mSeenEligibleMetaCharset(false),
      mChardetEof(false),
#ifdef DEBUG
      mStartedFeedingDetector(false),
      mStartedFeedingDevTools(false),
#endif
      mReparseForbidden(false),
      mForceAutoDetection(false),
      mChannelHadCharset(false),
      mLookingForMetaCharset(false),
      mStartsWithLtQuestion(false),
      mLookingForXmlDeclarationForXmlViewSource(false),
      mTemplatePushedOrHeadPopped(false),
      mGtBuffer(nullptr),
      mGtPos(0),
      mLastBuffer(nullptr),  // Will be filled when starting
      mExecutor(aExecutor),
      mTreeBuilder(new nsHtml5TreeBuilder(
          (aMode == VIEW_SOURCE_HTML || aMode == VIEW_SOURCE_XML)
              ? nullptr
              : mExecutor->GetStage(),
          mExecutor->GetStage(), aMode == NORMAL)),
      mTokenizer(
          new nsHtml5Tokenizer(mTreeBuilder.get(), aMode == VIEW_SOURCE_XML)),
      mTokenizerMutex("nsHtml5StreamParser mTokenizerMutex"),
      mOwner(aOwner),
      mLastWasCR(false),
      mStreamState(eHtml5StreamState::STREAM_NOT_STARTED),
      mSpeculating(false),
      mAtEOF(false),
      mSpeculationMutex("nsHtml5StreamParser mSpeculationMutex"),
      mSpeculationFailureCount(0),
      mNumBytesBuffered(0),
      mTerminated(false),
      mInterrupted(false),
      mEventTarget(nsHtml5Module::GetStreamParserEventTarget()),
      mExecutorFlusher(new nsHtml5ExecutorFlusher(aExecutor)),
      mLoadFlusher(new nsHtml5LoadFlusher(aExecutor)),
      mInitialEncodingWasFromParentFrame(false),
      mHasHadErrors(false),
      mDetectorHasSeenNonAscii(false),
      mDecodingLocalFileWithoutTokenizing(false),
      mBufferingBytes(false),
      mFlushTimer(NS_NewTimer(mEventTarget)),
      mFlushTimerMutex("nsHtml5StreamParser mFlushTimerMutex"),
      mFlushTimerArmed(false),
      mFlushTimerEverFired(false),
      mMode(aMode) {
  NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
#ifdef DEBUG
  mAtomTable.SetPermittedLookupEventTarget(mEventTarget);
#endif
  mTokenizer->setInterner(&mAtomTable);
  mTokenizer->setEncodingDeclarationHandler(this);

  if (aMode == VIEW_SOURCE_HTML || aMode == VIEW_SOURCE_XML) {
    nsHtml5Highlighter* highlighter =
        new nsHtml5Highlighter(mExecutor->GetStage());
    mTokenizer->EnableViewSource(highlighter);    // takes ownership
    mTreeBuilder->EnableViewSource(highlighter);  // doesn't own
  }

  // There's a zeroing operator new for everything else
}

nsHtml5StreamParser::~nsHtml5StreamParser() {
  NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
  mTokenizer->end();
#ifdef DEBUG
  {
    mozilla::MutexAutoLock flushTimerLock(mFlushTimerMutex);
    MOZ_ASSERT(!mFlushTimer, "Flush timer was not dropped before dtor!");
  }
  mRequest = nullptr;
  mUnicodeDecoder = nullptr;
  mFirstBuffer = nullptr;
  mExecutor = nullptr;
  mTreeBuilder = nullptr;
  mTokenizer = nullptr;
  mOwner = nullptr;
#endif
}

nsresult nsHtml5StreamParser::GetChannel(nsIChannel** aChannel) {
  NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
  return mRequest ? CallQueryInterface(mRequest, aChannel)
                  : NS_ERROR_NOT_AVAILABLE;
}

std::tuple<NotNull<const Encoding*>, nsCharsetSource>
nsHtml5StreamParser::GuessEncoding(bool aInitial) {
  MOZ_ASSERT(
      mCharsetSource != kCharsetFromFinalUserForcedAutoDetection &&
      mCharsetSource !=
          kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII &&
      mCharsetSource !=
          kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic &&
      mCharsetSource !=
          kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8GenericInitialWasASCII &&
      mCharsetSource !=
          kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content &&
      mCharsetSource !=
          kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8ContentInitialWasASCII &&
      mCharsetSource !=
          kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD &&
      mCharsetSource !=
          kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLDInitialWasASCII &&
      mCharsetSource != kCharsetFromFinalAutoDetectionFile);
  auto ifHadBeenForced = mDetector->Guess(EmptyCString(), true);
  auto encoding =
      mForceAutoDetection
          ? ifHadBeenForced
          : mDetector->Guess(mTLD, mDecodingLocalFileWithoutTokenizing);
  nsCharsetSource source =
      aInitial
          ? (mForceAutoDetection
                 ? kCharsetFromInitialUserForcedAutoDetection
                 : (mDecodingLocalFileWithoutTokenizing
                        ? kCharsetFromFinalAutoDetectionFile
                        : kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Generic))
          : (mForceAutoDetection
                 ? kCharsetFromFinalUserForcedAutoDetection
                 : (mDecodingLocalFileWithoutTokenizing
                        ? kCharsetFromFinalAutoDetectionFile
                        : kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic));
  if (source == kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Generic) {
    if (encoding == ISO_2022_JP_ENCODING) {
      if (EncodingDetector::TldMayAffectGuess(mTLD)) {
        source = kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content;
      }
    } else if (!mDetectorHasSeenNonAscii) {
      source = kCharsetFromInitialAutoDetectionASCII;  // deliberately Initial
    } else if (ifHadBeenForced == UTF_8_ENCODING) {
      MOZ_ASSERT(mCharsetSource == kCharsetFromInitialAutoDetectionASCII ||
                 mCharsetSource ==
                     kCharsetFromInitialAutoDetectionWouldHaveBeenUTF8);
      source = kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII;
    } else if (encoding != ifHadBeenForced) {
      if (mCharsetSource == kCharsetFromInitialAutoDetectionASCII) {
        source =
            kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLDInitialWasASCII;
      } else {
        source =
            kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD;
      }
    } else if (EncodingDetector::TldMayAffectGuess(mTLD)) {
      if (mCharsetSource == kCharsetFromInitialAutoDetectionASCII) {
        source =
            kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8ContentInitialWasASCII;
      } else {
        source = kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8Content;
      }
    } else if (mCharsetSource == kCharsetFromInitialAutoDetectionASCII) {
      source =
          kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8GenericInitialWasASCII;
    }
  } else if (source ==
             kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Generic) {
    if (encoding == ISO_2022_JP_ENCODING) {
      if (EncodingDetector::TldMayAffectGuess(mTLD)) {
        source = kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Content;
      }
    } else if (!mDetectorHasSeenNonAscii) {
      source = kCharsetFromInitialAutoDetectionASCII;
    } else if (ifHadBeenForced == UTF_8_ENCODING) {
      source = kCharsetFromInitialAutoDetectionWouldHaveBeenUTF8;
    } else if (encoding != ifHadBeenForced) {
      source =
          kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD;
    } else if (EncodingDetector::TldMayAffectGuess(mTLD)) {
      source = kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8Content;
    }
  }
  return {encoding, source};
}

void nsHtml5StreamParser::FeedDetector(Span<const uint8_t> aBuffer) {
#ifdef DEBUG
  mStartedFeedingDetector = true;
#endif
  MOZ_ASSERT(!mChardetEof);
  mDetectorHasSeenNonAscii = mDetector->Feed(aBuffer, false);
}

void nsHtml5StreamParser::DetectorEof() {
#ifdef DEBUG
  mStartedFeedingDetector = true;
#endif
  if (mChardetEof) {
    return;
  }
  mChardetEof = true;
  mDetectorHasSeenNonAscii = mDetector->Feed(Span<const uint8_t>(), true);
}

void nsHtml5StreamParser::SetViewSourceTitle(nsIURI* aURL) {
  MOZ_ASSERT(NS_IsMainThread());

  BrowsingContext* browsingContext =
      mExecutor->GetDocument()->GetBrowsingContext();
  if (browsingContext && browsingContext->WatchedByDevTools()) {
    mURIToSendToDevtools = aURL;

    nsID uuid;
    nsresult rv = nsID::GenerateUUIDInPlace(uuid);
    if (!NS_FAILED(rv)) {
      char buffer[NSID_LENGTH];
      uuid.ToProvidedString(buffer);
      mUUIDForDevtools = NS_ConvertASCIItoUTF16(buffer);
    }
  }

  if (aURL) {
    nsCOMPtr<nsIURI> temp;
    if (aURL->SchemeIs("view-source")) {
      nsCOMPtr<nsINestedURI> nested = do_QueryInterface(aURL);
      nested->GetInnerURI(getter_AddRefs(temp));
    } else {
      temp = aURL;
    }
    if (temp->SchemeIs("data")) {
      // Avoid showing potentially huge data: URLs. The three last bytes are
      // UTF-8 for an ellipsis.
      mViewSourceTitle.AssignLiteral("data:\xE2\x80\xA6");
    } else {
      nsresult rv = temp->GetSpec(mViewSourceTitle);
      if (NS_FAILED(rv)) {
        mViewSourceTitle.AssignLiteral("\xE2\x80\xA6");
      }
    }
  }
}

nsresult
nsHtml5StreamParser::SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
    Span<const uint8_t> aPrefix, Span<const uint8_t> aFromSegment) {
  NS_ASSERTION(IsParserThread(), "Wrong thread!");
  mUnicodeDecoder = mEncoding->NewDecoderWithBOMRemoval();
  nsresult rv = WriteStreamBytes(aPrefix);
  NS_ENSURE_SUCCESS(rv, rv);
  return WriteStreamBytes(aFromSegment);
}

void nsHtml5StreamParser::SetupDecodingFromBom(
    NotNull<const Encoding*> aEncoding) {
  MOZ_ASSERT(IsParserThread(), "Wrong thread!");
  mEncoding = aEncoding;
  mDecodingLocalFileWithoutTokenizing = false;
  mLookingForMetaCharset = false;
  mBufferingBytes = false;
  mUnicodeDecoder = mEncoding->NewDecoderWithoutBOMHandling();
  mCharsetSource = kCharsetFromByteOrderMark;
  mForceAutoDetection = false;
  mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, false);
  mBomState = BOM_SNIFFING_OVER;
  if (mMode == VIEW_SOURCE_HTML) {
    mTokenizer->StartViewSourceCharacters();
  }
}

void nsHtml5StreamParser::SetupDecodingFromUtf16BogoXml(
    NotNull<const Encoding*> aEncoding) {
  MOZ_ASSERT(IsParserThread(), "Wrong thread!");
  mEncoding = aEncoding;
  mDecodingLocalFileWithoutTokenizing = false;
  mLookingForMetaCharset = false;
  mBufferingBytes = false;
  mUnicodeDecoder = mEncoding->NewDecoderWithoutBOMHandling();
  mCharsetSource = kCharsetFromXmlDeclarationUtf16;
  mForceAutoDetection = false;
  mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, false);
  mBomState = BOM_SNIFFING_OVER;
  if (mMode == VIEW_SOURCE_HTML) {
    mTokenizer->StartViewSourceCharacters();
  }
  auto dst = mLastBuffer->TailAsSpan(READ_BUFFER_SIZE);
  dst[0] = '<';
  dst[1] = '?';
  dst[2] = 'x';
  mLastBuffer->AdvanceEnd(3);
  MOZ_ASSERT(!mStartedFeedingDevTools);
  OnNewContent(dst.To(3));
}

size_t nsHtml5StreamParser::LengthOfLtContainingPrefixInSecondBuffer() {
  MOZ_ASSERT(mBufferedBytes.Length() <= 2);
  if (mBufferedBytes.Length() < 2) {
    return 0;
  }
  Buffer<uint8_t>& second = mBufferedBytes[1];
  const uint8_t* elements = second.Elements();
  const uint8_t* lt = (const uint8_t*)memchr(elements, '>', second.Length());
  if (lt) {
    return (lt - elements) + 1;
  }
  return 0;
}

nsresult nsHtml5StreamParser::SniffStreamBytes(Span<const uint8_t> aFromSegment,
                                               bool aEof) {
  MOZ_ASSERT(IsParserThread(), "Wrong thread!");
  MOZ_ASSERT_IF(aEof, aFromSegment.IsEmpty());

  if (mCharsetSource >=
          kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII &&
      mCharsetSource <= kCharsetFromFinalUserForcedAutoDetection) {
    if (mMode == PLAIN_TEXT || mMode == VIEW_SOURCE_PLAIN) {
      mTreeBuilder->MaybeComplainAboutCharset("EncDetectorReloadPlain", true,
                                              0);
    } else {
      mTreeBuilder->MaybeComplainAboutCharset("EncDetectorReload", true, 0);
    }
  }

  // mEncoding and mCharsetSource potentially have come from channel or higher
  // by now. If we find a BOM, SetupDecodingFromBom() will overwrite them.
  // If we don't find a BOM, the previously set values of mEncoding and
  // mCharsetSource are not modified by the BOM sniffing here.
  static uint8_t utf8[] = {0xEF, 0xBB};
  static uint8_t utf16le[] = {0xFF};
  static uint8_t utf16be[] = {0xFE};
  static uint8_t utf16leXml[] = {'<', 0x00, '?', 0x00, 'x'};
  static uint8_t utf16beXml[] = {0x00, '<', 0x00, '?', 0x00};
  // Buffer for replaying past bytes based on state machine state. If
  // writing this from scratch, probably wouldn't do it this way, but
  // let's keep the changes to a minimum.
  const uint8_t* prefix = utf8;
  size_t prefixLength = 0;
  if (aEof && mBomState == BOM_SNIFFING_NOT_STARTED) {
    // Avoid handling aEof in the BOM_SNIFFING_NOT_STARTED state below.
    mBomState = BOM_SNIFFING_OVER;
  }
  for (size_t i = 0;
       (i < aFromSegment.Length() && mBomState != BOM_SNIFFING_OVER) || aEof;
       i++) {
    switch (mBomState) {
      case BOM_SNIFFING_NOT_STARTED:
        MOZ_ASSERT(i == 0, "Bad BOM sniffing state.");
        MOZ_ASSERT(!aEof, "Should have checked for aEof above!");
        switch (aFromSegment[0]) {
          case 0xEF:
            mBomState = SEEN_UTF_8_FIRST_BYTE;
            break;
          case 0xFF:
            mBomState = SEEN_UTF_16_LE_FIRST_BYTE;
            break;
          case 0xFE:
            mBomState = SEEN_UTF_16_BE_FIRST_BYTE;
            break;
          case 0x00:
            if (mCharsetSource < kCharsetFromXmlDeclarationUtf16 &&
                mCharsetSource != kCharsetFromChannel) {
              mBomState = SEEN_UTF_16_BE_XML_FIRST;
            } else {
              mBomState = BOM_SNIFFING_OVER;
            }
            break;
          case '<':
            if (mCharsetSource < kCharsetFromXmlDeclarationUtf16 &&
                mCharsetSource != kCharsetFromChannel) {
              mBomState = SEEN_UTF_16_LE_XML_FIRST;
            } else {
              mBomState = BOM_SNIFFING_OVER;
            }
            break;
          default:
            mBomState = BOM_SNIFFING_OVER;
            break;
        }
        break;
      case SEEN_UTF_16_LE_FIRST_BYTE:
        if (!aEof && aFromSegment[i] == 0xFE) {
          SetupDecodingFromBom(UTF_16LE_ENCODING);
          return WriteStreamBytes(aFromSegment.From(i + 1));
        }
        prefix = utf16le;
        prefixLength = 1 - i;
        mBomState = BOM_SNIFFING_OVER;
        break;
      case SEEN_UTF_16_BE_FIRST_BYTE:
        if (!aEof && aFromSegment[i] == 0xFF) {
          SetupDecodingFromBom(UTF_16BE_ENCODING);
          return WriteStreamBytes(aFromSegment.From(i + 1));
        }
        prefix = utf16be;
        prefixLength = 1 - i;
        mBomState = BOM_SNIFFING_OVER;
        break;
      case SEEN_UTF_8_FIRST_BYTE:
        if (!aEof && aFromSegment[i] == 0xBB) {
          mBomState = SEEN_UTF_8_SECOND_BYTE;
        } else {
          prefixLength = 1 - i;
          mBomState = BOM_SNIFFING_OVER;
        }
        break;
      case SEEN_UTF_8_SECOND_BYTE:
        if (!aEof && aFromSegment[i] == 0xBF) {
          SetupDecodingFromBom(UTF_8_ENCODING);
          return WriteStreamBytes(aFromSegment.From(i + 1));
        }
        prefixLength = 2 - i;
        mBomState = BOM_SNIFFING_OVER;
        break;
      case SEEN_UTF_16_BE_XML_FIRST:
        if (!aEof && aFromSegment[i] == '<') {
          mBomState = SEEN_UTF_16_BE_XML_SECOND;
        } else {
          prefix = utf16beXml;
          prefixLength = 1 - i;
          mBomState = BOM_SNIFFING_OVER;
        }
        break;
      case SEEN_UTF_16_BE_XML_SECOND:
        if (!aEof && aFromSegment[i] == 0x00) {
          mBomState = SEEN_UTF_16_BE_XML_THIRD;
        } else {
          prefix = utf16beXml;
          prefixLength = 2 - i;
          mBomState = BOM_SNIFFING_OVER;
        }
        break;
      case SEEN_UTF_16_BE_XML_THIRD:
        if (!aEof && aFromSegment[i] == '?') {
          mBomState = SEEN_UTF_16_BE_XML_FOURTH;
        } else {
          prefix = utf16beXml;
          prefixLength = 3 - i;
          mBomState = BOM_SNIFFING_OVER;
        }
        break;
      case SEEN_UTF_16_BE_XML_FOURTH:
        if (!aEof && aFromSegment[i] == 0x00) {
          mBomState = SEEN_UTF_16_BE_XML_FIFTH;
        } else {
          prefix = utf16beXml;
          prefixLength = 4 - i;
          mBomState = BOM_SNIFFING_OVER;
        }
        break;
      case SEEN_UTF_16_BE_XML_FIFTH:
        if (!aEof && aFromSegment[i] == 'x') {
          SetupDecodingFromUtf16BogoXml(UTF_16BE_ENCODING);
          return WriteStreamBytes(aFromSegment.From(i + 1));
        }
        prefix = utf16beXml;
        prefixLength = 5 - i;
        mBomState = BOM_SNIFFING_OVER;
        break;
      case SEEN_UTF_16_LE_XML_FIRST:
        if (!aEof && aFromSegment[i] == 0x00) {
          mBomState = SEEN_UTF_16_LE_XML_SECOND;
        } else {
          if (!aEof && aFromSegment[i] == '?' &&
              !(mMode == PLAIN_TEXT || mMode == VIEW_SOURCE_PLAIN)) {
            mStartsWithLtQuestion = true;
          }
          prefix = utf16leXml;
          prefixLength = 1 - i;
          mBomState = BOM_SNIFFING_OVER;
        }
        break;
      case SEEN_UTF_16_LE_XML_SECOND:
        if (!aEof && aFromSegment[i] == '?') {
          mBomState = SEEN_UTF_16_LE_XML_THIRD;
        } else {
          prefix = utf16leXml;
          prefixLength = 2 - i;
          mBomState = BOM_SNIFFING_OVER;
        }
        break;
      case SEEN_UTF_16_LE_XML_THIRD:
        if (!aEof && aFromSegment[i] == 0x00) {
          mBomState = SEEN_UTF_16_LE_XML_FOURTH;
        } else {
          prefix = utf16leXml;
          prefixLength = 3 - i;
          mBomState = BOM_SNIFFING_OVER;
        }
        break;
      case SEEN_UTF_16_LE_XML_FOURTH:
        if (!aEof && aFromSegment[i] == 'x') {
          mBomState = SEEN_UTF_16_LE_XML_FIFTH;
        } else {
          prefix = utf16leXml;
          prefixLength = 4 - i;
          mBomState = BOM_SNIFFING_OVER;
        }
        break;
      case SEEN_UTF_16_LE_XML_FIFTH:
        if (!aEof && aFromSegment[i] == 0x00) {
          SetupDecodingFromUtf16BogoXml(UTF_16LE_ENCODING);
          return WriteStreamBytes(aFromSegment.From(i + 1));
        }
        prefix = utf16leXml;
        prefixLength = 5 - i;
        mBomState = BOM_SNIFFING_OVER;
        break;
      default:
        mBomState = BOM_SNIFFING_OVER;
        break;
    }
    if (aEof) {
      break;
    }
  }
  // if we get here, there either was no BOM or the BOM sniffing isn't complete
  // yet

  MOZ_ASSERT(mCharsetSource != kCharsetFromByteOrderMark,
             "Should not come here if BOM was found.");
  MOZ_ASSERT(mCharsetSource != kCharsetFromXmlDeclarationUtf16,
             "Should not come here if UTF-16 bogo-XML declaration was found.");
  MOZ_ASSERT(mCharsetSource != kCharsetFromOtherComponent,
             "kCharsetFromOtherComponent is for XSLT.");

  if (mBomState == BOM_SNIFFING_OVER) {
    if (mMode == VIEW_SOURCE_XML && mStartsWithLtQuestion &&
        mCharsetSource < kCharsetFromChannel) {
      // Sniff for XML declaration only.
      MOZ_ASSERT(!mLookingForXmlDeclarationForXmlViewSource);
      MOZ_ASSERT(!aEof);
      MOZ_ASSERT(!mLookingForMetaCharset);
      MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing);
      // Maybe we've already buffered a '>'.
      MOZ_ASSERT(!mBufferedBytes.IsEmpty(),
                 "How did at least <? not get buffered?");
      Buffer<uint8_t>& first = mBufferedBytes[0];
      const Encoding* encoding =
          xmldecl_parse(first.Elements(), first.Length());
      if (encoding) {
        mEncoding = WrapNotNull(encoding);
        mCharsetSource = kCharsetFromXmlDeclaration;
      } else if (memchr(first.Elements(), '>', first.Length())) {
        // There was a '>', but an encoding still wasn't found.
        ;  // fall through to commit to the UTF-8 default.
      } else if (size_t lengthOfPrefix =
                     LengthOfLtContainingPrefixInSecondBuffer()) {
        // This can only happen if the first buffer was a lone '<', because
        // we come here upon seeing the second byte '?' if the first two bytes
        // were "<?". That is, the only way how we aren't dealing with the first
        // buffer is if the first buffer only contained a single '<' and we are
        // dealing with the second buffer that starts with '?'.
        MOZ_ASSERT(first.Length() == 1);
        MOZ_ASSERT(mBufferedBytes[1][0] == '?');
        // Our scanner for XML declaration-like syntax wants to see a contiguous
        // buffer, so let's linearize the data. (Ideally, the XML declaration
        // scanner would be incremental, but this is the rare path anyway.)
        Vector<uint8_t> contiguous;
        if (!contiguous.append(first.Elements(), first.Length())) {
          MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
          return NS_ERROR_OUT_OF_MEMORY;
        }
        if (!contiguous.append(mBufferedBytes[1].Elements(), lengthOfPrefix)) {
          MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
          return NS_ERROR_OUT_OF_MEMORY;
        }
        encoding = xmldecl_parse(contiguous.begin(), contiguous.length());
        if (encoding) {
          mEncoding = WrapNotNull(encoding);
          mCharsetSource = kCharsetFromXmlDeclaration;
        }
        // else no XML decl, commit to the UTF-8 default.
      } else {
        MOZ_ASSERT(mBufferingBytes);
        mLookingForXmlDeclarationForXmlViewSource = true;
        return NS_OK;
      }
    } else if (mMode != VIEW_SOURCE_XML &&
               (mForceAutoDetection || mCharsetSource < kCharsetFromChannel)) {
      // In order to use the buffering logic for meta with mForceAutoDetection,
      // we set mLookingForMetaCharset but still actually potentially ignore the
      // meta.
      mFirstBufferOfMetaScan = mFirstBuffer;
      MOZ_ASSERT(mLookingForMetaCharset);

      if (mMode == VIEW_SOURCE_HTML) {
        auto r = mTokenizer->FlushViewSource();
        if (r.isErr()) {
          return r.unwrapErr();
        }
      }
      auto r = mTreeBuilder->Flush();
      if (r.isErr()) {
        return r.unwrapErr();
      }
      // Encoding committer flushes the ops on the main thread.

      mozilla::MutexAutoLock speculationAutoLock(mSpeculationMutex);
      nsHtml5Speculation* speculation = new nsHtml5Speculation(
          mFirstBuffer, mFirstBuffer->getStart(), mTokenizer->getLineNumber(),
          mTreeBuilder->newSnapshot());
      MOZ_ASSERT(!mFlushTimerArmed, "How did we end up arming the timer?");
      if (mMode == VIEW_SOURCE_HTML) {
        mTokenizer->SetViewSourceOpSink(speculation);
        mTokenizer->StartViewSourceCharacters();
      } else {
        MOZ_ASSERT(mMode != VIEW_SOURCE_XML);
        mTreeBuilder->SetOpSink(speculation);
      }
      mSpeculations.AppendElement(speculation);  // adopts the pointer
      mSpeculating = true;
    } else {
      mLookingForMetaCharset = false;
      mBufferingBytes = false;
      mDecodingLocalFileWithoutTokenizing = false;
      if (mMode == VIEW_SOURCE_HTML) {
        mTokenizer->StartViewSourceCharacters();
      }
    }
    mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, false);
    return SetupDecodingAndWriteSniffingBufferAndCurrentSegment(
        Span(prefix, prefixLength), aFromSegment);
  }

  return NS_OK;
}

class AddContentRunnable : public Runnable {
 public:
  AddContentRunnable(const nsAString& aParserID, nsIURI* aURI,
                     Span<const char16_t> aData, bool aComplete)
      : Runnable("AddContent") {
    nsAutoCString spec;
    aURI->GetSpec(spec);
    mData.mUri.Construct(NS_ConvertUTF8toUTF16(spec));
    mData.mParserID.Construct(aParserID);
    mData.mContents.Construct(aData.Elements(), aData.Length());
    mData.mComplete.Construct(aComplete);
  }

  NS_IMETHOD Run() override {
    nsAutoString json;
    if (!mData.ToJSON(json)) {
      return NS_ERROR_FAILURE;
    }

    nsCOMPtr<nsIObserverService> obsService = services::GetObserverService();
    if (obsService) {
      obsService->NotifyObservers(nullptr, "devtools-html-content",
                                  PromiseFlatString(json).get());
    }

    return NS_OK;
  }

  HTMLContent mData;
};

inline void nsHtml5StreamParser::OnNewContent(Span<const char16_t> aData) {
#ifdef DEBUG
  mStartedFeedingDevTools = true;
#endif
  if (mURIToSendToDevtools) {
    if (aData.IsEmpty()) {
      // Optimize out the runnable.
      return;
    }
    NS_DispatchToMainThread(new AddContentRunnable(mUUIDForDevtools,
                                                   mURIToSendToDevtools, aData,
                                                   /* aComplete */ false));
  }
}

inline void nsHtml5StreamParser::OnContentComplete() {
#ifdef DEBUG
  mStartedFeedingDevTools = true;
#endif
  if (mURIToSendToDevtools) {
    NS_DispatchToMainThread(new AddContentRunnable(
        mUUIDForDevtools, mURIToSendToDevtools, Span<const char16_t>(),
        /* aComplete */ true));
    mURIToSendToDevtools = nullptr;
  }
}

nsresult nsHtml5StreamParser::WriteStreamBytes(
    Span<const uint8_t> aFromSegment) {
  NS_ASSERTION(IsParserThread(), "Wrong thread!");
  mTokenizerMutex.AssertCurrentThreadOwns();
  // mLastBuffer should always point to a buffer of the size
  // READ_BUFFER_SIZE.
  if (!mLastBuffer) {
    NS_WARNING("mLastBuffer should not be null!");
    MarkAsBroken(NS_ERROR_NULL_POINTER);
    return NS_ERROR_NULL_POINTER;
  }
  size_t totalRead = 0;
  auto src = aFromSegment;
  for (;;) {
    auto dst = mLastBuffer->TailAsSpan(READ_BUFFER_SIZE);
    auto [result, read, written, hadErrors] =
        mUnicodeDecoder->DecodeToUTF16(src, dst, false);
    if (!(mLookingForMetaCharset || mDecodingLocalFileWithoutTokenizing)) {
      OnNewContent(dst.To(written));
    }
    if (hadErrors && !mHasHadErrors) {
      mHasHadErrors = true;
      if (mEncoding == UTF_8_ENCODING) {
        mTreeBuilder->TryToEnableEncodingMenu();
      }
    }
    src = src.From(read);
    totalRead += read;
    mLastBuffer->AdvanceEnd(written);
    if (result == kOutputFull) {
      RefPtr<nsHtml5OwningUTF16Buffer> newBuf =
          nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE);
      if (!newBuf) {
        MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
        return NS_ERROR_OUT_OF_MEMORY;
      }
      mLastBuffer = (mLastBuffer->next = std::move(newBuf));
    } else {
      MOZ_ASSERT(totalRead == aFromSegment.Length(),
                 "The Unicode decoder consumed the wrong number of bytes.");
      (void)totalRead;
      if (!mLookingForMetaCharset && mDecodingLocalFileWithoutTokenizing &&
          mNumBytesBuffered == LOCAL_FILE_UTF_8_BUFFER_SIZE) {
        MOZ_ASSERT(!mStartedFeedingDetector);
        for (auto&& buffer : mBufferedBytes) {
          FeedDetector(buffer);
        }
        // If the file is exactly LOCAL_FILE_UTF_8_BUFFER_SIZE bytes long
        // we end up not considering the EOF. That's not fatal, since we
        // don't consider the EOF if the file is
        // LOCAL_FILE_UTF_8_BUFFER_SIZE + 1 bytes long.
        auto [encoding, source] = GuessEncoding(true);
        mCharsetSource = source;
        if (encoding != mEncoding) {
          mEncoding = encoding;
          nsresult rv = ReDecodeLocalFile();
          if (NS_FAILED(rv)) {
            return rv;
          }
        } else {
          MOZ_ASSERT(mEncoding == UTF_8_ENCODING);
          nsresult rv = CommitLocalFileToEncoding();
          if (NS_FAILED(rv)) {
            return rv;
          }
        }
      }
      return NS_OK;
    }
  }
}

[[nodiscard]] nsresult nsHtml5StreamParser::ReDecodeLocalFile() {
  MOZ_ASSERT(mDecodingLocalFileWithoutTokenizing && !mLookingForMetaCharset);
  MOZ_ASSERT(mFirstBufferOfMetaScan);
  MOZ_ASSERT(mCharsetSource == kCharsetFromFinalAutoDetectionFile ||
             (mForceAutoDetection &&
              mCharsetSource == kCharsetFromInitialUserForcedAutoDetection));

  DiscardMetaSpeculation();

  MOZ_ASSERT(mEncoding != UTF_8_ENCODING);

  mDecodingLocalFileWithoutTokenizing = false;

  mEncoding->NewDecoderWithBOMRemovalInto(*mUnicodeDecoder);
  mHasHadErrors = false;

  // Throw away previous decoded data
  mLastBuffer = mFirstBuffer;
  mLastBuffer->next = nullptr;
  mLastBuffer->setStart(0);
  mLastBuffer->setEnd(0);

  mBufferingBytes = false;
  mForceAutoDetection = false;  // To stop feeding the detector
  mFirstBufferOfMetaScan = nullptr;

  mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, true);

  // Decode again
  for (auto&& buffer : mBufferedBytes) {
    DoDataAvailable(buffer);
  }

  if (mMode == VIEW_SOURCE_HTML) {
    auto r = mTokenizer->FlushViewSource();
    if (r.isErr()) {
      return r.unwrapErr();
    }
  }
  auto r = mTreeBuilder->Flush();
  if (r.isErr()) {
    return r.unwrapErr();
  }
  return NS_OK;
}

[[nodiscard]] nsresult nsHtml5StreamParser::CommitLocalFileToEncoding() {
  MOZ_ASSERT(mDecodingLocalFileWithoutTokenizing && !mLookingForMetaCharset);
  MOZ_ASSERT(mFirstBufferOfMetaScan);
  mDecodingLocalFileWithoutTokenizing = false;
  MOZ_ASSERT(mCharsetSource == kCharsetFromFinalAutoDetectionFile ||
             (mForceAutoDetection &&
              mCharsetSource == kCharsetFromInitialUserForcedAutoDetection));
  MOZ_ASSERT(mEncoding == UTF_8_ENCODING);

  MOZ_ASSERT(!mStartedFeedingDevTools);
  if (mURIToSendToDevtools) {
    nsHtml5OwningUTF16Buffer* buffer = mFirstBufferOfMetaScan;
    while (buffer) {
      Span<const char16_t> data(buffer->getBuffer() + buffer->getStart(),
                                buffer->getLength());
      OnNewContent(data);
      buffer = buffer->next;
    }
  }

  mFirstBufferOfMetaScan = nullptr;

  mBufferingBytes = false;
  mForceAutoDetection = false;  // To stop feeding the detector
  mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, true);
  if (mMode == VIEW_SOURCE_HTML) {
    auto r = mTokenizer->FlushViewSource();
    if (r.isErr()) {
      return r.unwrapErr();
    }
  }
  auto r = mTreeBuilder->Flush();
  if (r.isErr()) {
    return r.unwrapErr();
  }
  return NS_OK;
}

class MaybeRunCollector : public Runnable {
 public:
  explicit MaybeRunCollector(nsIDocShell* aDocShell)
      : Runnable("MaybeRunCollector"), mDocShell(aDocShell) {}

  NS_IMETHOD Run() override {
    nsJSContext::MaybeRunNextCollectorSlice(mDocShell,
                                            JS::GCReason::HTML_PARSER);
    return NS_OK;
  }

  nsCOMPtr<nsIDocShell> mDocShell;
};

nsresult nsHtml5StreamParser::OnStartRequest(nsIRequest* aRequest) {
  MOZ_RELEASE_ASSERT(STREAM_NOT_STARTED == mStreamState,
                     "Got OnStartRequest when the stream had already started.");
  MOZ_ASSERT(
      !mExecutor->HasStarted(),
      "Got OnStartRequest at the wrong stage in the executor life cycle.");
  MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");

  // To avoid the cost of instantiating the detector when it's not needed,
  // let's instantiate only if we make it out of this method with the
  // intent to use it.
  auto detectorCreator = MakeScopeExit([&] {
    if ((mForceAutoDetection || mCharsetSource < kCharsetFromParentFrame) ||
        !(mMode == LOAD_AS_DATA || mMode == VIEW_SOURCE_XML)) {
      mDetector = mozilla::EncodingDetector::Create();
    }
  });

  mRequest = aRequest;

  mStreamState = STREAM_BEING_READ;

  // For View Source, the parser should run with scripts "enabled" if a normal
  // load would have scripts enabled.
  bool scriptingEnabled =
      mMode == LOAD_AS_DATA ? false : mExecutor->IsScriptEnabled();
  mOwner->StartTokenizer(scriptingEnabled);

  MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing);
  bool isSrcdoc = false;
  nsCOMPtr<nsIChannel> channel;
  nsresult rv = GetChannel(getter_AddRefs(channel));
  if (NS_SUCCEEDED(rv)) {
    isSrcdoc = NS_IsSrcdocChannel(channel);
    if (!isSrcdoc && mCharsetSource <= kCharsetFromFallback) {
      nsCOMPtr<nsIURI> originalURI;
      rv = channel->GetOriginalURI(getter_AddRefs(originalURI));
      if (NS_SUCCEEDED(rv)) {
        if (originalURI->SchemeIs("resource")) {
          mCharsetSource = kCharsetFromBuiltIn;
          mEncoding = UTF_8_ENCODING;
          mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, false);
        } else {
          nsCOMPtr<nsIURI> currentURI;
          rv = channel->GetURI(getter_AddRefs(currentURI));
          if (NS_SUCCEEDED(rv)) {
            nsCOMPtr<nsIURI> innermost = NS_GetInnermostURI(currentURI);
            if (innermost->SchemeIs("file")) {
              MOZ_ASSERT(mEncoding == UTF_8_ENCODING);
              if (!(mMode == LOAD_AS_DATA || mMode == VIEW_SOURCE_XML)) {
                mDecodingLocalFileWithoutTokenizing = true;
              }
            } else {
              nsAutoCString host;
              innermost->GetAsciiHost(host);
              if (!host.IsEmpty()) {
                // First let's see if the host is DNS-absolute and ends with a
                // dot and get rid of that one.
                if (host.Last() == '.') {
                  host.SetLength(host.Length() - 1);
                }
                int32_t index = host.RFindChar('.');
                if (index != kNotFound) {
                  // We tolerate an IPv4 component as generic "TLD", so don't
                  // bother checking.
                  ToLowerCase(
                      Substring(host, index + 1, host.Length() - (index + 1)),
                      mTLD);
                }
              }
            }
          }
        }
      }
    }
  }
  mTreeBuilder->setIsSrcdocDocument(isSrcdoc);
  mTreeBuilder->setScriptingEnabled(scriptingEnabled);
  mTreeBuilder->SetPreventScriptExecution(
      !((mMode == NORMAL) && scriptingEnabled));
  mTokenizer->start();
  mExecutor->Start();
  mExecutor->StartReadingFromStage();

  if (mMode == PLAIN_TEXT) {
    mTreeBuilder->StartPlainText();
    mTokenizer->StartPlainText();
    MOZ_ASSERT(
        mTemplatePushedOrHeadPopped);  // Needed to force 1024-byte sniffing
    // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
    // can find them.
    auto r = mTreeBuilder->Flush();
    if (r.isErr()) {
      return mExecutor->MarkAsBroken(r.unwrapErr());
    }
  } else if (mMode == VIEW_SOURCE_PLAIN) {
    nsAutoString viewSourceTitle;
    CopyUTF8toUTF16(mViewSourceTitle, viewSourceTitle);
    mTreeBuilder->EnsureBufferSpace(viewSourceTitle.Length());
    mTreeBuilder->StartPlainTextViewSource(viewSourceTitle);
    mTokenizer->StartPlainText();
    MOZ_ASSERT(
        mTemplatePushedOrHeadPopped);  // Needed to force 1024-byte sniffing
    // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
    // can find them.
    auto r = mTreeBuilder->Flush();
    if (r.isErr()) {
      return mExecutor->MarkAsBroken(r.unwrapErr());
    }
  } else if (mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML) {
    // Generate and flush the View Source document up to and including the
    // pre element start.
    mTokenizer->StartViewSource(NS_ConvertUTF8toUTF16(mViewSourceTitle));
    if (mMode == VIEW_SOURCE_XML) {
      mTokenizer->StartViewSourceCharacters();
    }
    // Flush the ops to put them where ContinueAfterScriptsOrEncodingCommitment
    // can find them.
    auto r = mTokenizer->FlushViewSource();
    if (r.isErr()) {
      return mExecutor->MarkAsBroken(r.unwrapErr());
    }
  }

  /*
   * If you move the following line, be very careful not to cause
   * WillBuildModel to be called before the document has had its
   * script global object set.
   */
  rv = mExecutor->WillBuildModel();
  NS_ENSURE_SUCCESS(rv, rv);

  RefPtr<nsHtml5OwningUTF16Buffer> newBuf =
      nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE);
  if (!newBuf) {
    // marks this stream parser as terminated,
    // which prevents entry to code paths that
    // would use mFirstBuffer or mLastBuffer.
    return mExecutor->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
  }
  MOZ_ASSERT(!mFirstBuffer, "How come we have the first buffer set?");
  MOZ_ASSERT(!mLastBuffer, "How come we have the last buffer set?");
  mFirstBuffer = mLastBuffer = newBuf;

  rv = NS_OK;

  mNetworkEventTarget =
      mExecutor->GetDocument()->EventTargetFor(TaskCategory::Network);

  nsCOMPtr<nsIHttpChannel> httpChannel(do_QueryInterface(mRequest, &rv));
  if (NS_SUCCEEDED(rv)) {
    // Non-HTTP channels are bogus enough that we let them work with unlabeled
    // runnables for now. Asserting for HTTP channels only.
    MOZ_ASSERT(mNetworkEventTarget || mMode == LOAD_AS_DATA,
               "How come the network event target is still null?");

    nsAutoCString method;
    Unused << httpChannel->GetRequestMethod(method);
    // XXX does Necko have a way to renavigate POST, etc. without hitting
    // the network?
    if (!method.EqualsLiteral("GET")) {
      // This is the old Gecko behavior but the HTML5 spec disagrees.
      // Don't reparse on POST.
      mReparseForbidden = true;
    }
  }

  // Attempt to retarget delivery of data (via OnDataAvailable) to the parser
  // thread, rather than through the main thread.
  nsCOMPtr<nsIThreadRetargetableRequest> threadRetargetableRequest =
      do_QueryInterface(mRequest, &rv);
  if (threadRetargetableRequest) {
    rv = threadRetargetableRequest->RetargetDeliveryTo(mEventTarget);
    if (NS_SUCCEEDED(rv)) {
      // Parser thread should be now ready to get data from necko and parse it
      // and main thread might have a chance to process a collector slice.
      // We need to do this asynchronously so that necko may continue processing
      // the request.
      nsCOMPtr<nsIRunnable> runnable =
          new MaybeRunCollector(mExecutor->GetDocument()->GetDocShell());
      mozilla::SchedulerGroup::Dispatch(
          mozilla::TaskCategory::GarbageCollection, runnable.forget());
    }
  }

  if (NS_FAILED(rv)) {
    NS_WARNING("Failed to retarget HTML data delivery to the parser thread.");
  }

  if (mCharsetSource == kCharsetFromParentFrame) {
    // Remember this for error reporting.
    mInitialEncodingWasFromParentFrame = true;
    MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing);
  }

  if (mForceAutoDetection || mCharsetSource < kCharsetFromChannel) {
    mBufferingBytes = true;
    if (mMode != VIEW_SOURCE_XML) {
      // We need to set mLookingForMetaCharset to true here in case the first
      // buffer to arrive is larger than 1024. We need the code that splits
      // the buffers at 1024 bytes to work even in that case.
      mLookingForMetaCharset = true;
    }
  }

  if (mCharsetSource < kCharsetFromUtf8OnlyMime) {
    // we aren't ready to commit to an encoding yet
    // leave converter uninstantiated for now
    return NS_OK;
  }

  MOZ_ASSERT(!(mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML));

  MOZ_ASSERT(mEncoding == UTF_8_ENCODING,
             "How come UTF-8-only MIME type didn't set encoding to UTF-8?");

  // We are loading JSON/WebVTT/etc. into a browsing context.
  // There's no need to remove the BOM manually here, because
  // the UTF-8 decoder removes it.
  mReparseForbidden = true;
  mForceAutoDetection = false;

  // Instantiate the converter here to avoid BOM sniffing.
  mDecodingLocalFileWithoutTokenizing = false;
  mUnicodeDecoder = mEncoding->NewDecoderWithBOMRemoval();
  return NS_OK;
}

void nsHtml5StreamParser::DoStopRequest() {
  MOZ_ASSERT(IsParserThread(), "Wrong thread!");
  MOZ_RELEASE_ASSERT(STREAM_BEING_READ == mStreamState,
                     "Stream ended without being open.");
  mTokenizerMutex.AssertCurrentThreadOwns();

  auto guard = MakeScopeExit([&] { OnContentComplete(); });

  if (IsTerminated()) {
    return;
  }

  if (MOZ_UNLIKELY(mLookingForXmlDeclarationForXmlViewSource)) {
    mLookingForXmlDeclarationForXmlViewSource = false;
    mBufferingBytes = false;
    mUnicodeDecoder = mEncoding->NewDecoderWithoutBOMHandling();
    mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, false);

    for (auto&& buffer : mBufferedBytes) {
      nsresult rv = WriteStreamBytes(buffer);
      if (NS_FAILED(rv)) {
        MarkAsBroken(rv);
        return;
      }
    }
  } else if (!mUnicodeDecoder) {
    nsresult rv;
    if (NS_FAILED(rv = SniffStreamBytes(Span<const uint8_t>(), true))) {
      MarkAsBroken(rv);
      return;
    }
  }

  MOZ_ASSERT(mUnicodeDecoder,
             "Should have a decoder after finalizing sniffing.");

  // mLastBuffer should always point to a buffer of the size
  // READ_BUFFER_SIZE.
  if (!mLastBuffer) {
    NS_WARNING("mLastBuffer should not be null!");
    MarkAsBroken(NS_ERROR_NULL_POINTER);
    return;
  }

  Span<uint8_t> src;  // empty span
  for (;;) {
    auto dst = mLastBuffer->TailAsSpan(READ_BUFFER_SIZE);
    uint32_t result;
    size_t read;
    size_t written;
    bool hadErrors;
    // Do not use structured binding lest deal with [-Werror=unused-variable]
    std::tie(result, read, written, hadErrors) =
        mUnicodeDecoder->DecodeToUTF16(src, dst, true);
    if (!(mLookingForMetaCharset || mDecodingLocalFileWithoutTokenizing)) {
      OnNewContent(dst.To(written));
    }
    if (hadErrors) {
      mHasHadErrors = true;
    }
    MOZ_ASSERT(read == 0, "How come an empty span was read form?");
    mLastBuffer->AdvanceEnd(written);
    if (result == kOutputFull) {
      RefPtr<nsHtml5OwningUTF16Buffer> newBuf =
          nsHtml5OwningUTF16Buffer::FalliblyCreate(READ_BUFFER_SIZE);
      if (!newBuf) {
        MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
        return;
      }
      mLastBuffer = (mLastBuffer->next = std::move(newBuf));
    } else {
      if (!mLookingForMetaCharset && mDecodingLocalFileWithoutTokenizing) {
        MOZ_ASSERT(mNumBytesBuffered < LOCAL_FILE_UTF_8_BUFFER_SIZE);
        MOZ_ASSERT(!mStartedFeedingDetector);
        for (auto&& buffer : mBufferedBytes) {
          FeedDetector(buffer);
        }
        MOZ_ASSERT(!mChardetEof);
        DetectorEof();
        auto [encoding, source] = GuessEncoding(true);
        mCharsetSource = source;
        if (encoding != mEncoding) {
          mEncoding = encoding;
          nsresult rv = ReDecodeLocalFile();
          if (NS_FAILED(rv)) {
            MarkAsBroken(rv);
            return;
          }
          DoStopRequest();
          return;
        }
        MOZ_ASSERT(mEncoding == UTF_8_ENCODING);
        nsresult rv = CommitLocalFileToEncoding();
        if (NS_FAILED(rv)) {
          MarkAsBroken(rv);
          return;
        }
      }
      break;
    }
  }

  mStreamState = STREAM_ENDED;

  if (IsTerminatedOrInterrupted()) {
    return;
  }

  ParseAvailableData();
}

class nsHtml5RequestStopper : public Runnable {
 private:
  nsHtml5StreamParserPtr mStreamParser;

 public:
  explicit nsHtml5RequestStopper(nsHtml5StreamParser* aStreamParser)
      : Runnable("nsHtml5RequestStopper"), mStreamParser(aStreamParser) {}
  NS_IMETHOD Run() override {
    mozilla::MutexAutoLock autoLock(mStreamParser->mTokenizerMutex);
    mStreamParser->DoStopRequest();
    mStreamParser->PostLoadFlusher();
    return NS_OK;
  }
};

nsresult nsHtml5StreamParser::OnStopRequest(nsIRequest* aRequest,
                                            nsresult status) {
  MOZ_ASSERT(mRequest == aRequest, "Got Stop on wrong stream.");
  MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
  nsCOMPtr<nsIRunnable> stopper = new nsHtml5RequestStopper(this);
  if (NS_FAILED(mEventTarget->Dispatch(stopper, nsIThread::DISPATCH_NORMAL))) {
    NS_WARNING("Dispatching StopRequest event failed.");
  }
  return NS_OK;
}

void nsHtml5StreamParser::DoDataAvailableBuffer(
    mozilla::Buffer<uint8_t>&& aBuffer) {
  if (MOZ_UNLIKELY(!mBufferingBytes)) {
    DoDataAvailable(aBuffer);
    return;
  }
  if (MOZ_UNLIKELY(mLookingForXmlDeclarationForXmlViewSource)) {
    const uint8_t* elements = aBuffer.Elements();
    size_t length = aBuffer.Length();
    const uint8_t* lt = (const uint8_t*)memchr(elements, '>', length);
    if (!lt) {
      mBufferedBytes.AppendElement(std::move(aBuffer));
      return;
    }

    // We found an '>'. Now there either is or isn't an XML decl.
    length = (lt - elements) + 1;
    Vector<uint8_t> contiguous;
    for (auto&& buffer : mBufferedBytes) {
      if (!contiguous.append(buffer.Elements(), buffer.Length())) {
        MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
        return;
      }
    }
    if (!contiguous.append(elements, length)) {
      MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
      return;
    }

    const Encoding* encoding =
        xmldecl_parse(contiguous.begin(), contiguous.length());
    if (encoding) {
      mEncoding = WrapNotNull(encoding);
      mCharsetSource = kCharsetFromXmlDeclaration;
    }

    mLookingForXmlDeclarationForXmlViewSource = false;
    mBufferingBytes = false;
    mUnicodeDecoder = mEncoding->NewDecoderWithoutBOMHandling();
    mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, false);

    for (auto&& buffer : mBufferedBytes) {
      DoDataAvailable(buffer);
    }
    DoDataAvailable(aBuffer);
    mBufferedBytes.Clear();
    return;
  }
  CheckedInt<size_t> bufferedPlusLength(aBuffer.Length());
  bufferedPlusLength += mNumBytesBuffered;
  if (!bufferedPlusLength.isValid()) {
    MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
    return;
  }
  // Ensure that WriteStreamBytes() sees buffers ending
  // exactly at the two special boundaries.
  bool metaBoundaryWithinBuffer =
      mLookingForMetaCharset &&
      mNumBytesBuffered < UNCONDITIONAL_META_SCAN_BOUNDARY &&
      bufferedPlusLength.value() > UNCONDITIONAL_META_SCAN_BOUNDARY;
  bool localFileLimitWithinBuffer =
      mDecodingLocalFileWithoutTokenizing &&
      mNumBytesBuffered < LOCAL_FILE_UTF_8_BUFFER_SIZE &&
      bufferedPlusLength.value() > LOCAL_FILE_UTF_8_BUFFER_SIZE;
  if (!metaBoundaryWithinBuffer && !localFileLimitWithinBuffer) {
    // Truncation OK, because we just checked the range.
    mNumBytesBuffered = bufferedPlusLength.value();
    mBufferedBytes.AppendElement(std::move(aBuffer));
    DoDataAvailable(mBufferedBytes.LastElement());
  } else {
    MOZ_RELEASE_ASSERT(
        !(metaBoundaryWithinBuffer && localFileLimitWithinBuffer),
        "How can Necko give us a buffer this large?");
    size_t boundary = metaBoundaryWithinBuffer
                          ? UNCONDITIONAL_META_SCAN_BOUNDARY
                          : LOCAL_FILE_UTF_8_BUFFER_SIZE;
    // Truncation OK, because the constant is small enough.
    size_t overBoundary = bufferedPlusLength.value() - boundary;
    MOZ_RELEASE_ASSERT(overBoundary < aBuffer.Length());
    size_t untilBoundary = aBuffer.Length() - overBoundary;
    auto span = aBuffer.AsSpan();
    auto head = span.To(untilBoundary);
    auto tail = span.From(untilBoundary);
    MOZ_RELEASE_ASSERT(mNumBytesBuffered + untilBoundary == boundary);
    // The following copies may end up being useless, but optimizing
    // them away would add complexity.
    Maybe<Buffer<uint8_t>> maybeHead = Buffer<uint8_t>::CopyFrom(head);
    if (maybeHead.isNothing()) {
      MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
      return;
    }
    mNumBytesBuffered = boundary;
    mBufferedBytes.AppendElement(std::move(*maybeHead));
    DoDataAvailable(mBufferedBytes.LastElement());
    // Re-decode may have happened here.

    Maybe<Buffer<uint8_t>> maybeTail = Buffer<uint8_t>::CopyFrom(tail);
    if (maybeTail.isNothing()) {
      MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
      return;
    }
    mNumBytesBuffered += tail.Length();
    mBufferedBytes.AppendElement(std::move(*maybeTail));
    DoDataAvailable(mBufferedBytes.LastElement());
  }
  // Do this clean-up here to avoid use-after-free when
  // DoDataAvailable is passed a span pointing into an
  // element of mBufferedBytes.
  if (!mBufferingBytes) {
    mBufferedBytes.Clear();
  }
}

void nsHtml5StreamParser::DoDataAvailable(Span<const uint8_t> aBuffer) {
  MOZ_ASSERT(IsParserThread(), "Wrong thread!");
  MOZ_RELEASE_ASSERT(STREAM_BEING_READ == mStreamState,
                     "DoDataAvailable called when stream not open.");
  mTokenizerMutex.AssertCurrentThreadOwns();

  if (IsTerminated()) {
    return;
  }

  nsresult rv;
  if (HasDecoder()) {
    if ((mForceAutoDetection || mCharsetSource < kCharsetFromParentFrame) &&
        !mBufferingBytes && !mReparseForbidden &&
        !(mMode == LOAD_AS_DATA || mMode == VIEW_SOURCE_XML)) {
      MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing,
                 "How is mBufferingBytes false if "
                 "mDecodingLocalFileWithoutTokenizing is true?");
      FeedDetector(aBuffer);
    }
    rv = WriteStreamBytes(aBuffer);
  } else {
    rv = SniffStreamBytes(aBuffer, false);
  }
  if (NS_FAILED(rv)) {
    MarkAsBroken(rv);
    return;
  }

  if (IsTerminatedOrInterrupted()) {
    return;
  }

  if (!mLookingForMetaCharset && mDecodingLocalFileWithoutTokenizing) {
    return;
  }

  ParseAvailableData();

  if (mBomState != BOM_SNIFFING_OVER || mFlushTimerArmed || mSpeculating) {
    return;
  }

  {
    mozilla::MutexAutoLock flushTimerLock(mFlushTimerMutex);
    mFlushTimer->InitWithNamedFuncCallback(
        nsHtml5StreamParser::TimerCallback, static_cast<void*>(this),
        mFlushTimerEverFired ? StaticPrefs::html5_flushtimer_initialdelay()
                             : StaticPrefs::html5_flushtimer_subsequentdelay(),
        nsITimer::TYPE_ONE_SHOT, "nsHtml5StreamParser::DoDataAvailable");
  }
  mFlushTimerArmed = true;
}

class nsHtml5DataAvailable : public Runnable {
 private:
  nsHtml5StreamParserPtr mStreamParser;
  Buffer<uint8_t> mData;

 public:
  nsHtml5DataAvailable(nsHtml5StreamParser* aStreamParser,
                       Buffer<uint8_t>&& aData)
      : Runnable("nsHtml5DataAvailable"),
        mStreamParser(aStreamParser),
        mData(std::move(aData)) {}
  NS_IMETHOD Run() override {
    mozilla::MutexAutoLock autoLock(mStreamParser->mTokenizerMutex);
    mStreamParser->DoDataAvailableBuffer(std::move(mData));
    mStreamParser->PostLoadFlusher();
    return NS_OK;
  }
};

nsresult nsHtml5StreamParser::OnDataAvailable(nsIRequest* aRequest,
                                              nsIInputStream* aInStream,
                                              uint64_t aSourceOffset,
                                              uint32_t aLength) {
  nsresult rv;

  MOZ_ASSERT(mRequest == aRequest, "Got data on wrong stream.");
  uint32_t totalRead;
  // Main thread to parser thread dispatch requires copying to buffer first.
  if (MOZ_UNLIKELY(NS_IsMainThread())) {
    if (NS_FAILED(rv = mExecutor->IsBroken())) {
      return rv;
    }
    Maybe<Buffer<uint8_t>> maybe = Buffer<uint8_t>::Alloc(aLength);
    if (maybe.isNothing()) {
      return mExecutor->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
    }
    Buffer<uint8_t> data(std::move(*maybe));
    rv = aInStream->Read(reinterpret_cast<char*>(data.Elements()),
                         data.Length(), &totalRead);
    NS_ENSURE_SUCCESS(rv, rv);
    MOZ_ASSERT(totalRead == aLength);

    nsCOMPtr<nsIRunnable> dataAvailable =
        new nsHtml5DataAvailable(this, std::move(data));
    if (NS_FAILED(mEventTarget->Dispatch(dataAvailable,
                                         nsIThread::DISPATCH_NORMAL))) {
      NS_WARNING("Dispatching DataAvailable event failed.");
    }
    return rv;
  }

  MOZ_ASSERT(IsParserThread(), "Wrong thread!");
  mozilla::MutexAutoLock autoLock(mTokenizerMutex);

  if (NS_FAILED(rv = mTreeBuilder->IsBroken())) {
    return rv;
  }

  // Since we're getting OnDataAvailable directly on the parser thread,
  // there is no nsHtml5DataAvailable that would call PostLoadFlusher.
  // Hence, we need to call PostLoadFlusher() before this method returns.
  // Braces for RAII clarity relative to the mutex despite not being
  // strictly necessary.
  {
    auto speculationFlusher = MakeScopeExit([&] { PostLoadFlusher(); });

    if (mBufferingBytes) {
      Maybe<Buffer<uint8_t>> maybe = Buffer<uint8_t>::Alloc(aLength);
      if (maybe.isNothing()) {
        MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
        return NS_ERROR_OUT_OF_MEMORY;
      }
      Buffer<uint8_t> data(std::move(*maybe));
      rv = aInStream->Read(reinterpret_cast<char*>(data.Elements()),
                           data.Length(), &totalRead);
      NS_ENSURE_SUCCESS(rv, rv);
      MOZ_ASSERT(totalRead == aLength);
      DoDataAvailableBuffer(std::move(data));
      return rv;
    }
    // Read directly from response buffer.
    rv = aInStream->ReadSegments(CopySegmentsToParser, this, aLength,
                                 &totalRead);
    NS_ENSURE_SUCCESS(rv, rv);
    MOZ_ASSERT(totalRead == aLength);
    return rv;
  }
}

// Called under lock by function ptr
/* static */
nsresult nsHtml5StreamParser::CopySegmentsToParser(
    nsIInputStream* aInStream, void* aClosure, const char* aFromSegment,
    uint32_t aToOffset, uint32_t aCount,
    uint32_t* aWriteCount) MOZ_NO_THREAD_SAFETY_ANALYSIS {
  nsHtml5StreamParser* parser = static_cast<nsHtml5StreamParser*>(aClosure);

  parser->DoDataAvailable(AsBytes(Span(aFromSegment, aCount)));
  // Assume DoDataAvailable consumed all available bytes.
  *aWriteCount = aCount;
  return NS_OK;
}

const Encoding* nsHtml5StreamParser::PreferredForInternalEncodingDecl(
    const nsAString& aEncoding) {
  const Encoding* newEncoding = Encoding::ForLabel(aEncoding);
  if (!newEncoding) {
    // the encoding name is bogus
    mTreeBuilder->MaybeComplainAboutCharset("EncMetaUnsupported", true,
                                            mTokenizer->getLineNumber());
    return nullptr;
  }

  if (newEncoding == UTF_16BE_ENCODING || newEncoding == UTF_16LE_ENCODING) {
    mTreeBuilder->MaybeComplainAboutCharset("EncMetaUtf16", true,
                                            mTokenizer->getLineNumber());
    newEncoding = UTF_8_ENCODING;
  }

  if (newEncoding == X_USER_DEFINED_ENCODING) {
    // WebKit/Blink hack for Indian and Armenian legacy sites
    mTreeBuilder->MaybeComplainAboutCharset("EncMetaUserDefined", true,
                                            mTokenizer->getLineNumber());
    newEncoding = WINDOWS_1252_ENCODING;
  }

  if (newEncoding == REPLACEMENT_ENCODING) {
    // No line number, because the replacement encoding doesn't allow
    // showing the lines.
    mTreeBuilder->MaybeComplainAboutCharset("EncMetaReplacement", true, 0);
  }

  return newEncoding;
}

bool nsHtml5StreamParser::internalEncodingDeclaration(nsHtml5String aEncoding) {
  MOZ_ASSERT(IsParserThread(), "Wrong thread!");
  if ((mCharsetSource >= kCharsetFromMetaTag &&
       mCharsetSource != kCharsetFromFinalAutoDetectionFile) ||
      mSeenEligibleMetaCharset) {
    return false;
  }

  nsString newEncoding;  // Not Auto, because using it to hold nsStringBuffer*
  aEncoding.ToString(newEncoding);
  auto encoding = PreferredForInternalEncodingDecl(newEncoding);
  if (!encoding) {
    return false;
  }

  mSeenEligibleMetaCharset = true;

  if (!mLookingForMetaCharset) {
    if (mInitialEncodingWasFromParentFrame) {
      mTreeBuilder->MaybeComplainAboutCharset("EncMetaTooLateFrame", true,
                                              mTokenizer->getLineNumber());
    } else {
      mTreeBuilder->MaybeComplainAboutCharset("EncMetaTooLate", true,
                                              mTokenizer->getLineNumber());
    }
    return false;
  }
  if (mTemplatePushedOrHeadPopped) {
    mTreeBuilder->MaybeComplainAboutCharset("EncMetaAfterHeadInKilobyte", false,
                                            mTokenizer->getLineNumber());
  }

  if (mForceAutoDetection &&
      (encoding->IsAsciiCompatible() || encoding == ISO_2022_JP_ENCODING)) {
    return false;
  }

  mNeedsEncodingSwitchTo = encoding;
  mEncodingSwitchSource = kCharsetFromMetaTag;
  return true;
}

bool nsHtml5StreamParser::TemplatePushedOrHeadPopped() {
  MOZ_ASSERT(
      IsParserThread() || mMode == PLAIN_TEXT || mMode == VIEW_SOURCE_PLAIN,
      "Wrong thread!");
  mTemplatePushedOrHeadPopped = true;
  return mNumBytesBuffered >= UNCONDITIONAL_META_SCAN_BOUNDARY;
}

void nsHtml5StreamParser::RememberGt(int32_t aPos) {
  if (mLookingForMetaCharset) {
    mGtBuffer = mFirstBuffer;
    mGtPos = aPos;
  }
}

void nsHtml5StreamParser::PostLoadFlusher() {
  MOZ_ASSERT(IsParserThread(), "Wrong thread!");
  mTokenizerMutex.AssertCurrentThreadOwns();

  mTreeBuilder->FlushLoads();
  // Dispatch this runnable unconditionally, because the loads
  // that need flushing may have been flushed earlier even if the
  // flush right above here did nothing. (Is this still true?)
  nsCOMPtr<nsIRunnable> runnable(mLoadFlusher);
  if (NS_FAILED(
          DispatchToMain(CreateRenderBlockingRunnable(runnable.forget())))) {
    NS_WARNING("failed to dispatch load flush event");
  }

  if ((mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML) &&
      mTokenizer->ShouldFlushViewSource()) {
    auto r = mTreeBuilder->Flush();  // delete useless ops
    MOZ_ASSERT(r.isOk(), "Should have null sink with View Source");
    r = mTokenizer->FlushViewSource();
    if (r.isErr()) {
      MarkAsBroken(r.unwrapErr());
      return;
    }
    if (r.unwrap()) {
      nsCOMPtr<nsIRunnable> runnable(mExecutorFlusher);
      if (NS_FAILED(DispatchToMain(runnable.forget()))) {
        NS_WARNING("failed to dispatch executor flush event");
      }
    }
  }
}

void nsHtml5StreamParser::FlushTreeOpsAndDisarmTimer() {
  MOZ_ASSERT(IsParserThread(), "Wrong thread!");
  if (mFlushTimerArmed) {
    // avoid calling Cancel if the flush timer isn't armed to avoid acquiring
    // a mutex
    {
      mozilla::MutexAutoLock flushTimerLock(mFlushTimerMutex);
      mFlushTimer->Cancel();
    }
    mFlushTimerArmed = false;
  }
  if (mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML) {
    auto r = mTokenizer->FlushViewSource();
    if (r.isErr()) {
      MarkAsBroken(r.unwrapErr());
    }
  }
  auto r = mTreeBuilder->Flush();
  if (r.isErr()) {
    MarkAsBroken(r.unwrapErr());
  }
  nsCOMPtr<nsIRunnable> runnable(mExecutorFlusher);
  if (NS_FAILED(DispatchToMain(runnable.forget()))) {
    NS_WARNING("failed to dispatch executor flush event");
  }
}

void nsHtml5StreamParser::SwitchDecoderIfAsciiSoFar(
    NotNull<const Encoding*> aEncoding) {
  if (mEncoding == aEncoding) {
    MOZ_ASSERT(!mStartedFeedingDevTools);
    // Report all already-decoded buffers to the dev tools if needed.
    if (mURIToSendToDevtools) {
      nsHtml5OwningUTF16Buffer* buffer = mFirstBufferOfMetaScan;
      while (buffer) {
        auto s = Span(buffer->getBuffer(), buffer->getEnd());
        OnNewContent(s);
        buffer = buffer->next;
      }
    }
    return;
  }
  if (!mEncoding->IsAsciiCompatible() || !aEncoding->IsAsciiCompatible()) {
    return;
  }
  size_t numAscii = 0;
  MOZ_ASSERT(mFirstBufferOfMetaScan,
             "Why did we come here without starting meta scan?");
  nsHtml5OwningUTF16Buffer* buffer = mFirstBufferOfMetaScan;
  while (buffer != mFirstBuffer) {
    MOZ_ASSERT(buffer, "mFirstBuffer should have acted as sentinel!");
    MOZ_ASSERT(buffer->getStart() == buffer->getEnd(),
               "Why wasn't an early buffer fully consumed?");
    auto s = Span(buffer->getBuffer(), buffer->getStart());
    if (!IsAscii(s)) {
      return;
    }
    numAscii += s.Length();
    buffer = buffer->next;
  }
  auto s = Span(mFirstBuffer->getBuffer(), mFirstBuffer->getStart());
  if (!IsAscii(s)) {
    return;
  }
  numAscii += s.Length();

  MOZ_ASSERT(!mStartedFeedingDevTools);
  // Report the ASCII prefix to dev tools if needed
  if (mURIToSendToDevtools) {
    buffer = mFirstBufferOfMetaScan;
    while (buffer != mFirstBuffer) {
      MOZ_ASSERT(buffer, "mFirstBuffer should have acted as sentinel!");
      MOZ_ASSERT(buffer->getStart() == buffer->getEnd(),
                 "Why wasn't an early buffer fully consumed?");
      auto s = Span(buffer->getBuffer(), buffer->getStart());
      OnNewContent(s);
      buffer = buffer->next;
    }
    auto s = Span(mFirstBuffer->getBuffer(), mFirstBuffer->getStart());
    OnNewContent(s);
  }

  // Success! Now let's get rid of the already-decoded but not tokenized data:
  mFirstBuffer->setEnd(mFirstBuffer->getStart());
  mLastBuffer = mFirstBuffer;
  mFirstBuffer->next = nullptr;

  // Note: We could have scanned further for ASCII, which could avoid some
  // buffer deallocation and reallocation. However, chances are that if we got
  // until meta without non-ASCII before, there's going to be a title with
  // non-ASCII soon after anyway, so let's avoid the complexity of finding out.

  MOZ_ASSERT(mUnicodeDecoder, "How come we scanned meta without a decoder?");
  mEncoding = aEncoding;
  mEncoding->NewDecoderWithoutBOMHandlingInto(*mUnicodeDecoder);
  mHasHadErrors = false;

  MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing,
             "Must have set mDecodingLocalFileWithoutTokenizing to false to "
             "report data to dev tools below");
  MOZ_ASSERT(!mLookingForMetaCharset,
             "Must have set mLookingForMetaCharset to false to report data to "
             "dev tools below");

  // Now skip over as many bytes and redecode the tail of the
  // buffered bytes.
  size_t skipped = 0;
  for (auto&& buffer : mBufferedBytes) {
    size_t nextSkipped = skipped + buffer.Length();
    if (nextSkipped <= numAscii) {
      skipped = nextSkipped;
      continue;
    }
    if (skipped >= numAscii) {
      WriteStreamBytes(buffer);
      skipped = nextSkipped;
      continue;
    }
    size_t tailLength = nextSkipped - numAscii;
    WriteStreamBytes(Span<uint8_t>(buffer).From(buffer.Length() - tailLength));
    skipped = nextSkipped;
  }
}

size_t nsHtml5StreamParser::CountGts() {
  if (!mGtBuffer) {
    return 0;
  }
  size_t gts = 0;
  nsHtml5OwningUTF16Buffer* buffer = mFirstBufferOfMetaScan;
  for (;;) {
    MOZ_ASSERT(buffer, "How did we walk past mGtBuffer?");
    char16_t* buf = buffer->getBuffer();
    if (buffer == mGtBuffer) {
      for (int32_t i = 0; i <= mGtPos; ++i) {
        if (buf[i] == u'>') {
          ++gts;
        }
      }
      break;
    }
    for (int32_t i = 0; i < buffer->getEnd(); ++i) {
      if (buf[i] == u'>') {
        ++gts;
      }
    }
    buffer = buffer->next;
  }
  return gts;
}

void nsHtml5StreamParser::DiscardMetaSpeculation() {
  mozilla::MutexAutoLock speculationAutoLock(mSpeculationMutex);
  // Rewind the stream
  MOZ_ASSERT(!mAtEOF, "How did we end up setting this?");
  mTokenizer->resetToDataState();
  mTokenizer->setLineNumber(1);
  mLastWasCR = false;

  if (mMode == PLAIN_TEXT || mMode == VIEW_SOURCE_PLAIN) {
    // resetToDataState() above logically rewinds to the state before
    // the plain text start, so we need to start plain text again to
    // put the tokenizer into the plain text state.
    mTokenizer->StartPlainText();
  }

  mFirstBuffer = mLastBuffer;
  mFirstBuffer->setStart(0);
  mFirstBuffer->setEnd(0);
  mFirstBuffer->next = nullptr;

  mTreeBuilder->flushCharacters();  // empty the pending buffer
  mTreeBuilder->ClearOps();         // now get rid of the failed ops

  if (mMode == VIEW_SOURCE_HTML) {
    mTokenizer->RewindViewSource();
  }

  {
    // We know that this resets the tree builder back to the start state.
    // This must happen _after_ the flushCharacters() call above!
    const auto& speculation = mSpeculations.ElementAt(0);
    mTreeBuilder->loadState(speculation->GetSnapshot());
  }

  // Experimentation suggests that we don't need to do anything special
  // for ignoring the leading LF in View Source here.

  mSpeculations.Clear();  // potentially a huge number of destructors
                          // run here synchronously...

  // Now set up a new speculation for the main thread to find.
  // Note that we stay in the speculating state, because the main thread
  // knows how to come out of that state and this thread does not.

  nsHtml5Speculation* speculation = new nsHtml5Speculation(
      mFirstBuffer, mFirstBuffer->getStart(), mTokenizer->getLineNumber(),
      mTreeBuilder->newSnapshot());
  MOZ_ASSERT(!mFlushTimerArmed, "How did we end up arming the timer?");
  if (mMode == VIEW_SOURCE_HTML) {
    mTokenizer->SetViewSourceOpSink(speculation);
    mTokenizer->StartViewSourceCharacters();
  } else {
    MOZ_ASSERT(mMode != VIEW_SOURCE_XML);
    mTreeBuilder->SetOpSink(speculation);
  }
  mSpeculations.AppendElement(speculation);  // adopts the pointer
  MOZ_ASSERT(mSpeculating, "How did we end speculating?");
}

/*
 * The general idea is to match WebKit and Blink exactly for meta
 * scan except:
 *
 * 1. WebKit and Blink look for meta as if scripting was disabled
 *    for `noscript` purposes. This implementation matches the
 *    `noscript` treatment of the observable DOM building (in order
 *    to be able to use the same tree builder run).
 * 2. WebKit and Blink look for meta as if the foreign content
 *    feedback from the tree builder to the tokenizer didn't exist.
 *    This implementation considers the foreign content rules in
 *    order to be able to use the same tree builder run for meta
 *    and the observable DOM building. Note that since <svg> and
 *    <math> imply the end of head, this only matters for meta after
 *    head but starting within the 1024-byte zone.
 *
 * Template is treated specially, because that WebKit/Blink behavior
 * is easy to emulate unlike the above two exceptions. In general,
 * the meta scan token handler in WebKit and Blink behaves as if there
 * was a scripting-disabled tree builder predating the introduction
 * of foreign content and template.
 *
 * Meta is honored if it _starts_ within the first 1024 kilobytes or,
 * if by the 1024-byte boundary head hasn't ended and a template
 * element hasn't started, a meta occurs before the first of the head
 * ending or a template element starting.
 *
 * If a meta isn't honored according to the above definition, and
 * we aren't dealing with plain text, the buffered bytes, which by
 * now have to contain `>` character unless we encountered EOF, are
 * scanned for syntax resembling an XML declaration.
 *
 * If neither a meta nor syntax resembling an XML declaration has
 * been honored and we aren't inheriting the encoding from a
 * same-origin parent or parsing for XHR, chardetng is used.
 * chardetng runs first for the part of the document that was searched
 * for meta and then at EOF. The part searched for meta is defined as
 * follows in order to avoid network buffer boundary-dependent
 * behavior:
 *
 * 1. At least the first 1024 bytes. (This is what happens for plain
 *    text.)
 * 2. If the 1024-byte boundary is within a tag, comment, doctype,
 *    or CDATA section, at least up to the end of that token or CDATA
 *    section. (Exception: If the 1024-byte boundary is in an RCDATA
 *    end tag that hasn't yet been decided to be an end tag, the
 *    token is not considered.)
 * 3. If at the 1024-byte boundary, head hasn't ended and there hasn't
 *    been a template tag, up to the end of the first template tag
 *    or token ending the head, whichever comes first.
 * 4. Except if head is ended by a text token, only to the end of the
 *    most recent tag, comment, or doctype token. (Because text is
 *    coalesced, so it would be harder to correlate the text to the
 *    bytes.)
 *
 * An encoding-related reload is still possible if chardetng's guess
 * at EOF differs from its initial guess.
 */
bool nsHtml5StreamParser::ProcessLookingForMetaCharset(bool aEof) {
  MOZ_ASSERT(mBomState == BOM_SNIFFING_OVER);
  MOZ_ASSERT(mMode != VIEW_SOURCE_XML);
  bool rewound = false;
  MOZ_ASSERT(mForceAutoDetection ||
                 mCharsetSource < kCharsetFromInitialAutoDetectionASCII ||
                 mCharsetSource == kCharsetFromParentFrame,
             "Why are we looking for meta charset if we've seen it?");
  // NOTE! We may come here multiple times with
  // mNumBytesBuffered == UNCONDITIONAL_META_SCAN_BOUNDARY
  // if the tokenizer suspends multiple times after decoding has reached
  // mNumBytesBuffered == UNCONDITIONAL_META_SCAN_BOUNDARY. That's why
  // we need to also check whether the we are at the end of the last
  // decoded buffer.
  // Note that DoDataAvailableBuffer() ensures that the code here has
  // the opportunity to run at the exact UNCONDITIONAL_META_SCAN_BOUNDARY
  // even if there isn't a network buffer boundary there.
  bool atKilobyte = false;
  if ((mNumBytesBuffered == UNCONDITIONAL_META_SCAN_BOUNDARY &&
       mFirstBuffer == mLastBuffer && !mFirstBuffer->hasMore())) {
    atKilobyte = true;
    mTokenizer->AtKilobyteBoundary();
  }
  if (!mNeedsEncodingSwitchTo &&
      (aEof || (mTemplatePushedOrHeadPopped &&
                !mTokenizer->IsInTokenStartedAtKilobyteBoundary() &&
                (atKilobyte ||
                 mNumBytesBuffered > UNCONDITIONAL_META_SCAN_BOUNDARY)))) {
    // meta charset was not found
    mLookingForMetaCharset = false;
    if (mStartsWithLtQuestion && mCharsetSource < kCharsetFromXmlDeclaration) {
      // Look for bogo XML declaration.
      // Search the first buffer in the hope that '>' is within it.
      MOZ_ASSERT(!mBufferedBytes.IsEmpty(),
                 "How did at least <? not get buffered?");
      Buffer<uint8_t>& first = mBufferedBytes[0];
      const Encoding* encoding =
          xmldecl_parse(first.Elements(), first.Length());
      if (!encoding) {
        // Our bogo XML declaration scanner wants to see a contiguous buffer, so
        // let's linearize the data. (Ideally, the XML declaration scanner would
        // be incremental, but this is the rare path anyway.)
        Vector<uint8_t> contiguous;
        if (!contiguous.append(first.Elements(), first.Length())) {
          MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
          return false;
        }
        for (size_t i = 1; i < mBufferedBytes.Length(); ++i) {
          Buffer<uint8_t>& buffer = mBufferedBytes[i];
          const uint8_t* elements = buffer.Elements();
          size_t length = buffer.Length();
          const uint8_t* lt = (const uint8_t*)memchr(elements, '>', length);
          bool stop = false;
          if (lt) {
            length = (lt - elements) + 1;
            stop = true;
          }
          if (!contiguous.append(elements, length)) {
            MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
            return false;
          }
          if (stop) {
            // Avoid linearizing all buffered bytes unnecessarily.
            break;
          }
        }
        encoding = xmldecl_parse(contiguous.begin(), contiguous.length());
      }
      if (encoding) {
        if (!(mForceAutoDetection && (encoding->IsAsciiCompatible() ||
                                      encoding == ISO_2022_JP_ENCODING))) {
          mForceAutoDetection = false;
          mNeedsEncodingSwitchTo = encoding;
          mEncodingSwitchSource = kCharsetFromXmlDeclaration;
        }
      }
    }
    // Check again in case we found an encoding in the bogo XML declaration.
    if (!mNeedsEncodingSwitchTo &&
        (mForceAutoDetection ||
         mCharsetSource < kCharsetFromInitialAutoDetectionASCII) &&
        !(mMode == LOAD_AS_DATA || mMode == VIEW_SOURCE_XML) &&
        !(mDecodingLocalFileWithoutTokenizing && !aEof &&
          mNumBytesBuffered <= LOCAL_FILE_UTF_8_BUFFER_SIZE)) {
      MOZ_ASSERT(!mStartedFeedingDetector);
      if (mNumBytesBuffered == UNCONDITIONAL_META_SCAN_BOUNDARY || aEof) {
        // We know that all the buffered bytes have been tokenized, so feed
        // them all to chardetng.
        for (auto&& buffer : mBufferedBytes) {
          FeedDetector(buffer);
        }
        if (aEof) {
          MOZ_ASSERT(!mChardetEof);
          DetectorEof();
        }
        auto [encoding, source] = GuessEncoding(true);
        mNeedsEncodingSwitchTo = encoding;
        mEncodingSwitchSource = source;
      } else if (mNumBytesBuffered > UNCONDITIONAL_META_SCAN_BOUNDARY) {
        size_t gtsLeftToFind = CountGts();
        size_t bytesSeen = 0;
        // We sync the bytes to the UTF-16 code units seen to avoid depending
        // on network buffer boundaries. We do the syncing by counting '>'
        // bytes / code units. However, we always scan at least 1024 bytes.
        // The 1024-byte boundary is guaranteed to be between buffers.
        // The guarantee is implemented in DoDataAvailableBuffer().
        for (auto&& buffer : mBufferedBytes) {
          if (!mNeedsEncodingSwitchTo) {
            if (gtsLeftToFind) {
              auto span = buffer.AsSpan();
              bool feed = true;
              for (size_t i = 0; i < span.Length(); ++i) {
                if (span[i] == uint8_t('>')) {
                  --gtsLeftToFind;
                  if (!gtsLeftToFind) {
                    if (bytesSeen < UNCONDITIONAL_META_SCAN_BOUNDARY) {
                      break;
                    }
                    ++i;  // Skip the gt
                    FeedDetector(span.To(i));
                    auto [encoding, source] = GuessEncoding(true);
                    mNeedsEncodingSwitchTo = encoding;
                    mEncodingSwitchSource = source;
                    FeedDetector(span.From(i));
                    bytesSeen += buffer.Length();
                    // No need to update bytesSeen anymore, but let's do it for
                    // debugging.
                    // We should do `continue outer;` but C++ can't.
                    feed = false;
                    break;
                  }
                }
              }
              if (feed) {
                FeedDetector(buffer);
                bytesSeen += buffer.Length();
              }
              continue;
            }
            if (bytesSeen == UNCONDITIONAL_META_SCAN_BOUNDARY) {
              auto [encoding, source] = GuessEncoding(true);
              mNeedsEncodingSwitchTo = encoding;
              mEncodingSwitchSource = source;
            }
          }
          FeedDetector(buffer);
          bytesSeen += buffer.Length();
        }
      }
      MOZ_ASSERT(mNeedsEncodingSwitchTo,
                 "How come we didn't call GuessEncoding()?");
    }
  }
  if (mNeedsEncodingSwitchTo) {
    mDecodingLocalFileWithoutTokenizing = false;
    mLookingForMetaCharset = false;

    auto needsEncodingSwitchTo = WrapNotNull(mNeedsEncodingSwitchTo);
    mNeedsEncodingSwitchTo = nullptr;

    SwitchDecoderIfAsciiSoFar(needsEncodingSwitchTo);
    // The above line may have changed mEncoding so that mEncoding equals
    // needsEncodingSwitchTo.

    mCharsetSource = mEncodingSwitchSource;

    if (mMode == VIEW_SOURCE_HTML) {
      auto r = mTokenizer->FlushViewSource();
      if (r.isErr()) {
        MarkAsBroken(r.unwrapErr());
        return false;
      }
    }
    auto r = mTreeBuilder->Flush();
    if (r.isErr()) {
      MarkAsBroken(r.unwrapErr());
      return false;
    }

    if (mEncoding != needsEncodingSwitchTo) {
      // Speculation failed
      rewound = true;

      if (mEncoding == ISO_2022_JP_ENCODING ||
          needsEncodingSwitchTo == ISO_2022_JP_ENCODING) {
        // Chances are no Web author will fix anything due to this message, so
        // this is here to help understanding issues when debugging sites made
        // by someone else.
        mTreeBuilder->MaybeComplainAboutCharset("EncSpeculationFail2022", false,
                                                mTokenizer->getLineNumber());
      } else {
        if (mCharsetSource == kCharsetFromMetaTag) {
          mTreeBuilder->MaybeComplainAboutCharset(
              "EncSpeculationFailMeta", false, mTokenizer->getLineNumber());
        } else if (mCharsetSource == kCharsetFromXmlDeclaration) {
          // This intentionally refers to the line number of how far ahead
          // the document was parsed even though the bogo XML decl is always
          // on line 1.
          mTreeBuilder->MaybeComplainAboutCharset(
              "EncSpeculationFailXml", false, mTokenizer->getLineNumber());
        }
      }

      DiscardMetaSpeculation();
      // Redecode the stream.
      mEncoding = needsEncodingSwitchTo;
      mUnicodeDecoder = mEncoding->NewDecoderWithBOMRemoval();
      mHasHadErrors = false;

      MOZ_ASSERT(!mDecodingLocalFileWithoutTokenizing,
                 "Must have set mDecodingLocalFileWithoutTokenizing to false "
                 "to report data to dev tools below");
      MOZ_ASSERT(!mLookingForMetaCharset,
                 "Must have set mLookingForMetaCharset to false to report data "
                 "to dev tools below");
      for (auto&& buffer : mBufferedBytes) {
        nsresult rv = WriteStreamBytes(buffer);
        if (NS_FAILED(rv)) {
          MarkAsBroken(rv);
          return false;
        }
      }
    }
  } else if (!mLookingForMetaCharset && !mDecodingLocalFileWithoutTokenizing) {
    MOZ_ASSERT(!mStartedFeedingDevTools);
    // Report all already-decoded buffers to the dev tools if needed.
    if (mURIToSendToDevtools) {
      nsHtml5OwningUTF16Buffer* buffer = mFirstBufferOfMetaScan;
      while (buffer) {
        auto s = Span(buffer->getBuffer(), buffer->getEnd());
        OnNewContent(s);
        buffer = buffer->next;
      }
    }
  }
  if (!mLookingForMetaCharset) {
    mGtBuffer = nullptr;
    mGtPos = 0;

    if (!mDecodingLocalFileWithoutTokenizing) {
      mFirstBufferOfMetaScan = nullptr;
      mBufferingBytes = false;
      mBufferedBytes.Clear();
      mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource, true);
      if (mMode == VIEW_SOURCE_HTML) {
        auto r = mTokenizer->FlushViewSource();
        if (r.isErr()) {
          MarkAsBroken(r.unwrapErr());
          return false;
        }
      }
      auto r = mTreeBuilder->Flush();
      if (r.isErr()) {
        MarkAsBroken(r.unwrapErr());
        return false;
      }
    }
  }
  return rewound;
}

void nsHtml5StreamParser::ParseAvailableData() {
  MOZ_ASSERT(IsParserThread(), "Wrong thread!");
  mTokenizerMutex.AssertCurrentThreadOwns();
  MOZ_ASSERT(!(mDecodingLocalFileWithoutTokenizing && !mLookingForMetaCharset));

  if (IsTerminatedOrInterrupted()) {
    return;
  }

  if (mSpeculating && !IsSpeculationEnabled()) {
    return;
  }

  bool requestedReload = false;
  for (;;) {
    if (!mFirstBuffer->hasMore()) {
      if (mFirstBuffer == mLastBuffer) {
        switch (mStreamState) {
          case STREAM_BEING_READ:
            // never release the last buffer.
            if (!mSpeculating) {
              // reuse buffer space if not speculating
              mFirstBuffer->setStart(0);
              mFirstBuffer->setEnd(0);
            }
            return;  // no more data for now but expecting more
          case STREAM_ENDED:
            if (mAtEOF) {
              return;
            }
            if (mLookingForMetaCharset) {
              // When called with aEof=true, ProcessLookingForMetaCharset()
              // is guaranteed to set mLookingForMetaCharset to false so
              // that we can't come here twice.
              if (ProcessLookingForMetaCharset(true)) {
                if (IsTerminatedOrInterrupted()) {
                  return;
                }
                continue;
              }
            } else if ((mForceAutoDetection ||
                        mCharsetSource < kCharsetFromParentFrame) &&
                       !(mMode == LOAD_AS_DATA || mMode == VIEW_SOURCE_XML) &&
                       !mReparseForbidden) {
              // An earlier DetectorEof() call is possible in which case
              // the one here is a no-op.
              DetectorEof();
              auto [encoding, source] = GuessEncoding(false);
              if (encoding != mEncoding) {
                // Request a reload from the docshell.
                MOZ_ASSERT(
                    (source >=
                         kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII &&
                     source <=
                         kCharsetFromFinalAutoDetectionWouldNotHaveBeenUTF8DependedOnTLDInitialWasASCII) ||
                    source == kCharsetFromFinalUserForcedAutoDetection);
                mTreeBuilder->NeedsCharsetSwitchTo(encoding, source, 0);
                requestedReload = true;
              } else if (mCharsetSource ==
                             kCharsetFromInitialAutoDetectionASCII &&
                         mDetectorHasSeenNonAscii) {
                mCharsetSource = source;
                mTreeBuilder->UpdateCharsetSource(mCharsetSource);
              }
            }

            mAtEOF = true;
            if (!mForceAutoDetection && !requestedReload) {
              if (mCharsetSource == kCharsetFromParentFrame) {
                mTreeBuilder->MaybeComplainAboutCharset("EncNoDeclarationFrame",
                                                        false, 0);
              } else if (mCharsetSource == kCharsetFromXmlDeclaration) {
                // We know the bogo XML decl is always on the first line.
                mTreeBuilder->MaybeComplainAboutCharset("EncXmlDecl", false, 1);
              } else if (
                  mCharsetSource >=
                      kCharsetFromInitialAutoDetectionWouldHaveBeenUTF8 &&
                  mCharsetSource <=
                      kCharsetFromInitialAutoDetectionWouldNotHaveBeenUTF8DependedOnTLD) {
                if (mMode == PLAIN_TEXT || mMode == VIEW_SOURCE_PLAIN) {
                  mTreeBuilder->MaybeComplainAboutCharset("EncNoDeclPlain",
                                                          true, 0);
                } else {
                  mTreeBuilder->MaybeComplainAboutCharset("EncNoDecl", true, 0);
                }
              }

              if (mHasHadErrors && mEncoding != REPLACEMENT_ENCODING) {
                if (mEncoding == UTF_8_ENCODING) {
                  mTreeBuilder->TryToEnableEncodingMenu();
                }
                if (mCharsetSource == kCharsetFromParentFrame) {
                  if (mMode == PLAIN_TEXT || mMode == VIEW_SOURCE_PLAIN) {
                    mTreeBuilder->MaybeComplainAboutCharset(
                        "EncErrorFramePlain", true, 0);
                  } else {
                    mTreeBuilder->MaybeComplainAboutCharset("EncErrorFrame",
                                                            true, 0);
                  }
                } else if (
                    mCharsetSource >= kCharsetFromXmlDeclaration &&
                    !(mCharsetSource >=
                          kCharsetFromFinalAutoDetectionWouldHaveBeenUTF8InitialWasASCII &&
                      mCharsetSource <=
                          kCharsetFromFinalUserForcedAutoDetection)) {
                  mTreeBuilder->MaybeComplainAboutCharset("EncError", true, 0);
                }
              }
            }
            if (NS_SUCCEEDED(mTreeBuilder->IsBroken())) {
              mTokenizer->eof();
              nsresult rv;
              if (NS_FAILED((rv = mTreeBuilder->IsBroken()))) {
                MarkAsBroken(rv);
              } else {
                mTreeBuilder->StreamEnded();
                if (mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML) {
                  if (!mTokenizer->EndViewSource()) {
                    MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
                  }
                }
              }
            }
            FlushTreeOpsAndDisarmTimer();
            return;  // no more data and not expecting more
          default:
            MOZ_ASSERT_UNREACHABLE("It should be impossible to reach this.");
            return;
        }
      }
      mFirstBuffer = mFirstBuffer->next;
      continue;
    }

    // now we have a non-empty buffer
    mFirstBuffer->adjust(mLastWasCR);
    mLastWasCR = false;
    if (mFirstBuffer->hasMore()) {
      if (!mTokenizer->EnsureBufferSpace(mFirstBuffer->getLength())) {
        MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
        return;
      }
      mLastWasCR = mTokenizer->tokenizeBuffer(mFirstBuffer);
      nsresult rv;
      if (NS_FAILED((rv = mTreeBuilder->IsBroken()))) {
        MarkAsBroken(rv);
        return;
      }
      if (mTreeBuilder->HasScript()) {
        // HasScript() cannot return true if the tree builder is preventing
        // script execution.
        MOZ_ASSERT(mMode == NORMAL);
        mozilla::MutexAutoLock speculationAutoLock(mSpeculationMutex);
        nsHtml5Speculation* speculation = new nsHtml5Speculation(
            mFirstBuffer, mFirstBuffer->getStart(), mTokenizer->getLineNumber(),
            mTreeBuilder->newSnapshot());
        mTreeBuilder->AddSnapshotToScript(speculation->GetSnapshot(),
                                          speculation->GetStartLineNumber());
        if (mLookingForMetaCharset) {
          if (mMode == VIEW_SOURCE_HTML) {
            auto r = mTokenizer->FlushViewSource();
            if (r.isErr()) {
              MarkAsBroken(r.unwrapErr());
              return;
            }
          }
          auto r = mTreeBuilder->Flush();
          if (r.isErr()) {
            MarkAsBroken(r.unwrapErr());
            return;
          }
        } else {
          FlushTreeOpsAndDisarmTimer();
        }
        mTreeBuilder->SetOpSink(speculation);
        mSpeculations.AppendElement(speculation);  // adopts the pointer
        mSpeculating = true;
      }
      if (IsTerminatedOrInterrupted()) {
        return;
      }
    }
    if (mLookingForMetaCharset) {
      Unused << ProcessLookingForMetaCharset(false);
    }
  }
}

class nsHtml5StreamParserContinuation : public Runnable {
 private:
  nsHtml5StreamParserPtr mStreamParser;

 public:
  explicit nsHtml5StreamParserContinuation(nsHtml5StreamParser* aStreamParser)
      : Runnable("nsHtml5StreamParserContinuation"),
        mStreamParser(aStreamParser) {}
  NS_IMETHOD Run() override {
    mozilla::MutexAutoLock autoLock(mStreamParser->mTokenizerMutex);
    mStreamParser->Uninterrupt();
    mStreamParser->ParseAvailableData();
    return NS_OK;
  }
};

void nsHtml5StreamParser::ContinueAfterScriptsOrEncodingCommitment(
    nsHtml5Tokenizer* aTokenizer, nsHtml5TreeBuilder* aTreeBuilder,
    bool aLastWasCR) {
  // nullptr for aTokenizer means encoding commitment as opposed to the "after
  // scripts" case.

  MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
  MOZ_ASSERT(mMode != VIEW_SOURCE_XML,
             "ContinueAfterScriptsOrEncodingCommitment called in XML view "
             "source mode!");
  MOZ_ASSERT(!(aTokenizer && mMode == VIEW_SOURCE_HTML),
             "ContinueAfterScriptsOrEncodingCommitment called with non-null "
             "tokenizer in HTML view "
             "source mode.");
  if (NS_FAILED(mExecutor->IsBroken())) {
    return;
  }
  MOZ_ASSERT(!(aTokenizer && mMode != NORMAL),
             "We should only be executing scripts in the normal mode.");
  if (!aTokenizer && (mMode == PLAIN_TEXT || mMode == VIEW_SOURCE_PLAIN ||
                      mMode == VIEW_SOURCE_HTML)) {
    // Take the ops that were generated from OnStartRequest for the synthetic
    // head section of the document for plain text and HTML View Source.
    // XML View Source never needs this kind of encoding commitment.
    // We need to take the ops here so that they end up in the queue before
    // the ops that we take from a speculation later in this method.
    if (!mExecutor->TakeOpsFromStage()) {
      mExecutor->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
      return;
    }
  } else {
#ifdef DEBUG
    mExecutor->AssertStageEmpty();
#endif
  }
  bool speculationFailed = false;
  {
    mozilla::MutexAutoLock speculationAutoLock(mSpeculationMutex);
    if (mSpeculations.IsEmpty()) {
      MOZ_ASSERT_UNREACHABLE(
          "ContinueAfterScriptsOrEncodingCommitment called without "
          "speculations.");
      return;
    }

    const auto& speculation = mSpeculations.ElementAt(0);
    if (aTokenizer &&
        (aLastWasCR || !aTokenizer->isInDataState() ||
         !aTreeBuilder->snapshotMatches(speculation->GetSnapshot()))) {
      speculationFailed = true;
      // We've got a failed speculation :-(
      MaybeDisableFutureSpeculation();
      Interrupt();  // Make the parser thread release the tokenizer mutex sooner
      // Note that the interrupted state continues across possible intervening
      // Necko events until the nsHtml5StreamParserContinuation posted at the
      // end of this method runs. Therefore, this thread is guaranteed to
      // acquire mTokenizerMutex soon even if an intervening Necko event grabbed
      // it between now and the acquisition below.

      // now fall out of the speculationAutoLock into the tokenizerAutoLock
      // block
    } else {
      // We've got a successful speculation!
      if (mSpeculations.Length() > 1) {
        // the first speculation isn't the current speculation, so there's
        // no need to bother the parser thread.
        if (!speculation->FlushToSink(mExecutor)) {
          mExecutor->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
          return;
        }
        MOZ_ASSERT(!mExecutor->IsScriptExecuting(),
                   "ParseUntilBlocked() was supposed to ensure we don't come "
                   "here when scripts are executing.");
        MOZ_ASSERT(!aTokenizer || mExecutor->IsInFlushLoop(),
                   "How are we here if "
                   "RunFlushLoop() didn't call ParseUntilBlocked() or we're "
                   "not committing to an encoding?");
        mSpeculations.RemoveElementAt(0);
        return;
      }
      // else
      Interrupt();  // Make the parser thread release the tokenizer mutex sooner
      // Note that the interrupted state continues across possible intervening
      // Necko events until the nsHtml5StreamParserContinuation posted at the
      // end of this method runs. Therefore, this thread is guaranteed to
      // acquire mTokenizerMutex soon even if an intervening Necko event grabbed
      // it between now and the acquisition below.

      // now fall through
      // the first speculation is the current speculation. Need to
      // release the the speculation mutex and acquire the tokenizer
      // mutex. (Just acquiring the other mutex here would deadlock)
    }
  }
  {
    mozilla::MutexAutoLock tokenizerAutoLock(mTokenizerMutex);
#ifdef DEBUG
    {
      mAtomTable.SetPermittedLookupEventTarget(
          GetMainThreadSerialEventTarget());
    }
#endif
    // In principle, the speculation mutex should be acquired here,
    // but there's no point, because the parser thread only acquires it
    // when it has also acquired the tokenizer mutex and we are already
    // holding the tokenizer mutex.
    if (speculationFailed) {
      MOZ_ASSERT(mMode == NORMAL);
      // Rewind the stream
      mAtEOF = false;
      const auto& speculation = mSpeculations.ElementAt(0);
      mFirstBuffer = speculation->GetBuffer();
      mFirstBuffer->setStart(speculation->GetStart());
      mTokenizer->setLineNumber(speculation->GetStartLineNumber());

      nsContentUtils::ReportToConsole(
          nsIScriptError::warningFlag, "DOM Events"_ns,
          mExecutor->GetDocument(), nsContentUtils::eDOM_PROPERTIES,
          "SpeculationFailed2", nsTArray<nsString>(), nullptr, u""_ns,
          speculation->GetStartLineNumber());

      nsHtml5OwningUTF16Buffer* buffer = mFirstBuffer->next;
      while (buffer) {
        buffer->setStart(0);
        buffer = buffer->next;
      }

      mSpeculations.Clear();  // potentially a huge number of destructors
                              // run here synchronously on the main thread...

      mTreeBuilder->flushCharacters();  // empty the pending buffer
      mTreeBuilder->ClearOps();         // now get rid of the failed ops

      mTreeBuilder->SetOpSink(mExecutor->GetStage());
      mExecutor->StartReadingFromStage();
      mSpeculating = false;

      // Copy state over
      mLastWasCR = aLastWasCR;
      mTokenizer->loadState(aTokenizer);
      mTreeBuilder->loadState(aTreeBuilder);
    } else {
      // We've got a successful speculation and at least a moment ago it was
      // the current speculation
      if (!mSpeculations.ElementAt(0)->FlushToSink(mExecutor)) {
        mExecutor->MarkAsBroken(NS_ERROR_OUT_OF_MEMORY);
        return;
      }
      MOZ_ASSERT(!mExecutor->IsScriptExecuting(),
                 "ParseUntilBlocked() was supposed to ensure we don't come "
                 "here when scripts are executing.");
      MOZ_ASSERT(!aTokenizer || mExecutor->IsInFlushLoop(),
                 "How are we here if "
                 "RunFlushLoop() didn't call ParseUntilBlocked() or we're not "
                 "committing to an encoding?");
      mSpeculations.RemoveElementAt(0);
      if (mSpeculations.IsEmpty()) {
        if (mMode == VIEW_SOURCE_HTML) {
          // If we looked for meta charset in the HTML View Source case.
          mTokenizer->SetViewSourceOpSink(mExecutor->GetStage());
        } else {
          // yes, it was still the only speculation. Now stop speculating
          // However, before telling the executor to read from stage, flush
          // any pending ops straight to the executor, because otherwise
          // they remain unflushed until we get more data from the network.
          mTreeBuilder->SetOpSink(mExecutor);
          auto r = mTreeBuilder->Flush(true);
          if (r.isErr()) {
            mExecutor->MarkAsBroken(r.unwrapErr());
            return;
          }
          mTreeBuilder->SetOpSink(mExecutor->GetStage());
        }
        mExecutor->StartReadingFromStage();
        mSpeculating = false;
      }
    }
    nsCOMPtr<nsIRunnable> event = new nsHtml5StreamParserContinuation(this);
    if (NS_FAILED(mEventTarget->Dispatch(event, nsIThread::DISPATCH_NORMAL))) {
      NS_WARNING("Failed to dispatch nsHtml5StreamParserContinuation");
    }
// A stream event might run before this event runs, but that's harmless.
#ifdef DEBUG
    mAtomTable.SetPermittedLookupEventTarget(mEventTarget);
#endif
  }
}

void nsHtml5StreamParser::ContinueAfterFailedCharsetSwitch() {
  MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
  nsCOMPtr<nsIRunnable> event = new nsHtml5StreamParserContinuation(this);
  if (NS_FAILED(mEventTarget->Dispatch(event, nsIThread::DISPATCH_NORMAL))) {
    NS_WARNING("Failed to dispatch nsHtml5StreamParserContinuation");
  }
}

class nsHtml5TimerKungFu : public Runnable {
 private:
  nsHtml5StreamParserPtr mStreamParser;

 public:
  explicit nsHtml5TimerKungFu(nsHtml5StreamParser* aStreamParser)
      : Runnable("nsHtml5TimerKungFu"), mStreamParser(aStreamParser) {}
  NS_IMETHOD Run() override {
    mozilla::MutexAutoLock flushTimerLock(mStreamParser->mFlushTimerMutex);
    if (mStreamParser->mFlushTimer) {
      mStreamParser->mFlushTimer->Cancel();
      mStreamParser->mFlushTimer = nullptr;
    }
    return NS_OK;
  }
};

void nsHtml5StreamParser::DropTimer() {
  MOZ_ASSERT(NS_IsMainThread(), "Wrong thread!");
  /*
   * Simply nulling out the timer wouldn't work, because if the timer is
   * armed, it needs to be canceled first. Simply canceling it first wouldn't
   * work, because nsTimerImpl::Cancel is not safe for calling from outside
   * the thread where nsTimerImpl::Fire would run. It's not safe to
   * dispatch a runnable to cancel the timer from the destructor of this
   * class, because the timer has a weak (void*) pointer back to this instance
   * of the stream parser and having the timer fire before the runnable
   * cancels it would make the timer access a deleted object.
   *
   * This DropTimer method addresses these issues. This method must be called
   * on the main thread before the destructor of this class is reached.
   * The nsHtml5TimerKungFu object has an nsHtml5StreamParserPtr that addrefs
   * this
   * stream parser object to keep it alive until the runnable is done.
   * The runnable cancels the timer on the parser thread, drops the timer
   * and lets nsHtml5StreamParserPtr send a runnable back to the main thread to
   * release the stream parser.
   */
  mozilla::MutexAutoLock flushTimerLock(mFlushTimerMutex);
  if (mFlushTimer) {
    nsCOMPtr<nsIRunnable> event = new nsHtml5TimerKungFu(this);
    if (NS_FAILED(mEventTarget->Dispatch(event, nsIThread::DISPATCH_NORMAL))) {
      NS_WARNING("Failed to dispatch TimerKungFu event");
    }
  }
}

// Using a static, because the method name Notify is taken by the chardet
// callback.
void nsHtml5StreamParser::TimerCallback(nsITimer* aTimer, void* aClosure) {
  (static_cast<nsHtml5StreamParser*>(aClosure))->TimerFlush();
}

void nsHtml5StreamParser::TimerFlush() {
  MOZ_ASSERT(IsParserThread(), "Wrong thread!");
  mozilla::MutexAutoLock autoLock(mTokenizerMutex);

  MOZ_ASSERT(!mSpeculating, "Flush timer fired while speculating.");

  // The timer fired if we got here. No need to cancel it. Mark it as
  // not armed, though.
  mFlushTimerArmed = false;

  mFlushTimerEverFired = true;

  if (IsTerminatedOrInterrupted()) {
    return;
  }

  if (mMode == VIEW_SOURCE_HTML || mMode == VIEW_SOURCE_XML) {
    auto r = mTreeBuilder->Flush();  // delete useless ops
    if (r.isErr()) {
      MarkAsBroken(r.unwrapErr());
      return;
    }
    r = mTokenizer->FlushViewSource();
    if (r.isErr()) {
      MarkAsBroken(r.unwrapErr());
      return;
    }
    if (r.unwrap()) {
      nsCOMPtr<nsIRunnable> runnable(mExecutorFlusher);
      if (NS_FAILED(DispatchToMain(runnable.forget()))) {
        NS_WARNING("failed to dispatch executor flush event");
      }
    }
  } else {
    // we aren't speculating and we don't know when new data is
    // going to arrive. Send data to the main thread.
    auto r = mTreeBuilder->Flush(true);
    if (r.isErr()) {
      MarkAsBroken(r.unwrapErr());
      return;
    }
    if (r.unwrap()) {
      nsCOMPtr<nsIRunnable> runnable(mExecutorFlusher);
      if (NS_FAILED(DispatchToMain(runnable.forget()))) {
        NS_WARNING("failed to dispatch executor flush event");
      }
    }
  }
}

void nsHtml5StreamParser::MarkAsBroken(nsresult aRv) {
  MOZ_ASSERT(IsParserThread(), "Wrong thread!");
  mTokenizerMutex.AssertCurrentThreadOwns();

  Terminate();
  mTreeBuilder->MarkAsBroken(aRv);
  auto r = mTreeBuilder->Flush(false);
  if (r.isOk()) {
    MOZ_ASSERT(r.unwrap(), "Should have had the markAsBroken op!");
  } else {
    MOZ_CRASH("OOM prevents propagation of OOM state");
  }
  nsCOMPtr<nsIRunnable> runnable(mExecutorFlusher);
  if (NS_FAILED(DispatchToMain(runnable.forget()))) {
    NS_WARNING("failed to dispatch executor flush event");
  }
}

nsresult nsHtml5StreamParser::DispatchToMain(
    already_AddRefed<nsIRunnable>&& aRunnable) {
  if (mNetworkEventTarget) {
    return mNetworkEventTarget->Dispatch(std::move(aRunnable));
  }
  return SchedulerGroup::UnlabeledDispatch(TaskCategory::Network,
                                           std::move(aRunnable));
}