From 0e41b5d52fdc6af6442816b5f465c9db9f84e126 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 19 Sep 2024 06:51:18 +0200 Subject: Adding upstream version 6.6.1+dfsg1. Signed-off-by: Daniel Baumann --- wp-includes/html-api/class-wp-html-decoder.php | 461 +++++++ .../html-api/class-wp-html-open-elements.php | 81 +- wp-includes/html-api/class-wp-html-processor.php | 617 ++++++++- wp-includes/html-api/class-wp-html-stack-event.php | 82 ++ .../html-api/class-wp-html-tag-processor.php | 40 +- .../html-api/html5-named-character-references.php | 1313 ++++++++++++++++++++ 6 files changed, 2538 insertions(+), 56 deletions(-) create mode 100644 wp-includes/html-api/class-wp-html-decoder.php create mode 100644 wp-includes/html-api/class-wp-html-stack-event.php create mode 100644 wp-includes/html-api/html5-named-character-references.php (limited to 'wp-includes/html-api') diff --git a/wp-includes/html-api/class-wp-html-decoder.php b/wp-includes/html-api/class-wp-html-decoder.php new file mode 100644 index 0000000..7897600 --- /dev/null +++ b/wp-includes/html-api/class-wp-html-decoder.php @@ -0,0 +1,461 @@ += $end ) { + break; + } + + $character_reference = self::read_character_reference( $context, $text, $next_character_reference_at, $token_length ); + if ( isset( $character_reference ) ) { + $at = $next_character_reference_at; + $decoded .= substr( $text, $was_at, $at - $was_at ); + $decoded .= $character_reference; + $at += $token_length; + $was_at = $at; + continue; + } + + ++$at; + } + + if ( 0 === $was_at ) { + return $text; + } + + if ( $was_at < $end ) { + $decoded .= substr( $text, $was_at, $end - $was_at ); + } + + return $decoded; + } + + /** + * Attempt to read a character reference at the given location in a given string, + * depending on the context in which it's found. + * + * If a character reference is found, this function will return the translated value + * that the reference maps to. It will then set `$match_byte_length` the + * number of bytes of input it read while consuming the character reference. This + * gives calling code the opportunity to advance its cursor when traversing a string + * and decoding. + * + * Example: + * + * null === WP_HTML_Decoder::read_character_reference( 'attribute', 'Ships…', 0 ); + * '…' === WP_HTML_Decoder::read_character_reference( 'attribute', 'Ships…', 5, $token_length ); + * 8 === $token_length; // `…` + * + * null === WP_HTML_Decoder::read_character_reference( 'attribute', '¬in', 0 ); + * '∉' === WP_HTML_Decoder::read_character_reference( 'attribute', '∉', 0, $token_length ); + * 7 === $token_length; // `∉` + * + * '¬' === WP_HTML_Decoder::read_character_reference( 'data', '¬in', 0, $token_length ); + * 4 === $token_length; // `¬` + * '∉' === WP_HTML_Decoder::read_character_reference( 'data', '∉', 0, $token_length ); + * 7 === $token_length; // `∉` + * + * @since 6.6.0 + * + * @param string $context `attribute` for decoding attribute values, `data` otherwise. + * @param string $text Text document containing span of text to decode. + * @param int $at Optional. Byte offset into text where span begins, defaults to the beginning (0). + * @param int &$match_byte_length Optional. Set to byte-length of character reference if provided and if a match + * is found, otherwise not set. Default null. + * @return string|false Decoded character reference in UTF-8 if found, otherwise `false`. + */ + public static function read_character_reference( $context, $text, $at = 0, &$match_byte_length = null ) { + /** + * Mappings for HTML5 named character references. + * + * @var WP_Token_Map $html5_named_character_references + */ + global $html5_named_character_references; + + $length = strlen( $text ); + if ( $at + 1 >= $length ) { + return null; + } + + if ( '&' !== $text[ $at ] ) { + return null; + } + + /* + * Numeric character references. + * + * When truncated, these will encode the code point found by parsing the + * digits that are available. For example, when `🅰` is truncated + * to `DZ` it will encode `DZ`. It does not: + * - know how to parse the original `🅰`. + * - fail to parse and return plaintext `DZ`. + * - fail to parse and return the replacement character `�` + */ + if ( '#' === $text[ $at + 1 ] ) { + if ( $at + 2 >= $length ) { + return null; + } + + /** Tracks inner parsing within the numeric character reference. */ + $digits_at = $at + 2; + + if ( 'x' === $text[ $digits_at ] || 'X' === $text[ $digits_at ] ) { + $numeric_base = 16; + $numeric_digits = '0123456789abcdefABCDEF'; + $max_digits = 6; // 􏿿 + ++$digits_at; + } else { + $numeric_base = 10; + $numeric_digits = '0123456789'; + $max_digits = 7; // 􏿿 + } + + // Cannot encode invalid Unicode code points. Max is to U+10FFFF. + $zero_count = strspn( $text, '0', $digits_at ); + $digit_count = strspn( $text, $numeric_digits, $digits_at + $zero_count ); + $after_digits = $digits_at + $zero_count + $digit_count; + $has_semicolon = $after_digits < $length && ';' === $text[ $after_digits ]; + $end_of_span = $has_semicolon ? $after_digits + 1 : $after_digits; + + // `&#` or `&#x` without digits returns into plaintext. + if ( 0 === $digit_count && 0 === $zero_count ) { + return null; + } + + // Whereas `&#` and only zeros is invalid. + if ( 0 === $digit_count ) { + $match_byte_length = $end_of_span - $at; + return '�'; + } + + // If there are too many digits then it's not worth parsing. It's invalid. + if ( $digit_count > $max_digits ) { + $match_byte_length = $end_of_span - $at; + return '�'; + } + + $digits = substr( $text, $digits_at + $zero_count, $digit_count ); + $code_point = intval( $digits, $numeric_base ); + + /* + * Noncharacters, 0x0D, and non-ASCII-whitespace control characters. + * + * > A noncharacter is a code point that is in the range U+FDD0 to U+FDEF, + * > inclusive, or U+FFFE, U+FFFF, U+1FFFE, U+1FFFF, U+2FFFE, U+2FFFF, + * > U+3FFFE, U+3FFFF, U+4FFFE, U+4FFFF, U+5FFFE, U+5FFFF, U+6FFFE, + * > U+6FFFF, U+7FFFE, U+7FFFF, U+8FFFE, U+8FFFF, U+9FFFE, U+9FFFF, + * > U+AFFFE, U+AFFFF, U+BFFFE, U+BFFFF, U+CFFFE, U+CFFFF, U+DFFFE, + * > U+DFFFF, U+EFFFE, U+EFFFF, U+FFFFE, U+FFFFF, U+10FFFE, or U+10FFFF. + * + * A C0 control is a code point that is in the range of U+00 to U+1F, + * but ASCII whitespace includes U+09, U+0A, U+0C, and U+0D. + * + * These characters are invalid but still decode as any valid character. + * This comment is here to note and explain why there's no check to + * remove these characters or replace them. + * + * @see https://infra.spec.whatwg.org/#noncharacter + */ + + /* + * Code points in the C1 controls area need to be remapped as if they + * were stored in Windows-1252. Note! This transformation only happens + * for numeric character references. The raw code points in the byte + * stream are not translated. + * + * > If the number is one of the numbers in the first column of + * > the following table, then find the row with that number in + * > the first column, and set the character reference code to + * > the number in the second column of that row. + */ + if ( $code_point >= 0x80 && $code_point <= 0x9F ) { + $windows_1252_mapping = array( + 0x20AC, // 0x80 -> EURO SIGN (€). + 0x81, // 0x81 -> (no change). + 0x201A, // 0x82 -> SINGLE LOW-9 QUOTATION MARK (‚). + 0x0192, // 0x83 -> LATIN SMALL LETTER F WITH HOOK (ƒ). + 0x201E, // 0x84 -> DOUBLE LOW-9 QUOTATION MARK („). + 0x2026, // 0x85 -> HORIZONTAL ELLIPSIS (…). + 0x2020, // 0x86 -> DAGGER (†). + 0x2021, // 0x87 -> DOUBLE DAGGER (‡). + 0x02C6, // 0x88 -> MODIFIER LETTER CIRCUMFLEX ACCENT (ˆ). + 0x2030, // 0x89 -> PER MILLE SIGN (‰). + 0x0160, // 0x8A -> LATIN CAPITAL LETTER S WITH CARON (Š). + 0x2039, // 0x8B -> SINGLE LEFT-POINTING ANGLE QUOTATION MARK (‹). + 0x0152, // 0x8C -> LATIN CAPITAL LIGATURE OE (Œ). + 0x8D, // 0x8D -> (no change). + 0x017D, // 0x8E -> LATIN CAPITAL LETTER Z WITH CARON (Ž). + 0x8F, // 0x8F -> (no change). + 0x90, // 0x90 -> (no change). + 0x2018, // 0x91 -> LEFT SINGLE QUOTATION MARK (‘). + 0x2019, // 0x92 -> RIGHT SINGLE QUOTATION MARK (’). + 0x201C, // 0x93 -> LEFT DOUBLE QUOTATION MARK (“). + 0x201D, // 0x94 -> RIGHT DOUBLE QUOTATION MARK (”). + 0x2022, // 0x95 -> BULLET (•). + 0x2013, // 0x96 -> EN DASH (–). + 0x2014, // 0x97 -> EM DASH (—). + 0x02DC, // 0x98 -> SMALL TILDE (˜). + 0x2122, // 0x99 -> TRADE MARK SIGN (™). + 0x0161, // 0x9A -> LATIN SMALL LETTER S WITH CARON (š). + 0x203A, // 0x9B -> SINGLE RIGHT-POINTING ANGLE QUOTATION MARK (›). + 0x0153, // 0x9C -> LATIN SMALL LIGATURE OE (œ). + 0x9D, // 0x9D -> (no change). + 0x017E, // 0x9E -> LATIN SMALL LETTER Z WITH CARON (ž). + 0x0178, // 0x9F -> LATIN CAPITAL LETTER Y WITH DIAERESIS (Ÿ). + ); + + $code_point = $windows_1252_mapping[ $code_point - 0x80 ]; + } + + $match_byte_length = $end_of_span - $at; + return self::code_point_to_utf8_bytes( $code_point ); + } + + /** Tracks inner parsing within the named character reference. */ + $name_at = $at + 1; + // Minimum named character reference is two characters. E.g. `GT`. + if ( $name_at + 2 > $length ) { + return null; + } + + $name_length = 0; + $replacement = $html5_named_character_references->read_token( $text, $name_at, $name_length ); + if ( false === $replacement ) { + return null; + } + + $after_name = $name_at + $name_length; + + // If the match ended with a semicolon then it should always be decoded. + if ( ';' === $text[ $name_at + $name_length - 1 ] ) { + $match_byte_length = $after_name - $at; + return $replacement; + } + + /* + * At this point though there's a match for an entry in the named + * character reference table but the match doesn't end in `;`. + * It may be allowed if it's followed by something unambiguous. + */ + $ambiguous_follower = ( + $after_name < $length && + $name_at < $length && + ( + ctype_alnum( $text[ $after_name ] ) || + '=' === $text[ $after_name ] + ) + ); + + // It's non-ambiguous, safe to leave it in. + if ( ! $ambiguous_follower ) { + $match_byte_length = $after_name - $at; + return $replacement; + } + + // It's ambiguous, which isn't allowed inside attributes. + if ( 'attribute' === $context ) { + return null; + } + + $match_byte_length = $after_name - $at; + return $replacement; + } + + /** + * Encode a code point number into the UTF-8 encoding. + * + * This encoder implements the UTF-8 encoding algorithm for converting + * a code point into a byte sequence. If it receives an invalid code + * point it will return the Unicode Replacement Character U+FFFD `�`. + * + * Example: + * + * '🅰' === WP_HTML_Decoder::code_point_to_utf8_bytes( 0x1f170 ); + * + * // Half of a surrogate pair is an invalid code point. + * '�' === WP_HTML_Decoder::code_point_to_utf8_bytes( 0xd83c ); + * + * @since 6.6.0 + * + * @see https://www.rfc-editor.org/rfc/rfc3629 For the UTF-8 standard. + * + * @param int $code_point Which code point to convert. + * @return string Converted code point, or `�` if invalid. + */ + public static function code_point_to_utf8_bytes( $code_point ) { + // Pre-check to ensure a valid code point. + if ( + $code_point <= 0 || + ( $code_point >= 0xD800 && $code_point <= 0xDFFF ) || + $code_point > 0x10FFFF + ) { + return '�'; + } + + if ( $code_point <= 0x7F ) { + return chr( $code_point ); + } + + if ( $code_point <= 0x7FF ) { + $byte1 = ( $code_point >> 6 ) | 0xC0; + $byte2 = $code_point & 0x3F | 0x80; + + return pack( 'CC', $byte1, $byte2 ); + } + + if ( $code_point <= 0xFFFF ) { + $byte1 = ( $code_point >> 12 ) | 0xE0; + $byte2 = ( $code_point >> 6 ) & 0x3F | 0x80; + $byte3 = $code_point & 0x3F | 0x80; + + return pack( 'CCC', $byte1, $byte2, $byte3 ); + } + + // Any values above U+10FFFF are eliminated above in the pre-check. + $byte1 = ( $code_point >> 18 ) | 0xF0; + $byte2 = ( $code_point >> 12 ) & 0x3F | 0x80; + $byte3 = ( $code_point >> 6 ) & 0x3F | 0x80; + $byte4 = $code_point & 0x3F | 0x80; + + return pack( 'CCCC', $byte1, $byte2, $byte3, $byte4 ); + } +} diff --git a/wp-includes/html-api/class-wp-html-open-elements.php b/wp-includes/html-api/class-wp-html-open-elements.php index 1234abc..1547980 100644 --- a/wp-includes/html-api/class-wp-html-open-elements.php +++ b/wp-includes/html-api/class-wp-html-open-elements.php @@ -51,6 +51,56 @@ class WP_HTML_Open_Elements { */ private $has_p_in_button_scope = false; + /** + * A function that will be called when an item is popped off the stack of open elements. + * + * The function will be called with the popped item as its argument. + * + * @since 6.6.0 + * + * @var Closure + */ + private $pop_handler = null; + + /** + * A function that will be called when an item is pushed onto the stack of open elements. + * + * The function will be called with the pushed item as its argument. + * + * @since 6.6.0 + * + * @var Closure + */ + private $push_handler = null; + + /** + * Sets a pop handler that will be called when an item is popped off the stack of + * open elements. + * + * The function will be called with the pushed item as its argument. + * + * @since 6.6.0 + * + * @param Closure $handler The handler function. + */ + public function set_pop_handler( Closure $handler ) { + $this->pop_handler = $handler; + } + + /** + * Sets a push handler that will be called when an item is pushed onto the stack of + * open elements. + * + * The function will be called with the pushed item as its argument. + * + * @since 6.6.0 + * + * @param Closure $handler The handler function. + */ + public function set_push_handler( Closure $handler ) { + $this->push_handler = $handler; + } + /** * Reports if a specific node is in the stack of open elements. * @@ -258,11 +308,15 @@ class WP_HTML_Open_Elements { */ public function pop() { $item = array_pop( $this->stack ); - if ( null === $item ) { return false; } + if ( 'context-node' === $item->bookmark_name ) { + $this->stack[] = $item; + return false; + } + $this->after_element_pop( $item ); return true; } @@ -279,6 +333,10 @@ class WP_HTML_Open_Elements { */ public function pop_until( $tag_name ) { foreach ( $this->walk_up() as $item ) { + if ( 'context-node' === $item->bookmark_name ) { + return true; + } + $this->pop(); if ( @@ -319,6 +377,10 @@ class WP_HTML_Open_Elements { * @return bool Whether the node was found and removed from the stack of open elements. */ public function remove_node( $token ) { + if ( 'context-node' === $token->bookmark_name ) { + return false; + } + foreach ( $this->walk_up() as $position_from_end => $item ) { if ( $token->bookmark_name !== $item->bookmark_name ) { continue; @@ -429,6 +491,10 @@ class WP_HTML_Open_Elements { $this->has_p_in_button_scope = true; break; } + + if ( null !== $this->push_handler ) { + ( $this->push_handler )( $item ); + } } /** @@ -458,5 +524,18 @@ class WP_HTML_Open_Elements { $this->has_p_in_button_scope = $this->has_element_in_button_scope( 'P' ); break; } + + if ( null !== $this->pop_handler ) { + ( $this->pop_handler )( $item ); + } + } + + /** + * Wakeup magic method. + * + * @since 6.6.0 + */ + public function __wakeup() { + throw new \LogicException( __CLASS__ . ' should never be unserialized' ); } } diff --git a/wp-includes/html-api/class-wp-html-processor.php b/wp-includes/html-api/class-wp-html-processor.php index c76cc19..29f1c7a 100644 --- a/wp-includes/html-api/class-wp-html-processor.php +++ b/wp-includes/html-api/class-wp-html-processor.php @@ -201,6 +201,52 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { */ private $release_internal_bookmark_on_destruct = null; + /** + * Stores stack events which arise during parsing of the + * HTML document, which will then supply the "match" events. + * + * @since 6.6.0 + * + * @var WP_HTML_Stack_Event[] + */ + private $element_queue = array(); + + /** + * Current stack event, if set, representing a matched token. + * + * Because the parser may internally point to a place further along in a document + * than the nodes which have already been processed (some "virtual" nodes may have + * appeared while scanning the HTML document), this will point at the "current" node + * being processed. It comes from the front of the element queue. + * + * @since 6.6.0 + * + * @var ?WP_HTML_Stack_Event + */ + private $current_element = null; + + /** + * Context node if created as a fragment parser. + * + * @var ?WP_HTML_Token + */ + private $context_node = null; + + /** + * Whether the parser has yet processed the context node, + * if created as a fragment parser. + * + * The context node will be initially pushed onto the stack of open elements, + * but when created as a fragment parser, this context element (and the implicit + * HTML document node above it) should not be exposed as a matched token or node. + * + * This boolean indicates whether the processor should skip over the current + * node in its initial search for the first node created from the input HTML. + * + * @var bool + */ + private $has_seen_context_node = false; + /* * Public Interface Functions */ @@ -230,18 +276,19 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { * - The only supported document encoding is `UTF-8`, which is the default value. * * @since 6.4.0 + * @since 6.6.0 Returns `static` instead of `self` so it can create subclass instances. * * @param string $html Input HTML fragment to process. * @param string $context Context element for the fragment, must be default of ``. * @param string $encoding Text encoding of the document; must be default of 'UTF-8'. - * @return WP_HTML_Processor|null The created processor if successful, otherwise null. + * @return static|null The created processor if successful, otherwise null. */ public static function create_fragment( $html, $context = '', $encoding = 'UTF-8' ) { if ( '' !== $context || 'UTF-8' !== $encoding ) { return null; } - $processor = new self( $html, self::CONSTRUCTOR_UNLOCK_CODE ); + $processor = new static( $html, self::CONSTRUCTOR_UNLOCK_CODE ); $processor->state->context_node = array( 'BODY', array() ); $processor->state->insertion_mode = WP_HTML_Processor_State::INSERTION_MODE_IN_BODY; @@ -257,14 +304,15 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { ) ); - $processor->state->stack_of_open_elements->push( - new WP_HTML_Token( - 'context-node', - $processor->state->context_node[0], - false - ) + $context_node = new WP_HTML_Token( + 'context-node', + $processor->state->context_node[0], + false ); + $processor->state->stack_of_open_elements->push( $context_node ); + $processor->context_node = $context_node; + return $processor; } @@ -299,6 +347,24 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { $this->state = new WP_HTML_Processor_State(); + $this->state->stack_of_open_elements->set_push_handler( + function ( WP_HTML_Token $token ) { + $is_virtual = ! isset( $this->state->current_token ) || $this->is_tag_closer(); + $same_node = isset( $this->state->current_token ) && $token->node_name === $this->state->current_token->node_name; + $provenance = ( ! $same_node || $is_virtual ) ? 'virtual' : 'real'; + $this->element_queue[] = new WP_HTML_Stack_Event( $token, WP_HTML_Stack_Event::PUSH, $provenance ); + } + ); + + $this->state->stack_of_open_elements->set_pop_handler( + function ( WP_HTML_Token $token ) { + $is_virtual = ! isset( $this->state->current_token ) || ! $this->is_tag_closer(); + $same_node = isset( $this->state->current_token ) && $token->node_name === $this->state->current_token->node_name; + $provenance = ( ! $same_node || $is_virtual ) ? 'virtual' : 'real'; + $this->element_queue[] = new WP_HTML_Stack_Event( $token, WP_HTML_Stack_Event::POP, $provenance ); + } + ); + /* * Create this wrapper so that it's possible to pass * a private method into WP_HTML_Token classes without @@ -342,6 +408,7 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { * @todo Support matching the class name and tag name. * * @since 6.4.0 + * @since 6.6.0 Visits all tokens, including virtual ones. * * @throws Exception When unable to allocate a bookmark for the next token in the input HTML document. * @@ -349,6 +416,7 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { * Optional. Which tag name to find, having which class, etc. Default is to find any tag. * * @type string|null $tag_name Which tag to find, or `null` for "any tag." + * @type string $tag_closers 'visit' to pause at tag closers, 'skip' or unset to only visit openers. * @type int|null $match_offset Find the Nth tag matching all search criteria. * 1 for "first" tag, 3 for "third," etc. * Defaults to first tag. @@ -359,13 +427,15 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { * @return bool Whether a tag was matched. */ public function next_tag( $query = null ) { + $visit_closers = isset( $query['tag_closers'] ) && 'visit' === $query['tag_closers']; + if ( null === $query ) { - while ( $this->step() ) { + while ( $this->next_token() ) { if ( '#tag' !== $this->get_token_type() ) { continue; } - if ( ! $this->is_tag_closer() ) { + if ( ! $this->is_tag_closer() || $visit_closers ) { return true; } } @@ -386,13 +456,21 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { return false; } + $needs_class = ( isset( $query['class_name'] ) && is_string( $query['class_name'] ) ) + ? $query['class_name'] + : null; + if ( ! ( array_key_exists( 'breadcrumbs', $query ) && is_array( $query['breadcrumbs'] ) ) ) { - while ( $this->step() ) { + while ( $this->next_token() ) { if ( '#tag' !== $this->get_token_type() ) { continue; } - if ( ! $this->is_tag_closer() ) { + if ( isset( $needs_class ) && ! $this->has_class( $needs_class ) ) { + continue; + } + + if ( ! $this->is_tag_closer() || $visit_closers ) { return true; } } @@ -400,20 +478,15 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { return false; } - if ( isset( $query['tag_closers'] ) && 'visit' === $query['tag_closers'] ) { - _doing_it_wrong( - __METHOD__, - __( 'Cannot visit tag closers in HTML Processor.' ), - '6.4.0' - ); - return false; - } - $breadcrumbs = $query['breadcrumbs']; $match_offset = isset( $query['match_offset'] ) ? (int) $query['match_offset'] : 1; - while ( $match_offset > 0 && $this->step() ) { - if ( '#tag' !== $this->get_token_type() ) { + while ( $match_offset > 0 && $this->next_token() ) { + if ( '#tag' !== $this->get_token_type() || $this->is_tag_closer() ) { + continue; + } + + if ( isset( $needs_class ) && ! $this->has_class( $needs_class ) ) { continue; } @@ -440,7 +513,92 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { * @return bool */ public function next_token() { - return $this->step(); + $this->current_element = null; + + if ( isset( $this->last_error ) ) { + return false; + } + + if ( 'done' !== $this->has_seen_context_node && 0 === count( $this->element_queue ) && ! $this->step() ) { + while ( 'context-node' !== $this->state->stack_of_open_elements->current_node()->bookmark_name && $this->state->stack_of_open_elements->pop() ) { + continue; + } + $this->has_seen_context_node = 'done'; + return $this->next_token(); + } + + $this->current_element = array_shift( $this->element_queue ); + while ( isset( $this->context_node ) && ! $this->has_seen_context_node ) { + if ( isset( $this->current_element ) ) { + if ( $this->context_node === $this->current_element->token && WP_HTML_Stack_Event::PUSH === $this->current_element->operation ) { + $this->has_seen_context_node = true; + return $this->next_token(); + } + } + $this->current_element = array_shift( $this->element_queue ); + } + + if ( ! isset( $this->current_element ) ) { + if ( 'done' === $this->has_seen_context_node ) { + return false; + } else { + return $this->next_token(); + } + } + + if ( isset( $this->context_node ) && WP_HTML_Stack_Event::POP === $this->current_element->operation && $this->context_node === $this->current_element->token ) { + $this->element_queue = array(); + $this->current_element = null; + return false; + } + + // Avoid sending close events for elements which don't expect a closing. + if ( + WP_HTML_Stack_Event::POP === $this->current_element->operation && + ! static::expects_closer( $this->current_element->token ) + ) { + return $this->next_token(); + } + + return true; + } + + + /** + * Indicates if the current tag token is a tag closer. + * + * Example: + * + * $p = WP_HTML_Processor::create_fragment( '
' ); + * $p->next_tag( array( 'tag_name' => 'div', 'tag_closers' => 'visit' ) ); + * $p->is_tag_closer() === false; + * + * $p->next_tag( array( 'tag_name' => 'div', 'tag_closers' => 'visit' ) ); + * $p->is_tag_closer() === true; + * + * @since 6.6.0 Subclassed for HTML Processor. + * + * @return bool Whether the current tag is a tag closer. + */ + public function is_tag_closer() { + return $this->is_virtual() + ? ( WP_HTML_Stack_Event::POP === $this->current_element->operation && '#tag' === $this->get_token_type() ) + : parent::is_tag_closer(); + } + + /** + * Indicates if the currently-matched token is virtual, created by a stack operation + * while processing HTML, rather than a token found in the HTML text itself. + * + * @since 6.6.0 + * + * @return bool Whether the current token is virtual. + */ + private function is_virtual() { + return ( + isset( $this->current_element->provenance ) && + 'virtual' === $this->current_element->provenance + ); } /** @@ -496,6 +654,45 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { return false; } + /** + * Indicates if the currently-matched node expects a closing + * token, or if it will self-close on the next step. + * + * Most HTML elements expect a closer, such as a P element or + * a DIV element. Others, like an IMG element are void and don't + * have a closing tag. Special elements, such as SCRIPT and STYLE, + * are treated just like void tags. Text nodes and self-closing + * foreign content will also act just like a void tag, immediately + * closing as soon as the processor advances to the next token. + * + * @since 6.6.0 + * + * @todo When adding support for foreign content, ensure that + * this returns false for self-closing elements in the + * SVG and MathML namespace. + * + * @param ?WP_HTML_Token $node Node to examine instead of current node, if provided. + * @return bool Whether to expect a closer for the currently-matched node, + * or `null` if not matched on any token. + */ + public function expects_closer( $node = null ) { + $token_name = $node->node_name ?? $this->get_token_name(); + if ( ! isset( $token_name ) ) { + return null; + } + + return ! ( + // Comments, text nodes, and other atomic tokens. + '#' === $token_name[0] || + // Doctype declarations. + 'html' === $token_name || + // Void elements. + self::is_void( $token_name ) || + // Special atomic elements. + in_array( $token_name, array( 'IFRAME', 'NOEMBED', 'NOFRAMES', 'SCRIPT', 'STYLE', 'TEXTAREA', 'TITLE', 'XMP' ), true ) + ); + } + /** * Steps through the HTML document and stop at the next tag, if any. * @@ -531,16 +728,7 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { * is provided in the opening tag, otherwise it expects a tag closer. */ $top_node = $this->state->stack_of_open_elements->current_node(); - if ( - $top_node && ( - // Void elements. - self::is_void( $top_node->node_name ) || - // Comments, text nodes, and other atomic tokens. - '#' === $top_node->node_name[0] || - // Doctype declarations. - 'html' === $top_node->node_name - ) - ) { + if ( isset( $top_node ) && ! static::expects_closer( $top_node ) ) { $this->state->stack_of_open_elements->pop(); } } @@ -604,13 +792,78 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { */ public function get_breadcrumbs() { $breadcrumbs = array(); + foreach ( $this->state->stack_of_open_elements->walk_down() as $stack_item ) { $breadcrumbs[] = $stack_item->node_name; } + if ( ! $this->is_virtual() ) { + return $breadcrumbs; + } + + foreach ( $this->element_queue as $queue_item ) { + if ( $this->current_element->token->bookmark_name === $queue_item->token->bookmark_name ) { + break; + } + + if ( 'context-node' === $queue_item->token->bookmark_name ) { + break; + } + + if ( 'real' === $queue_item->provenance ) { + break; + } + + if ( WP_HTML_Stack_Event::PUSH === $queue_item->operation ) { + $breadcrumbs[] = $queue_item->token->node_name; + } else { + array_pop( $breadcrumbs ); + } + } + + if ( null !== parent::get_token_name() && ! parent::is_tag_closer() ) { + array_pop( $breadcrumbs ); + } + + // Add the virtual node we're at. + if ( WP_HTML_Stack_Event::PUSH === $this->current_element->operation ) { + $breadcrumbs[] = $this->current_element->token->node_name; + } + return $breadcrumbs; } + /** + * Returns the nesting depth of the current location in the document. + * + * Example: + * + * $processor = WP_HTML_Processor::create_fragment( '

' ); + * // The processor starts in the BODY context, meaning it has depth from the start: HTML > BODY. + * 2 === $processor->get_current_depth(); + * + * // Opening the DIV element increases the depth. + * $processor->next_token(); + * 3 === $processor->get_current_depth(); + * + * // Opening the P element increases the depth. + * $processor->next_token(); + * 4 === $processor->get_current_depth(); + * + * // The P element is closed during `next_token()` so the depth is decreased to reflect that. + * $processor->next_token(); + * 3 === $processor->get_current_depth(); + * + * @since 6.6.0 + * + * @return int Nesting-depth of current location in the document. + */ + public function get_current_depth() { + return $this->is_virtual() + ? count( $this->get_breadcrumbs() ) + : $this->state->stack_of_open_elements->count(); + } + /** * Parses next element in the 'in body' insertion mode. * @@ -629,7 +882,7 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { private function step_in_body() { $token_name = $this->get_token_name(); $token_type = $this->get_token_type(); - $op_sigil = '#tag' === $token_type ? ( $this->is_tag_closer() ? '-' : '+' ) : ''; + $op_sigil = '#tag' === $token_type ? ( parent::is_tag_closer() ? '-' : '+' ) : ''; $op = "{$op_sigil}{$token_name}"; switch ( $op ) { @@ -1152,7 +1405,7 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { throw new WP_HTML_Unsupported_Exception( "Cannot process {$token_name} element." ); } - if ( ! $this->is_tag_closer() ) { + if ( ! parent::is_tag_closer() ) { /* * > Any other start tag */ @@ -1248,6 +1501,10 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { return null; } + if ( $this->is_virtual() ) { + return $this->current_element->token->node_name; + } + $tag_name = parent::get_tag(); switch ( $tag_name ) { @@ -1263,6 +1520,286 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { } } + /** + * Indicates if the currently matched tag contains the self-closing flag. + * + * No HTML elements ought to have the self-closing flag and for those, the self-closing + * flag will be ignored. For void elements this is benign because they "self close" + * automatically. For non-void HTML elements though problems will appear if someone + * intends to use a self-closing element in place of that element with an empty body. + * For HTML foreign elements and custom elements the self-closing flag determines if + * they self-close or not. + * + * This function does not determine if a tag is self-closing, + * but only if the self-closing flag is present in the syntax. + * + * @since 6.6.0 Subclassed for the HTML Processor. + * + * @return bool Whether the currently matched tag contains the self-closing flag. + */ + public function has_self_closing_flag() { + return $this->is_virtual() ? false : parent::has_self_closing_flag(); + } + + /** + * Returns the node name represented by the token. + * + * This matches the DOM API value `nodeName`. Some values + * are static, such as `#text` for a text node, while others + * are dynamically generated from the token itself. + * + * Dynamic names: + * - Uppercase tag name for tag matches. + * - `html` for DOCTYPE declarations. + * + * Note that if the Tag Processor is not matched on a token + * then this function will return `null`, either because it + * hasn't yet found a token or because it reached the end + * of the document without matching a token. + * + * @since 6.6.0 Subclassed for the HTML Processor. + * + * @return string|null Name of the matched token. + */ + public function get_token_name() { + return $this->is_virtual() + ? $this->current_element->token->node_name + : parent::get_token_name(); + } + + /** + * Indicates the kind of matched token, if any. + * + * This differs from `get_token_name()` in that it always + * returns a static string indicating the type, whereas + * `get_token_name()` may return values derived from the + * token itself, such as a tag name or processing + * instruction tag. + * + * Possible values: + * - `#tag` when matched on a tag. + * - `#text` when matched on a text node. + * - `#cdata-section` when matched on a CDATA node. + * - `#comment` when matched on a comment. + * - `#doctype` when matched on a DOCTYPE declaration. + * - `#presumptuous-tag` when matched on an empty tag closer. + * - `#funky-comment` when matched on a funky comment. + * + * @since 6.6.0 Subclassed for the HTML Processor. + * + * @return string|null What kind of token is matched, or null. + */ + public function get_token_type() { + if ( $this->is_virtual() ) { + /* + * This logic comes from the Tag Processor. + * + * @todo It would be ideal not to repeat this here, but it's not clearly + * better to allow passing a token name to `get_token_type()`. + */ + $node_name = $this->current_element->token->node_name; + $starting_char = $node_name[0]; + if ( 'A' <= $starting_char && 'Z' >= $starting_char ) { + return '#tag'; + } + + if ( 'html' === $node_name ) { + return '#doctype'; + } + + return $node_name; + } + + return parent::get_token_type(); + } + + /** + * Returns the value of a requested attribute from a matched tag opener if that attribute exists. + * + * Example: + * + * $p = WP_HTML_Processor::create_fragment( '
Test
' ); + * $p->next_token() === true; + * $p->get_attribute( 'data-test-id' ) === '14'; + * $p->get_attribute( 'enabled' ) === true; + * $p->get_attribute( 'aria-label' ) === null; + * + * $p->next_tag() === false; + * $p->get_attribute( 'class' ) === null; + * + * @since 6.6.0 Subclassed for HTML Processor. + * + * @param string $name Name of attribute whose value is requested. + * @return string|true|null Value of attribute or `null` if not available. Boolean attributes return `true`. + */ + public function get_attribute( $name ) { + return $this->is_virtual() ? null : parent::get_attribute( $name ); + } + + /** + * Updates or creates a new attribute on the currently matched tag with the passed value. + * + * For boolean attributes special handling is provided: + * - When `true` is passed as the value, then only the attribute name is added to the tag. + * - When `false` is passed, the attribute gets removed if it existed before. + * + * For string attributes, the value is escaped using the `esc_attr` function. + * + * @since 6.6.0 Subclassed for the HTML Processor. + * + * @param string $name The attribute name to target. + * @param string|bool $value The new attribute value. + * @return bool Whether an attribute value was set. + */ + public function set_attribute( $name, $value ) { + return $this->is_virtual() ? false : parent::set_attribute( $name, $value ); + } + + /** + * Remove an attribute from the currently-matched tag. + * + * @since 6.6.0 Subclassed for HTML Processor. + * + * @param string $name The attribute name to remove. + * @return bool Whether an attribute was removed. + */ + public function remove_attribute( $name ) { + return $this->is_virtual() ? false : parent::remove_attribute( $name ); + } + + /** + * Gets lowercase names of all attributes matching a given prefix in the current tag. + * + * Note that matching is case-insensitive. This is in accordance with the spec: + * + * > There must never be two or more attributes on + * > the same start tag whose names are an ASCII + * > case-insensitive match for each other. + * - HTML 5 spec + * + * Example: + * + * $p = new WP_HTML_Tag_Processor( '
Test
' ); + * $p->next_tag( array( 'class_name' => 'test' ) ) === true; + * $p->get_attribute_names_with_prefix( 'data-' ) === array( 'data-enabled', 'data-test-id' ); + * + * $p->next_tag() === false; + * $p->get_attribute_names_with_prefix( 'data-' ) === null; + * + * @since 6.6.0 Subclassed for the HTML Processor. + * + * @see https://html.spec.whatwg.org/multipage/syntax.html#attributes-2:ascii-case-insensitive + * + * @param string $prefix Prefix of requested attribute names. + * @return array|null List of attribute names, or `null` when no tag opener is matched. + */ + public function get_attribute_names_with_prefix( $prefix ) { + return $this->is_virtual() ? null : parent::get_attribute_names_with_prefix( $prefix ); + } + + /** + * Adds a new class name to the currently matched tag. + * + * @since 6.6.0 Subclassed for the HTML Processor. + * + * @param string $class_name The class name to add. + * @return bool Whether the class was set to be added. + */ + public function add_class( $class_name ) { + return $this->is_virtual() ? false : parent::add_class( $class_name ); + } + + /** + * Removes a class name from the currently matched tag. + * + * @since 6.6.0 Subclassed for the HTML Processor. + * + * @param string $class_name The class name to remove. + * @return bool Whether the class was set to be removed. + */ + public function remove_class( $class_name ) { + return $this->is_virtual() ? false : parent::remove_class( $class_name ); + } + + /** + * Returns if a matched tag contains the given ASCII case-insensitive class name. + * + * @since 6.6.0 Subclassed for the HTML Processor. + * + * @param string $wanted_class Look for this CSS class name, ASCII case-insensitive. + * @return bool|null Whether the matched tag contains the given class name, or null if not matched. + */ + public function has_class( $wanted_class ) { + return $this->is_virtual() ? null : parent::has_class( $wanted_class ); + } + + /** + * Generator for a foreach loop to step through each class name for the matched tag. + * + * This generator function is designed to be used inside a "foreach" loop. + * + * Example: + * + * $p = WP_HTML_Processor::create_fragment( "
" ); + * $p->next_tag(); + * foreach ( $p->class_list() as $class_name ) { + * echo "{$class_name} "; + * } + * // Outputs: "free lang-en " + * + * @since 6.6.0 Subclassed for the HTML Processor. + */ + public function class_list() { + return $this->is_virtual() ? null : parent::class_list(); + } + + /** + * Returns the modifiable text for a matched token, or an empty string. + * + * Modifiable text is text content that may be read and changed without + * changing the HTML structure of the document around it. This includes + * the contents of `#text` nodes in the HTML as well as the inner + * contents of HTML comments, Processing Instructions, and others, even + * though these nodes aren't part of a parsed DOM tree. They also contain + * the contents of SCRIPT and STYLE tags, of TEXTAREA tags, and of any + * other section in an HTML document which cannot contain HTML markup (DATA). + * + * If a token has no modifiable text then an empty string is returned to + * avoid needless crashing or type errors. An empty string does not mean + * that a token has modifiable text, and a token with modifiable text may + * have an empty string (e.g. a comment with no contents). + * + * @since 6.6.0 Subclassed for the HTML Processor. + * + * @return string + */ + public function get_modifiable_text() { + return $this->is_virtual() ? '' : parent::get_modifiable_text(); + } + + /** + * Indicates what kind of comment produced the comment node. + * + * Because there are different kinds of HTML syntax which produce + * comments, the Tag Processor tracks and exposes this as a type + * for the comment. Nominally only regular HTML comments exist as + * they are commonly known, but a number of unrelated syntax errors + * also produce comments. + * + * @see self::COMMENT_AS_ABRUPTLY_CLOSED_COMMENT + * @see self::COMMENT_AS_CDATA_LOOKALIKE + * @see self::COMMENT_AS_INVALID_HTML + * @see self::COMMENT_AS_HTML_COMMENT + * @see self::COMMENT_AS_PI_NODE_LOOKALIKE + * + * @since 6.6.0 Subclassed for the HTML Processor. + * + * @return string|null + */ + public function get_comment_type() { + return $this->is_virtual() ? null : parent::get_comment_type(); + } + /** * Removes a bookmark that is no longer needed. * @@ -1304,6 +1841,7 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { ? $this->bookmarks[ $this->state->current_token->bookmark_name ]->start : 0; $bookmark_starts_at = $this->bookmarks[ $actual_bookmark_name ]->start; + $bookmark_length = $this->bookmarks[ $actual_bookmark_name ]->length; $direction = $bookmark_starts_at > $processor_started_at ? 'forward' : 'backward'; /* @@ -1359,6 +1897,8 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { parent::seek( 'context-node' ); $this->state->insertion_mode = WP_HTML_Processor_State::INSERTION_MODE_IN_BODY; $this->state->frameset_ok = true; + $this->element_queue = array(); + $this->current_element = null; } // When moving forwards, reparse the document until reaching the same location as the original bookmark. @@ -1366,8 +1906,11 @@ class WP_HTML_Processor extends WP_HTML_Tag_Processor { return true; } - while ( $this->step() ) { + while ( $this->next_token() ) { if ( $bookmark_starts_at === $this->bookmarks[ $this->state->current_token->bookmark_name ]->start ) { + while ( isset( $this->current_element ) && WP_HTML_Stack_Event::POP === $this->current_element->operation ) { + $this->current_element = array_shift( $this->element_queue ); + } return true; } } diff --git a/wp-includes/html-api/class-wp-html-stack-event.php b/wp-includes/html-api/class-wp-html-stack-event.php new file mode 100644 index 0000000..aa4763c --- /dev/null +++ b/wp-includes/html-api/class-wp-html-stack-event.php @@ -0,0 +1,82 @@ +token = $token; + $this->operation = $operation; + $this->provenance = $provenance; + } +} diff --git a/wp-includes/html-api/class-wp-html-tag-processor.php b/wp-includes/html-api/class-wp-html-tag-processor.php index bf7e026..8fc7593 100644 --- a/wp-includes/html-api/class-wp-html-tag-processor.php +++ b/wp-includes/html-api/class-wp-html-tag-processor.php @@ -15,10 +15,6 @@ * - Prune the whitespace when removing classes/attributes: e.g. "a b c" -> "c" not " c". * This would increase the size of the changes for some operations but leave more * natural-looking output HTML. - * - Properly decode HTML character references in `get_attribute()`. PHP's - * `html_entity_decode()` is wrong in a couple ways: it doesn't account for the - * no-ambiguous-ampersand rule, and it improperly handles the way semicolons may - * or may not terminate a character reference. * * @package WordPress * @subpackage HTML-API @@ -298,8 +294,8 @@ * * The special elements are: * - `SCRIPT` whose contents are treated as raw plaintext but supports a legacy - * style of including Javascript inside of HTML comments to avoid accidentally - * closing the SCRIPT from inside a Javascript string. E.g. `console.log( '' )`. + * style of including JavaScript inside of HTML comments to avoid accidentally + * closing the SCRIPT from inside a JavaScript string. E.g. `console.log( '' )`. * - `TITLE` and `TEXTAREA` whose contents are treated as plaintext and then any * character references are decoded. E.g. `1 < 2 < 3` becomes `1 < 2 < 3`. * - `IFRAME`, `NOSCRIPT`, `NOEMBED`, `NOFRAME`, `STYLE` whose contents are treated as @@ -926,8 +922,8 @@ class WP_HTML_Tag_Processor { return false; } $this->parser_state = self::STATE_MATCHED_TAG; - $this->token_length = $tag_ends_at - $this->token_starts_at; $this->bytes_already_parsed = $tag_ends_at + 1; + $this->token_length = $this->bytes_already_parsed - $this->token_starts_at; /* * For non-DATA sections which might contain text that looks like HTML tags but @@ -1013,7 +1009,7 @@ class WP_HTML_Tag_Processor { */ $this->token_starts_at = $was_at; $this->token_length = $this->bytes_already_parsed - $this->token_starts_at; - $this->text_starts_at = $tag_ends_at + 1; + $this->text_starts_at = $tag_ends_at; $this->text_length = $this->tag_name_starts_at - $this->text_starts_at; $this->tag_name_starts_at = $tag_name_starts_at; $this->tag_name_length = $tag_name_length; @@ -1629,7 +1625,7 @@ class WP_HTML_Tag_Processor { * `is_closing_tag && '!' === $html[ $at + 1 ] ) { /* * `