diff options
Diffstat (limited to '')
| -rw-r--r-- | src/html-parse.c | 1204 |
1 files changed, 1204 insertions, 0 deletions
diff --git a/src/html-parse.c b/src/html-parse.c new file mode 100644 index 0000000..658f90f --- /dev/null +++ b/src/html-parse.c @@ -0,0 +1,1204 @@ +/* HTML parser for Wget. + Copyright (C) 1998-2011, 2015, 2018 Free Software Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +/* The only entry point to this module is map_html_tags(), which see. */ + +/* TODO: + + - Allow hooks for callers to process contents outside tags. This + is needed to implement handling <style> and <script>. The + taginfo structure already carries the information about where the + tags are, but this is not enough, because one would also want to + skip the comments. (The funny thing is that for <style> and + <script> you *don't* want to skip comments!) + + - Create a test suite for regression testing. */ + +/* HISTORY: + + This is the third HTML parser written for Wget. The first one was + written some time during the Geturl 1.0 beta cycle, and was very + inefficient and buggy. It also contained some very complex code to + remember a list of parser states, because it was supposed to be + reentrant. + + The second HTML parser was written for Wget 1.4 (the first version + by the name `Wget'), and was a complete rewrite. Although the new + parser behaved much better and made no claims of reentrancy, it + still shared many of the fundamental flaws of the old version -- it + only regarded HTML in terms tag-attribute pairs, where the + attribute's value was a URL to be returned. Any other property of + HTML, such as <base href=...>, or strange way to specify a URL, + such as <meta http-equiv=Refresh content="0; URL=..."> had to be + crudely hacked in -- and the caller had to be aware of these hacks. + Like its predecessor, this parser did not support HTML comments. + + After Wget 1.5.1 was released, I set out to write a third HTML + parser. The objectives of the new parser were to: (1) provide a + clean way to analyze HTML lexically, (2) separate interpretation of + the markup from the parsing process, (3) be as correct as possible, + e.g. correctly skipping comments and other SGML declarations, (4) + understand the most common errors in markup and skip them or be + relaxed towrds them, and (5) be reasonably efficient (no regexps, + minimum copying and minimum or no heap allocation). + + I believe this parser meets all of the above goals. It is + reasonably well structured, and could be relatively easily + separated from Wget and used elsewhere. While some of its + intrinsic properties limit its value as a general-purpose HTML + parser, I believe that, with minimum modifications, it could serve + as a backend for one. + + Due to time and other constraints, this parser was not integrated + into Wget until the version 1.7. */ + +/* DESCRIPTION: + + The single entry point of this parser is map_html_tags(), which + works by calling a function you specify for each tag. The function + gets called with the pointer to a structure describing the tag and + its attributes. */ + +/* To test as standalone, compile with `-DSTANDALONE -I.'. You'll + still need Wget headers to compile. */ + +#include "wget.h" + +#ifdef STANDALONE +# define I_REALLY_WANT_CTYPE_MACROS +#endif + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#include "utils.h" +#include "html-parse.h" + +#ifdef STANDALONE +# undef xmalloc +# undef xrealloc +# undef xfree +# define xmalloc malloc +# define xrealloc realloc +# define xfree free + +# undef c_isspace +# undef c_isdigit +# undef c_isxdigit +# undef c_isalpha +# undef c_isalnum +# undef c_tolower +# undef c_toupper + +# define c_isspace(x) isspace (x) +# define c_isdigit(x) isdigit (x) +# define c_isxdigit(x) isxdigit (x) +# define c_isalpha(x) isalpha (x) +# define c_isalnum(x) isalnum (x) +# define c_tolower(x) tolower (x) +# define c_toupper(x) toupper (x) + +struct hash_table { + int dummy; +}; +static void * +hash_table_get (const struct hash_table *ht, void *ptr) +{ + return ptr; +} +#else /* not STANDALONE */ +# include "hash.h" +#endif + +/* Pool support. A pool is a resizable chunk of memory. It is first + allocated on the stack, and moved to the heap if it needs to be + larger than originally expected. map_html_tags() uses it to store + the zero-terminated names and values of tags and attributes. + + Thus taginfo->name, and attr->name and attr->value for each + attribute, do not point into separately allocated areas, but into + different parts of the pool, separated only by terminating zeros. + This ensures minimum amount of allocation and, for most tags, no + allocation because the entire pool is kept on the stack. */ + +struct pool { + char *contents; /* pointer to the contents. */ + int size; /* size of the pool. */ + int tail; /* next available position index. */ + bool resized; /* whether the pool has been resized + using malloc. */ + + char *orig_contents; /* original pool contents, usually + stack-allocated. used by POOL_FREE + to restore the pool to the initial + state. */ + int orig_size; +}; + +/* Initialize the pool to hold INITIAL_SIZE bytes of storage. */ + +#define POOL_INIT(p, initial_storage, initial_size) do { \ + struct pool *P = (p); \ + P->contents = (initial_storage); \ + P->size = (initial_size); \ + P->tail = 0; \ + P->resized = false; \ + P->orig_contents = P->contents; \ + P->orig_size = P->size; \ +} while (0) + +/* Grow the pool to accommodate at least SIZE new bytes. If the pool + already has room to accommodate SIZE bytes of data, this is a no-op. */ + +#define POOL_GROW(p, increase) \ + GROW_ARRAY ((p)->contents, (p)->size, (p)->tail + (increase), \ + (p)->resized, char) + +/* Append text in the range [beg, end) to POOL. No zero-termination + is done. */ + +#define POOL_APPEND(p, beg, end) do { \ + const char *PA_beg = (beg); \ + int PA_size = (end) - PA_beg; \ + POOL_GROW (p, PA_size); \ + memcpy ((p)->contents + (p)->tail, PA_beg, PA_size); \ + (p)->tail += PA_size; \ +} while (0) + +/* Append one character to the pool. Can be used to zero-terminate + pool strings. */ + +#define POOL_APPEND_CHR(p, ch) do { \ + char PAC_char = (ch); \ + POOL_GROW (p, 1); \ + (p)->contents[(p)->tail++] = PAC_char; \ +} while (0) + +/* Forget old pool contents. The allocated memory is not freed. */ +#define POOL_REWIND(p) (p)->tail = 0 + +/* Free heap-allocated memory for contents of POOL. This calls + xfree() if the memory was allocated through malloc. It also + restores `contents' and `size' to their original, pre-malloc + values. That way after POOL_FREE, the pool is fully usable, just + as if it were freshly initialized with POOL_INIT. */ + +#define POOL_FREE(p) do { \ + struct pool *P = p; \ + if (P->resized) \ + xfree (P->contents); \ + P->contents = P->orig_contents; \ + P->size = P->orig_size; \ + P->tail = 0; \ + P->resized = false; \ +} while (0) + +/* Used for small stack-allocated memory chunks that might grow. Like + DO_REALLOC, this macro grows BASEVAR as necessary to take + NEEDED_SIZE items of TYPE. + + The difference is that on the first resize, it will use + malloc+memcpy rather than realloc. That way you can stack-allocate + the initial chunk, and only resort to heap allocation if you + stumble upon large data. + + After the first resize, subsequent ones are performed with realloc, + just like DO_REALLOC. */ + +#define GROW_ARRAY(basevar, sizevar, needed_size, resized, type) do { \ + long ga_needed_size = (needed_size); \ + long ga_newsize = (sizevar); \ + while (ga_newsize < ga_needed_size) \ + ga_newsize <<= 1; \ + if (ga_newsize != (sizevar)) \ + { \ + if (resized) \ + basevar = xrealloc (basevar, ga_newsize * sizeof (type)); \ + else \ + { \ + void *ga_new = xmalloc (ga_newsize * sizeof (type)); \ + memcpy (ga_new, basevar, (sizevar) * sizeof (type)); \ + (basevar) = ga_new; \ + resized = true; \ + } \ + (sizevar) = ga_newsize; \ + } \ +} while (0) + +/* Test whether n+1-sized entity name fits in P. We don't support + IE-style non-terminated entities, e.g. "<foo" -> "<foo". + However, "<foo" will work, as will "<!foo", "<", etc. In + other words an entity needs to be terminated by either a + non-alphanumeric or the end of string. */ +#define FITS(p, n) (p + n == end || (p + n < end && !c_isalnum (p[n]))) + +/* Macros that test entity names by returning true if P is followed by + the specified characters. */ +#define ENT1(p, c0) (FITS (p, 1) && p[0] == c0) +#define ENT2(p, c0, c1) (FITS (p, 2) && p[0] == c0 && p[1] == c1) +#define ENT3(p, c0, c1, c2) (FITS (p, 3) && p[0]==c0 && p[1]==c1 && p[2]==c2) + +/* Increment P by INC chars. If P lands at a semicolon, increment it + past the semicolon. This ensures that e.g. "<foo" is converted + to "<foo", but "<,foo" to "<,foo". */ +#define SKIP_SEMI(p, inc) (p += inc, p < end && *p == ';' ? ++p : p) + +struct tagstack_item { + const char *tagname_begin; + const char *tagname_end; + const char *contents_begin; + struct tagstack_item *prev; + struct tagstack_item *next; +}; + +static struct tagstack_item * +tagstack_push (struct tagstack_item **head, struct tagstack_item **tail) +{ + struct tagstack_item *ts = xmalloc(sizeof(struct tagstack_item)); + if (*head == NULL) + { + *head = *tail = ts; + ts->prev = ts->next = NULL; + } + else + { + (*tail)->next = ts; + ts->prev = *tail; + *tail = ts; + ts->next = NULL; + } + + return ts; +} + +/* remove ts and everything after it from the stack */ +static void +tagstack_pop (struct tagstack_item **head, struct tagstack_item **tail, + struct tagstack_item *ts) +{ + if (*head == NULL) + return; + + if (ts == *tail) + { + if (ts == *head) + { + xfree (ts); + *head = *tail = NULL; + } + else + { + ts->prev->next = NULL; + *tail = ts->prev; + xfree (ts); + } + } + else + { + if (ts == *head) + { + *head = NULL; + } + *tail = ts->prev; + + if (ts->prev) + { + ts->prev->next = NULL; + } + while (ts) + { + struct tagstack_item *p = ts->next; + xfree (ts); + ts = p; + } + } +} + +static struct tagstack_item * +tagstack_find (struct tagstack_item *tail, const char *tagname_begin, + const char *tagname_end) +{ + int len = tagname_end - tagname_begin; + while (tail) + { + if (len == (tail->tagname_end - tail->tagname_begin)) + { + if (0 == strncasecmp (tail->tagname_begin, tagname_begin, len)) + return tail; + } + tail = tail->prev; + } + return NULL; +} + +/* Decode the HTML character entity at *PTR, considering END to be end + of buffer. It is assumed that the "&" character that marks the + beginning of the entity has been seen at *PTR-1. If a recognized + ASCII entity is seen, it is returned, and *PTR is moved to the end + of the entity. Otherwise, -1 is returned and *PTR left unmodified. + + The recognized entities are: <, >, &, &apos, and ". */ + +static int +decode_entity (const char **ptr, const char *end) +{ + const char *p = *ptr; + int value = -1; + + if (++p == end) + return -1; + + switch (*p++) + { + case '#': + /* Process numeric entities "&#DDD;" and "&#xHH;". */ + { + int digits = 0; + value = 0; + if (*p == 'x') + for (++p; value < 256 && p < end && c_isxdigit (*p); p++, digits++) + value = (value << 4) + XDIGIT_TO_NUM (*p); + else + for (; value < 256 && p < end && c_isdigit (*p); p++, digits++) + value = (value * 10) + (*p - '0'); + if (!digits) + return -1; + /* Don't interpret 128+ codes and NUL because we cannot + portably reinserted them into HTML. */ + if (!value || (value & ~0x7f)) + return -1; + *ptr = SKIP_SEMI (p, 0); + return value; + } + /* Process named ASCII entities. */ + case 'g': + if (ENT1 (p, 't')) + value = '>', *ptr = SKIP_SEMI (p, 1); + break; + case 'l': + if (ENT1 (p, 't')) + value = '<', *ptr = SKIP_SEMI (p, 1); + break; + case 'a': + if (ENT2 (p, 'm', 'p')) + value = '&', *ptr = SKIP_SEMI (p, 2); + else if (ENT3 (p, 'p', 'o', 's')) + /* handle &apos for the sake of the XML/XHTML crowd. */ + value = '\'', *ptr = SKIP_SEMI (p, 3); + break; + case 'q': + if (ENT3 (p, 'u', 'o', 't')) + value = '\"', *ptr = SKIP_SEMI (p, 3); + break; + } + return value; +} +#undef ENT1 +#undef ENT2 +#undef ENT3 +#undef FITS +#undef SKIP_SEMI + +enum { + AP_DOWNCASE = 1, + AP_DECODE_ENTITIES = 2, + AP_TRIM_BLANKS = 4 +}; + +/* Copy the text in the range [BEG, END) to POOL, optionally + performing operations specified by FLAGS. FLAGS may be any + combination of AP_DOWNCASE, AP_DECODE_ENTITIES and AP_TRIM_BLANKS + with the following meaning: + + * AP_DOWNCASE -- downcase all the letters; + + * AP_DECODE_ENTITIES -- decode the named and numeric entities in + the ASCII range when copying the string. + + * AP_TRIM_BLANKS -- ignore blanks at the beginning and at the end + of text, as well as embedded newlines. */ + +static void +convert_and_copy (struct pool *pool, const char *beg, const char *end, int flags) +{ + int old_tail = pool->tail; + + /* Skip blanks if required. We must do this before entities are + processed, so that blanks can still be inserted as, for instance, + ` '. */ + if (flags & AP_TRIM_BLANKS) + { + while (beg < end && c_isspace (*beg)) + ++beg; + while (end > beg && c_isspace (end[-1])) + --end; + } + + if (flags & AP_DECODE_ENTITIES) + { + /* Grow the pool, then copy the text to the pool character by + character, processing the encountered entities as we go + along. + + It's safe (and necessary) to grow the pool in advance because + processing the entities can only *shorten* the string, it can + never lengthen it. */ + const char *from = beg; + char *to; + bool squash_newlines = !!(flags & AP_TRIM_BLANKS); + + POOL_GROW (pool, end - beg); + to = pool->contents + pool->tail; + + while (from < end) + { + if (*from == '&') + { + int entity = decode_entity (&from, end); + if (entity != -1) + *to++ = entity; + else + *to++ = *from++; + } + else if ((*from == '\n' || *from == '\r') && squash_newlines) + ++from; + else + *to++ = *from++; + } + /* Verify that we haven't exceeded the original size. (It + shouldn't happen, hence the assert.) */ + assert (to - (pool->contents + pool->tail) <= end - beg); + + /* Make POOL's tail point to the position following the string + we've written. */ + pool->tail = to - pool->contents; + POOL_APPEND_CHR (pool, '\0'); + } + else + { + /* Just copy the text to the pool. */ + POOL_APPEND (pool, beg, end); + POOL_APPEND_CHR (pool, '\0'); + } + + if (flags & AP_DOWNCASE) + { + char *p = pool->contents + old_tail; + for (; *p; p++) + *p = c_tolower (*p); + } +} + +/* Originally we used to adhere to rfc 1866 here, and allowed only + letters, digits, periods, and hyphens as names (of tags or + attributes). However, this broke too many pages which used + proprietary or strange attributes, e.g. <img src="a.gif" + v:shapes="whatever">. + + So now we allow any character except: + * whitespace + * 8-bit and control chars + * characters that clearly cannot be part of name: + '=', '<', '>', '/'. + + This only affects attribute and tag names; attribute values allow + an even greater variety of characters. */ + +#define NAME_CHAR_P(x) ((x) > 32 && (x) < 127 \ + && (x) != '=' && (x) != '<' && (x) != '>' \ + && (x) != '/') + +#ifdef STANDALONE +static int comment_backout_count; +#endif + +/* Advance over an SGML declaration, such as <!DOCTYPE ...>. In + strict comments mode, this is used for skipping over comments as + well. + + To recap: any SGML declaration may have comments associated with + it, e.g. + <!MY-DECL -- isn't this fun? -- foo bar> + + An HTML comment is merely an empty declaration (<!>) with a comment + attached, like this: + <!-- some stuff here --> + + Several comments may be embedded in one comment declaration: + <!-- have -- -- fun --> + + Whitespace is allowed between and after the comments, but not + before the first comment. Additionally, this function attempts to + handle double quotes in SGML declarations correctly. */ + +static const char * +advance_declaration (const char *beg, const char *end) +{ + const char *p = beg; + char quote_char = '\0'; /* shut up, gcc! */ + char ch; + + enum { + AC_S_DONE, + AC_S_BACKOUT, + AC_S_BANG, + AC_S_DEFAULT, + AC_S_DCLNAME, + AC_S_DASH1, + AC_S_DASH2, + AC_S_COMMENT, + AC_S_DASH3, + AC_S_DASH4, + AC_S_QUOTE1, + AC_S_IN_QUOTE, + AC_S_QUOTE2 + } state = AC_S_BANG; + + if (beg == end) + return beg; + ch = *p++; + + /* It looked like a good idea to write this as a state machine, but + now I wonder... */ + + while (state != AC_S_DONE && state != AC_S_BACKOUT) + { + if (p == end) + state = AC_S_BACKOUT; + switch (state) + { + case AC_S_DONE: + case AC_S_BACKOUT: + break; + case AC_S_BANG: + if (ch == '!') + { + ch = *p++; + state = AC_S_DEFAULT; + } + else + state = AC_S_BACKOUT; + break; + case AC_S_DEFAULT: + switch (ch) + { + case '-': + state = AC_S_DASH1; + break; + case ' ': + case '\t': + case '\r': + case '\n': + ch = *p++; + break; + case '<': + case '>': + state = AC_S_DONE; + break; + case '\'': + case '\"': + state = AC_S_QUOTE1; + break; + default: + if (NAME_CHAR_P (ch)) + state = AC_S_DCLNAME; + else + state = AC_S_BACKOUT; + break; + } + break; + case AC_S_DCLNAME: + if (ch == '-') + state = AC_S_DASH1; + else if (NAME_CHAR_P (ch)) + ch = *p++; + else + state = AC_S_DEFAULT; + break; + case AC_S_QUOTE1: + /* We must use 0x22 because broken assert macros choke on + '"' and '\"'. */ + assert (ch == '\'' || ch == 0x22); + quote_char = ch; /* cheating -- I really don't feel like + introducing more different states for + different quote characters. */ + ch = *p++; + state = AC_S_IN_QUOTE; + break; + case AC_S_IN_QUOTE: + if (ch == quote_char) + state = AC_S_QUOTE2; + else + ch = *p++; + break; + case AC_S_QUOTE2: + assert (ch == quote_char); + ch = *p++; + state = AC_S_DEFAULT; + break; + case AC_S_DASH1: + assert (ch == '-'); + ch = *p++; + state = AC_S_DASH2; + break; + case AC_S_DASH2: + switch (ch) + { + case '-': + ch = *p++; + state = AC_S_COMMENT; + break; + default: + state = AC_S_BACKOUT; + } + break; + case AC_S_COMMENT: + switch (ch) + { + case '-': + state = AC_S_DASH3; + break; + default: + ch = *p++; + break; + } + break; + case AC_S_DASH3: + assert (ch == '-'); + ch = *p++; + state = AC_S_DASH4; + break; + case AC_S_DASH4: + switch (ch) + { + case '-': + ch = *p++; + state = AC_S_DEFAULT; + break; + default: + state = AC_S_COMMENT; + break; + } + break; + } + } + + if (state == AC_S_BACKOUT) + { +#ifdef STANDALONE + ++comment_backout_count; +#endif + return beg + 1; + } + return p; +} + +/* Find the first occurrence of the substring "-->" in [BEG, END) and + return the pointer to the character after the substring. If the + substring is not found, return NULL. */ + +static const char * +find_comment_end (const char *beg, const char *end) +{ + /* Open-coded Boyer-Moore search for "-->". Examine the third char; + if it's not '>' or '-', advance by three characters. Otherwise, + look at the preceding characters and try to find a match. */ + + const char *p = beg - 1; + + while ((p += 3) < end) + switch (p[0]) + { + case '>': + if (p[-1] == '-' && p[-2] == '-') + return p + 1; + break; + case '-': + at_dash: + if (p[-1] == '-') + { + at_dash_dash: + if (++p == end) return NULL; + switch (p[0]) + { + case '>': return p + 1; + case '-': goto at_dash_dash; + } + } + else + { + if ((p += 2) >= end) return NULL; + switch (p[0]) + { + case '>': + if (p[-1] == '-') + return p + 1; + break; + case '-': + goto at_dash; + } + } + } + return NULL; +} + +/* Return true if the string containing of characters inside [b, e) is + present in hash table HT. */ + +static bool +name_allowed (const struct hash_table *ht, const char *b, const char *e) +{ + char *copy; + if (!ht) + return true; + BOUNDED_TO_ALLOCA (b, e, copy); + return hash_table_get (ht, copy) != NULL; +} + +/* Advance P (a char pointer), with the explicit intent of being able + to read the next character. If this is not possible, go to finish. */ + +#define ADVANCE(p) do { \ + ++p; \ + if (p >= end) \ + goto finish; \ +} while (0) + +/* Skip whitespace, if any. */ + +#define SKIP_WS(p) do { \ + while (c_isspace (*p)) { \ + ADVANCE (p); \ + } \ +} while (0) + +#ifdef STANDALONE +static int tag_backout_count; +#endif + +/* Map MAPFUN over HTML tags in TEXT, which is SIZE characters long. + MAPFUN will be called with two arguments: pointer to an initialized + struct taginfo, and MAPARG. + + ALLOWED_TAGS and ALLOWED_ATTRIBUTES are hash tables the keys of + which are the tags and attribute names that this function should + use. If ALLOWED_TAGS is NULL, all tags are processed; if + ALLOWED_ATTRIBUTES is NULL, all attributes are returned. + + (Obviously, the caller can filter out unwanted tags and attributes + just as well, but this is just an optimization designed to avoid + unnecessary copying of tags/attributes which the caller doesn't + care about.) */ + +void +map_html_tags (const char *text, int size, + void (*mapfun) (struct taginfo *, void *), void *maparg, + int flags, + const struct hash_table *allowed_tags, + const struct hash_table *allowed_attributes) +{ + /* storage for strings passed to MAPFUN callback; if 256 bytes is + too little, POOL_APPEND allocates more with malloc. */ + char pool_initial_storage[256]; + struct pool pool; + + const char *p = text; + const char *end = text + size; + + struct attr_pair attr_pair_initial_storage[8]; + int attr_pair_size = countof (attr_pair_initial_storage); + bool attr_pair_resized = false; + struct attr_pair *pairs = attr_pair_initial_storage; + + struct tagstack_item *head = NULL; + struct tagstack_item *tail = NULL; + + if (!size) + return; + + POOL_INIT (&pool, pool_initial_storage, countof (pool_initial_storage)); + + { + int nattrs, end_tag; + const char *tag_name_begin, *tag_name_end; + const char *tag_start_position; + bool uninteresting_tag; + + look_for_tag: + POOL_REWIND (&pool); + + nattrs = 0; + end_tag = 0; + + /* Find beginning of tag. We use memchr() instead of the usual + looping with ADVANCE() for speed. */ + p = memchr (p, '<', end - p); + if (!p) + goto finish; + + tag_start_position = p; + ADVANCE (p); + + /* Establish the type of the tag (start-tag, end-tag or + declaration). */ + if (*p == '!') + { + if (!(flags & MHT_STRICT_COMMENTS) + && p + 3 < end && p[1] == '-' && p[2] == '-') + { + /* If strict comments are not enforced and if we know + we're looking at a comment, simply look for the + terminating "-->". Non-strict is the default because + it works in other browsers and most HTML writers can't + be bothered with getting the comments right. */ + const char *comment_end = find_comment_end (p + 3, end); + if (comment_end) + p = comment_end; + } + else + { + /* Either in strict comment mode or looking at a non-empty + declaration. Real declarations are much less likely to + be misused the way comments are, so advance over them + properly regardless of strictness. */ + p = advance_declaration (p, end); + } + if (p == end) + goto finish; + goto look_for_tag; + } + else if (*p == '/') + { + end_tag = 1; + ADVANCE (p); + } + tag_name_begin = p; + while (NAME_CHAR_P (*p)) + ADVANCE (p); + if (p == tag_name_begin) + goto look_for_tag; + tag_name_end = p; + SKIP_WS (p); + + if (!end_tag) + { + struct tagstack_item *ts = tagstack_push (&head, &tail); + if (ts) + { + ts->tagname_begin = tag_name_begin; + ts->tagname_end = tag_name_end; + ts->contents_begin = NULL; + } + } + + if (end_tag && *p != '>' && *p != '<') + goto backout_tag; + + if (!name_allowed (allowed_tags, tag_name_begin, tag_name_end)) + /* We can't just say "goto look_for_tag" here because we need + the loop below to properly advance over the tag's attributes. */ + uninteresting_tag = true; + else + { + uninteresting_tag = false; + convert_and_copy (&pool, tag_name_begin, tag_name_end, AP_DOWNCASE); + } + + /* Find the attributes. */ + while (1) + { + const char *attr_name_begin, *attr_name_end; + const char *attr_value_begin, *attr_value_end; + const char *attr_raw_value_begin, *attr_raw_value_end; + int operation = AP_DOWNCASE; /* stupid compiler. */ + + SKIP_WS (p); + + if (*p == '/') + { + /* A slash at this point means the tag is about to be + closed. This is legal in XML and has been popularized + in HTML via XHTML. */ + /* <foo a=b c=d /> */ + /* ^ */ + ADVANCE (p); + SKIP_WS (p); + if (*p != '<' && *p != '>') + goto backout_tag; + } + + /* Check for end of tag definition. */ + if (*p == '<' || *p == '>') + break; + + /* Establish bounds of attribute name. */ + attr_name_begin = p; /* <foo bar ...> */ + /* ^ */ + while (NAME_CHAR_P (*p)) + ADVANCE (p); + attr_name_end = p; /* <foo bar ...> */ + /* ^ */ + if (attr_name_begin == attr_name_end) + goto backout_tag; + + /* Establish bounds of attribute value. */ + SKIP_WS (p); + + if (NAME_CHAR_P (*p) || *p == '/' || *p == '<' || *p == '>') + { + /* Minimized attribute syntax allows `=' to be omitted. + For example, <UL COMPACT> is a valid shorthand for <UL + COMPACT="compact">. Even if such attributes are not + useful to Wget, we need to support them, so that the + tags containing them can be parsed correctly. */ + attr_raw_value_begin = attr_value_begin = attr_name_begin; + attr_raw_value_end = attr_value_end = attr_name_end; + } + else if (*p == '=') + { + ADVANCE (p); + SKIP_WS (p); + if (*p == '\"' || *p == '\'') + { + bool newline_seen = false; + char quote_char = *p; + attr_raw_value_begin = p; + ADVANCE (p); + attr_value_begin = p; /* <foo bar="baz"> */ + /* ^ */ + while (*p != quote_char) + { + if (!newline_seen && *p == '\n') + { + /* If a newline is seen within the quotes, it + is most likely that someone forgot to close + the quote. In that case, we back out to + the value beginning, and terminate the tag + at either `>' or the delimiter, whichever + comes first. Such a tag terminated at `>' + is discarded. */ + p = attr_value_begin; + newline_seen = true; + continue; + } + else if (newline_seen && (*p == '<' || *p == '>')) + break; + ADVANCE (p); + } + attr_value_end = p; /* <foo bar="baz"> */ + /* ^ */ + if (*p == quote_char) + ADVANCE (p); + else + goto look_for_tag; + attr_raw_value_end = p; /* <foo bar="baz"> */ + /* ^ */ + operation = AP_DECODE_ENTITIES; + if (flags & MHT_TRIM_VALUES) + operation |= AP_TRIM_BLANKS; + } + else + { + attr_value_begin = p; /* <foo bar=baz> */ + /* ^ */ + /* According to SGML, a name token should consist only + of alphanumerics, . and -. However, this is often + violated by, for instance, `%' in `width=75%'. + We'll be liberal and allow just about anything as + an attribute value. */ + while (!c_isspace (*p) && *p != '<' && *p != '>') + ADVANCE (p); + attr_value_end = p; /* <foo bar=baz qux=quix> */ + /* ^ */ + if (attr_value_begin == attr_value_end) + /* <foo bar=> */ + /* ^ */ + goto backout_tag; + attr_raw_value_begin = attr_value_begin; + attr_raw_value_end = attr_value_end; + operation = AP_DECODE_ENTITIES; + } + } + else + { + /* We skipped the whitespace and found something that is + neither `=' nor the beginning of the next attribute's + name. Back out. */ + goto backout_tag; /* <foo bar [... */ + /* ^ */ + } + + /* If we're not interested in the tag, don't bother with any + of the attributes. */ + if (uninteresting_tag) + continue; + + /* If we aren't interested in the attribute, skip it. We + cannot do this test any sooner, because our text pointer + needs to correctly advance over the attribute. */ + if (!name_allowed (allowed_attributes, attr_name_begin, attr_name_end)) + continue; + + GROW_ARRAY (pairs, attr_pair_size, nattrs + 1, attr_pair_resized, + struct attr_pair); + + pairs[nattrs].name_pool_index = pool.tail; + convert_and_copy (&pool, attr_name_begin, attr_name_end, AP_DOWNCASE); + + pairs[nattrs].value_pool_index = pool.tail; + convert_and_copy (&pool, attr_value_begin, attr_value_end, operation); + pairs[nattrs].value_raw_beginning = attr_raw_value_begin; + pairs[nattrs].value_raw_size = (attr_raw_value_end + - attr_raw_value_begin); + ++nattrs; + } + + if (!end_tag && tail && (tail->tagname_begin == tag_name_begin)) + { + tail->contents_begin = p+1; + } + + if (uninteresting_tag) + { + ADVANCE (p); + goto look_for_tag; + } + + /* By now, we have a valid tag with a name and zero or more + attributes. Fill in the data and call the mapper function. */ + { + int i; + struct taginfo taginfo; + struct tagstack_item *ts = NULL; + + taginfo.name = pool.contents; + taginfo.end_tag_p = end_tag; + taginfo.nattrs = nattrs; + /* We fill in the char pointers only now, when pool can no + longer get realloc'ed. If we did that above, we could get + hosed by reallocation. Obviously, after this point, the pool + may no longer be grown. */ + for (i = 0; i < nattrs; i++) + { + pairs[i].name = pool.contents + pairs[i].name_pool_index; + pairs[i].value = pool.contents + pairs[i].value_pool_index; + } + taginfo.attrs = pairs; + taginfo.start_position = tag_start_position; + taginfo.end_position = p + 1; + taginfo.contents_begin = NULL; + taginfo.contents_end = NULL; + + if (end_tag) + { + ts = tagstack_find (tail, tag_name_begin, tag_name_end); + if (ts) + { + if (ts->contents_begin) + { + taginfo.contents_begin = ts->contents_begin; + taginfo.contents_end = tag_start_position; + } + tagstack_pop (&head, &tail, ts); + } + } + + mapfun (&taginfo, maparg); + if (*p != '<') + ADVANCE (p); + } + goto look_for_tag; + + backout_tag: +#ifdef STANDALONE + ++tag_backout_count; +#endif + /* The tag wasn't really a tag. Treat its contents as ordinary + data characters. */ + p = tag_start_position + 1; + goto look_for_tag; + } + + finish: + POOL_FREE (&pool); + if (attr_pair_resized) + xfree (pairs); + /* pop any tag stack that's left */ + tagstack_pop (&head, &tail, head); +} + +#undef ADVANCE +#undef SKIP_WS +#undef SKIP_NON_WS + +#ifdef STANDALONE +static void +test_mapper (struct taginfo *taginfo, void *arg) +{ + int i; + + printf ("%s%s", taginfo->end_tag_p ? "/" : "", taginfo->name); + for (i = 0; i < taginfo->nattrs; i++) + printf (" %s=%s", taginfo->attrs[i].name, taginfo->attrs[i].value); + putchar ('\n'); + ++*(int *)arg; +} + +int main () +{ + int size = 256; + char *x = xmalloc (size); + int length = 0; + int read_count; + int tag_counter = 0; + +#ifdef ENABLE_NLS + /* Set the current locale. */ + setlocale (LC_ALL, ""); + /* Set the text message domain. */ + bindtextdomain ("wget", LOCALEDIR); + textdomain ("wget"); +#endif /* ENABLE_NLS */ + + while ((read_count = fread (x + length, 1, size - length, stdin))) + { + length += read_count; + size <<= 1; + x = xrealloc (x, size); + } + + map_html_tags (x, length, test_mapper, &tag_counter, 0, NULL, NULL); + printf ("TAGS: %d\n", tag_counter); + printf ("Tag backouts: %d\n", tag_backout_count); + printf ("Comment backouts: %d\n", comment_backout_count); + return 0; +} +#endif /* STANDALONE */ |