summaryrefslogtreecommitdiffstats
path: root/src/url.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/url.c')
-rw-r--r--src/url.c2523
1 files changed, 2523 insertions, 0 deletions
diff --git a/src/url.c b/src/url.c
new file mode 100644
index 0000000..4b2263c
--- /dev/null
+++ b/src/url.c
@@ -0,0 +1,2523 @@
+/* URL handling.
+ Copyright (C) 1996-2011, 2015, 2018 Free Software Foundation, Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or (at
+your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
+
+#include "wget.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <assert.h>
+
+#include "utils.h"
+#include "url.h"
+#include "host.h" /* for is_valid_ipv6_address */
+#include "c-strcase.h"
+
+#ifdef HAVE_ICONV
+# include <iconv.h>
+#endif
+#include <langinfo.h>
+
+#ifdef __VMS
+#include "vms.h"
+#endif /* def __VMS */
+
+#ifdef TESTING
+#include "../tests/unit-tests.h"
+#endif
+
+enum {
+ scm_disabled = 1, /* for https when OpenSSL fails to init. */
+ scm_has_params = 2, /* whether scheme has ;params */
+ scm_has_query = 4, /* whether scheme has ?query */
+ scm_has_fragment = 8 /* whether scheme has #fragment */
+};
+
+struct scheme_data
+{
+ /* Short name of the scheme, such as "http" or "ftp". */
+ const char *name;
+ /* Leading string that identifies the scheme, such as "https://". */
+ const char *leading_string;
+ /* Default port of the scheme when none is specified. */
+ int default_port;
+ /* Various flags. */
+ int flags;
+};
+
+/* Supported schemes: */
+static struct scheme_data supported_schemes[] =
+{
+ { "http", "http://", DEFAULT_HTTP_PORT, scm_has_query|scm_has_fragment },
+#ifdef HAVE_SSL
+ { "https", "https://", DEFAULT_HTTPS_PORT, scm_has_query|scm_has_fragment },
+#endif
+ { "ftp", "ftp://", DEFAULT_FTP_PORT, scm_has_params|scm_has_fragment },
+#ifdef HAVE_SSL
+ /*
+ * Explicit FTPS uses the same port as FTP.
+ * Implicit FTPS has its own port (990), but it is disabled by default.
+ */
+ { "ftps", "ftps://", DEFAULT_FTP_PORT, scm_has_params|scm_has_fragment },
+#endif
+
+ /* SCHEME_INVALID */
+ { NULL, NULL, -1, 0 }
+};
+
+/* Forward declarations: */
+
+static bool path_simplify (enum url_scheme, char *);
+
+/* Support for escaping and unescaping of URL strings. */
+
+/* Table of "reserved" and "unsafe" characters. Those terms are
+ rfc1738-speak, as such largely obsoleted by rfc2396 and later
+ specs, but the general idea remains.
+
+ A reserved character is the one that you can't decode without
+ changing the meaning of the URL. For example, you can't decode
+ "/foo/%2f/bar" into "/foo///bar" because the number and contents of
+ path components is different. Non-reserved characters can be
+ changed, so "/foo/%78/bar" is safe to change to "/foo/x/bar". The
+ unsafe characters are loosely based on rfc1738, plus "$" and ",",
+ as recommended by rfc2396, and minus "~", which is very frequently
+ used (and sometimes unrecognized as %7E by broken servers).
+
+ An unsafe character is the one that should be encoded when URLs are
+ placed in foreign environments. E.g. space and newline are unsafe
+ in HTTP contexts because HTTP uses them as separator and line
+ terminator, so they must be encoded to %20 and %0A respectively.
+ "*" is unsafe in shell context, etc.
+
+ We determine whether a character is unsafe through static table
+ lookup. This code assumes ASCII character set and 8-bit chars. */
+
+enum {
+ /* rfc1738 reserved chars + "$" and ",". */
+ urlchr_reserved = 1,
+
+ /* rfc1738 unsafe chars, plus non-printables. */
+ urlchr_unsafe = 2
+};
+
+#define urlchr_test(c, mask) (urlchr_table[(unsigned char)(c)] & (mask))
+#define URL_RESERVED_CHAR(c) urlchr_test(c, urlchr_reserved)
+#define URL_UNSAFE_CHAR(c) urlchr_test(c, urlchr_unsafe)
+
+/* Shorthands for the table: */
+#define R urlchr_reserved
+#define U urlchr_unsafe
+#define RU R|U
+
+static const unsigned char urlchr_table[256] =
+{
+ U, U, U, U, U, U, U, U, /* NUL SOH STX ETX EOT ENQ ACK BEL */
+ U, U, U, U, U, U, U, U, /* BS HT LF VT FF CR SO SI */
+ U, U, U, U, U, U, U, U, /* DLE DC1 DC2 DC3 DC4 NAK SYN ETB */
+ U, U, U, U, U, U, U, U, /* CAN EM SUB ESC FS GS RS US */
+ U, 0, U, RU, R, U, R, 0, /* SP ! " # $ % & ' */
+ 0, 0, 0, R, R, 0, 0, R, /* ( ) * + , - . / */
+ 0, 0, 0, 0, 0, 0, 0, 0, /* 0 1 2 3 4 5 6 7 */
+ 0, 0, RU, R, U, R, U, R, /* 8 9 : ; < = > ? */
+ RU, 0, 0, 0, 0, 0, 0, 0, /* @ A B C D E F G */
+ 0, 0, 0, 0, 0, 0, 0, 0, /* H I J K L M N O */
+ 0, 0, 0, 0, 0, 0, 0, 0, /* P Q R S T U V W */
+ 0, 0, 0, RU, U, RU, U, 0, /* X Y Z [ \ ] ^ _ */
+ U, 0, 0, 0, 0, 0, 0, 0, /* ` a b c d e f g */
+ 0, 0, 0, 0, 0, 0, 0, 0, /* h i j k l m n o */
+ 0, 0, 0, 0, 0, 0, 0, 0, /* p q r s t u v w */
+ 0, 0, 0, U, U, U, 0, U, /* x y z { | } ~ DEL */
+
+ U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
+ U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
+ U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
+ U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
+
+ U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
+ U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
+ U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
+ U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U,
+};
+#undef R
+#undef U
+#undef RU
+
+static void
+url_unescape_1 (char *s, unsigned char mask)
+{
+ char *t = s; /* t - tortoise */
+ char *h = s; /* h - hare */
+
+ for (; *h; h++, t++)
+ {
+ if (*h != '%')
+ {
+ copychar:
+ *t = *h;
+ }
+ else
+ {
+ char c;
+ /* Do nothing if '%' is not followed by two hex digits. */
+ if (!h[1] || !h[2] || !(c_isxdigit (h[1]) && c_isxdigit (h[2])))
+ goto copychar;
+ c = X2DIGITS_TO_NUM (h[1], h[2]);
+ if (urlchr_test(c, mask))
+ goto copychar;
+ /* Don't unescape %00 because there is no way to insert it
+ into a C string without effectively truncating it. */
+ if (c == '\0')
+ goto copychar;
+ *t = c;
+ h += 2;
+ }
+ }
+ *t = '\0';
+}
+
+/* URL-unescape the string S.
+
+ This is done by transforming the sequences "%HH" to the character
+ represented by the hexadecimal digits HH. If % is not followed by
+ two hexadecimal digits, it is inserted literally.
+
+ The transformation is done in place. If you need the original
+ string intact, make a copy before calling this function. */
+void
+url_unescape (char *s)
+{
+ url_unescape_1 (s, 0);
+}
+
+/* URL-unescape the string S.
+
+ This functions behaves identically as url_unescape(), but does not
+ convert characters from "reserved". In other words, it only converts
+ "unsafe" characters. */
+void
+url_unescape_except_reserved (char *s)
+{
+ url_unescape_1 (s, urlchr_reserved);
+}
+
+/* The core of url_escape_* functions. Escapes the characters that
+ match the provided mask in urlchr_table.
+
+ If ALLOW_PASSTHROUGH is true, a string with no unsafe chars will be
+ returned unchanged. If ALLOW_PASSTHROUGH is false, a freshly
+ allocated string will be returned in all cases. */
+
+static char *
+url_escape_1 (const char *s, unsigned char mask, bool allow_passthrough)
+{
+ const char *p1;
+ char *p2, *newstr;
+ int newlen;
+ int addition = 0;
+
+ for (p1 = s; *p1; p1++)
+ if (urlchr_test (*p1, mask))
+ addition += 2; /* Two more characters (hex digits) */
+
+ if (!addition)
+ return allow_passthrough ? (char *)s : xstrdup (s);
+
+ newlen = (p1 - s) + addition;
+ newstr = xmalloc (newlen + 1);
+
+ p1 = s;
+ p2 = newstr;
+ while (*p1)
+ {
+ /* Quote the characters that match the test mask. */
+ if (urlchr_test (*p1, mask))
+ {
+ unsigned char c = *p1++;
+ *p2++ = '%';
+ *p2++ = XNUM_TO_DIGIT (c >> 4);
+ *p2++ = XNUM_TO_DIGIT (c & 0xf);
+ }
+ else
+ *p2++ = *p1++;
+ }
+ assert (p2 - newstr == newlen);
+ *p2 = '\0';
+
+ return newstr;
+}
+
+/* URL-escape the unsafe characters (see urlchr_table) in a given
+ string, returning a freshly allocated string. */
+
+char *
+url_escape (const char *s)
+{
+ return url_escape_1 (s, urlchr_unsafe, false);
+}
+
+/* URL-escape the unsafe and reserved characters (see urlchr_table) in
+ a given string, returning a freshly allocated string. */
+
+char *
+url_escape_unsafe_and_reserved (const char *s)
+{
+ return url_escape_1 (s, urlchr_unsafe|urlchr_reserved, false);
+}
+
+/* URL-escape the unsafe characters (see urlchr_table) in a given
+ string. If no characters are unsafe, S is returned. */
+
+static char *
+url_escape_allow_passthrough (const char *s)
+{
+ return url_escape_1 (s, urlchr_unsafe, true);
+}
+
+/* Decide whether the char at position P needs to be encoded. (It is
+ not enough to pass a single char *P because the function may need
+ to inspect the surrounding context.)
+
+ Return true if the char should be escaped as %XX, false otherwise. */
+
+static inline bool
+char_needs_escaping (const char *p)
+{
+ if (*p == '%')
+ {
+ if (c_isxdigit (*(p + 1)) && c_isxdigit (*(p + 2)))
+ return false;
+ else
+ /* Garbled %.. sequence: encode `%'. */
+ return true;
+ }
+ else if (URL_UNSAFE_CHAR (*p) && !URL_RESERVED_CHAR (*p))
+ return true;
+ else
+ return false;
+}
+
+/* Translate a %-escaped (but possibly non-conformant) input string S
+ into a %-escaped (and conformant) output string. If no characters
+ are encoded or decoded, return the same string S; otherwise, return
+ a freshly allocated string with the new contents.
+
+ After a URL has been run through this function, the protocols that
+ use `%' as the quote character can use the resulting string as-is,
+ while those that don't can use url_unescape to get to the intended
+ data. This function is stable: once the input is transformed,
+ further transformations of the result yield the same output.
+
+ Let's discuss why this function is needed.
+
+ Imagine Wget is asked to retrieve `http://abc.xyz/abc def'. Since
+ a raw space character would mess up the HTTP request, it needs to
+ be quoted, like this:
+
+ GET /abc%20def HTTP/1.0
+
+ It would appear that the unsafe chars need to be quoted, for
+ example with url_escape. But what if we're requested to download
+ `abc%20def'? url_escape transforms "%" to "%25", which would leave
+ us with `abc%2520def'. This is incorrect -- since %-escapes are
+ part of URL syntax, "%20" is the correct way to denote a literal
+ space on the Wget command line. This leads to the conclusion that
+ in that case Wget should not call url_escape, but leave the `%20'
+ as is. This is clearly contradictory, but it only gets worse.
+
+ What if the requested URI is `abc%20 def'? If we call url_escape,
+ we end up with `/abc%2520%20def', which is almost certainly not
+ intended. If we don't call url_escape, we are left with the
+ embedded space and cannot complete the request. What the user
+ meant was for Wget to request `/abc%20%20def', and this is where
+ reencode_escapes kicks in.
+
+ Wget used to solve this by first decoding %-quotes, and then
+ encoding all the "unsafe" characters found in the resulting string.
+ This was wrong because it didn't preserve certain URL special
+ (reserved) characters. For instance, URI containing "a%2B+b" (0x2b
+ == '+') would get translated to "a%2B%2Bb" or "a++b" depending on
+ whether we considered `+' reserved (it is). One of these results
+ is inevitable because by the second step we would lose information
+ on whether the `+' was originally encoded or not. Both results
+ were wrong because in CGI parameters + means space, while %2B means
+ literal plus. reencode_escapes correctly translates the above to
+ "a%2B+b", i.e. returns the original string.
+
+ This function uses a modified version of the algorithm originally
+ proposed by Anon Sricharoenchai:
+
+ * Encode all "unsafe" characters, except those that are also
+ "reserved", to %XX. See urlchr_table for which characters are
+ unsafe and reserved.
+
+ * Encode the "%" characters not followed by two hex digits to
+ "%25".
+
+ * Pass through all other characters and %XX escapes as-is. (Up to
+ Wget 1.10 this decoded %XX escapes corresponding to "safe"
+ characters, but that was obtrusive and broke some servers.)
+
+ Anon's test case:
+
+ "http://abc.xyz/%20%3F%%36%31%25aa% a?a=%61+a%2Ba&b=b%26c%3Dc"
+ ->
+ "http://abc.xyz/%20%3F%25%36%31%25aa%25%20a?a=%61+a%2Ba&b=b%26c%3Dc"
+
+ Simpler test cases:
+
+ "foo bar" -> "foo%20bar"
+ "foo%20bar" -> "foo%20bar"
+ "foo %20bar" -> "foo%20%20bar"
+ "foo%%20bar" -> "foo%25%20bar" (0x25 == '%')
+ "foo%25%20bar" -> "foo%25%20bar"
+ "foo%2%20bar" -> "foo%252%20bar"
+ "foo+bar" -> "foo+bar" (plus is reserved!)
+ "foo%2b+bar" -> "foo%2b+bar" */
+
+static char *
+reencode_escapes (const char *s)
+{
+ const char *p1;
+ char *newstr, *p2;
+ int oldlen, newlen;
+
+ int encode_count = 0;
+
+ /* First pass: inspect the string to see if there's anything to do,
+ and to calculate the new length. */
+ for (p1 = s; *p1; p1++)
+ if (char_needs_escaping (p1))
+ ++encode_count;
+
+ if (!encode_count)
+ /* The string is good as it is. */
+ return (char *) s; /* C const model sucks. */
+
+ oldlen = p1 - s;
+ /* Each encoding adds two characters (hex digits). */
+ newlen = oldlen + 2 * encode_count;
+ newstr = xmalloc (newlen + 1);
+
+ /* Second pass: copy the string to the destination address, encoding
+ chars when needed. */
+ p1 = s;
+ p2 = newstr;
+
+ while (*p1)
+ if (char_needs_escaping (p1))
+ {
+ unsigned char c = *p1++;
+ *p2++ = '%';
+ *p2++ = XNUM_TO_DIGIT (c >> 4);
+ *p2++ = XNUM_TO_DIGIT (c & 0xf);
+ }
+ else
+ *p2++ = *p1++;
+
+ *p2 = '\0';
+ assert (p2 - newstr == newlen);
+ return newstr;
+}
+
+/* Returns the scheme type if the scheme is supported, or
+ SCHEME_INVALID if not. */
+
+enum url_scheme
+url_scheme (const char *url)
+{
+ int i;
+
+ for (i = 0; supported_schemes[i].leading_string; i++)
+ if (0 == c_strncasecmp (url, supported_schemes[i].leading_string,
+ strlen (supported_schemes[i].leading_string)))
+ {
+ if (!(supported_schemes[i].flags & scm_disabled))
+ return (enum url_scheme) i;
+ else
+ return SCHEME_INVALID;
+ }
+
+ return SCHEME_INVALID;
+}
+
+#define SCHEME_CHAR(ch) (c_isalnum (ch) || (ch) == '-' || (ch) == '+')
+
+/* Return 1 if the URL begins with any "scheme", 0 otherwise. As
+ currently implemented, it returns true if URL begins with
+ [-+a-zA-Z0-9]+: . */
+
+bool
+url_has_scheme (const char *url)
+{
+ const char *p = url;
+
+ /* The first char must be a scheme char. */
+ if (!*p || !SCHEME_CHAR (*p))
+ return false;
+ ++p;
+ /* Followed by 0 or more scheme chars. */
+ while (*p && SCHEME_CHAR (*p))
+ ++p;
+ /* Terminated by ':'. */
+ return *p == ':';
+}
+
+bool
+url_valid_scheme (const char *url)
+{
+ enum url_scheme scheme = url_scheme (url);
+ return scheme != SCHEME_INVALID;
+}
+
+int
+scheme_default_port (enum url_scheme scheme)
+{
+ return supported_schemes[scheme].default_port;
+}
+
+void
+scheme_disable (enum url_scheme scheme)
+{
+ supported_schemes[scheme].flags |= scm_disabled;
+}
+
+const char *
+scheme_leading_string (enum url_scheme scheme)
+{
+ return supported_schemes[scheme].leading_string;
+}
+
+/* Skip the username and password, if present in the URL. The
+ function should *not* be called with the complete URL, but with the
+ portion after the scheme.
+
+ If no username and password are found, return URL. */
+
+static const char *
+url_skip_credentials (const char *url)
+{
+ /* Look for '@' that comes before terminators, such as '/', '?',
+ '#', or ';'. */
+ const char *p = (const char *)strpbrk (url, "@/?#;");
+ if (!p || *p != '@')
+ return url;
+ return p + 1;
+}
+
+/* Parse credentials contained in [BEG, END). The region is expected
+ to have come from a URL and is unescaped. */
+
+static bool
+parse_credentials (const char *beg, const char *end, char **user, char **passwd)
+{
+ char *colon;
+ const char *userend;
+
+ if (beg == end)
+ return false; /* empty user name */
+
+ colon = memchr (beg, ':', end - beg);
+ if (colon == beg)
+ return false; /* again empty user name */
+
+ if (colon)
+ {
+ *passwd = strdupdelim (colon + 1, end);
+ userend = colon;
+ url_unescape (*passwd);
+ }
+ else
+ {
+ *passwd = NULL;
+ userend = end;
+ }
+ *user = strdupdelim (beg, userend);
+ url_unescape (*user);
+ return true;
+}
+
+/* Used by main.c: detect URLs written using the "shorthand" URL forms
+ originally popularized by Netscape and NcFTP. HTTP shorthands look
+ like this:
+
+ www.foo.com[:port]/dir/file -> http://www.foo.com[:port]/dir/file
+ www.foo.com[:port] -> http://www.foo.com[:port]
+
+ FTP shorthands look like this:
+
+ foo.bar.com:dir/file -> ftp://foo.bar.com/dir/file
+ foo.bar.com:/absdir/file -> ftp://foo.bar.com//absdir/file
+
+ If the URL needs not or cannot be rewritten, return NULL. */
+
+char *
+rewrite_shorthand_url (const char *url)
+{
+ const char *p;
+ char *ret;
+
+ if (url_scheme (url) != SCHEME_INVALID)
+ return NULL;
+
+ /* Look for a ':' or '/'. The former signifies NcFTP syntax, the
+ latter Netscape. */
+ p = strpbrk (url, ":/");
+ if (p == url)
+ return NULL;
+
+ /* If we're looking at "://", it means the URL uses a scheme we
+ don't support, which may include "https" when compiled without
+ SSL support. Don't bogusly rewrite such URLs. */
+ if (p && p[0] == ':' && p[1] == '/' && p[2] == '/')
+ return NULL;
+
+ if (p && *p == ':')
+ {
+ /* Colon indicates ftp, as in foo.bar.com:path. Check for
+ special case of http port number ("localhost:10000"). */
+ int digits = strspn (p + 1, "0123456789");
+ if (digits && (p[1 + digits] == '/' || p[1 + digits] == '\0'))
+ goto http;
+
+ /* Turn "foo.bar.com:path" to "ftp://foo.bar.com/path". */
+ if ((ret = aprintf ("ftp://%s", url)) != NULL)
+ ret[6 + (p - url)] = '/';
+ }
+ else
+ {
+ http:
+ /* Just prepend "http://" to URL. */
+ ret = aprintf ("http://%s", url);
+ }
+ return ret;
+}
+
+static void split_path (const char *, char **, char **);
+
+/* Like strpbrk, with the exception that it returns the pointer to the
+ terminating zero (end-of-string aka "eos") if no matching character
+ is found. */
+
+static inline char *
+strpbrk_or_eos (const char *s, const char *accept)
+{
+ char *p = strpbrk (s, accept);
+ if (!p)
+ p = strchr (s, '\0');
+ return p;
+}
+
+/* Turn STR into lowercase; return true if a character was actually
+ changed. */
+
+static bool
+lowercase_str (char *str)
+{
+ bool changed = false;
+ for (; *str; str++)
+ if (c_isupper (*str))
+ {
+ changed = true;
+ *str = c_tolower (*str);
+ }
+ return changed;
+}
+
+static const char *
+init_seps (enum url_scheme scheme)
+{
+ static char seps[8] = ":/";
+ char *p = seps + 2;
+ int flags = supported_schemes[scheme].flags;
+
+ if (flags & scm_has_params)
+ *p++ = ';';
+ if (flags & scm_has_query)
+ *p++ = '?';
+ if (flags & scm_has_fragment)
+ *p++ = '#';
+ *p = '\0';
+ return seps;
+}
+
+static const char *parse_errors[] = {
+#define PE_NO_ERROR 0
+ N_("No error"),
+#define PE_UNSUPPORTED_SCHEME 1
+ N_("Unsupported scheme %s"), /* support for format token only here */
+#define PE_MISSING_SCHEME 2
+ N_("Scheme missing"),
+#define PE_INVALID_HOST_NAME 3
+ N_("Invalid host name"),
+#define PE_BAD_PORT_NUMBER 4
+ N_("Bad port number"),
+#define PE_INVALID_USER_NAME 5
+ N_("Invalid user name"),
+#define PE_UNTERMINATED_IPV6_ADDRESS 6
+ N_("Unterminated IPv6 numeric address"),
+#define PE_IPV6_NOT_SUPPORTED 7
+ N_("IPv6 addresses not supported"),
+#define PE_INVALID_IPV6_ADDRESS 8
+ N_("Invalid IPv6 numeric address")
+};
+
+/* Parse a URL.
+
+ Return a new struct url if successful, NULL on error. In case of
+ error, and if ERROR is not NULL, also set *ERROR to the appropriate
+ error code. */
+struct url *
+url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
+{
+ struct url *u;
+ const char *p;
+ bool path_modified, host_modified;
+
+ enum url_scheme scheme;
+ const char *seps;
+
+ const char *uname_b, *uname_e;
+ const char *host_b, *host_e;
+ const char *path_b, *path_e;
+ const char *params_b, *params_e;
+ const char *query_b, *query_e;
+ const char *fragment_b, *fragment_e;
+
+ int port;
+ char *user = NULL, *passwd = NULL;
+
+ const char *url_encoded = NULL;
+
+ int error_code;
+
+ scheme = url_scheme (url);
+ if (scheme == SCHEME_INVALID)
+ {
+ if (url_has_scheme (url))
+ error_code = PE_UNSUPPORTED_SCHEME;
+ else
+ error_code = PE_MISSING_SCHEME;
+ goto error;
+ }
+
+ url_encoded = url;
+
+ if (iri && iri->utf8_encode)
+ {
+ char *new_url = NULL;
+
+ iri->utf8_encode = remote_to_utf8 (iri, iri->orig_url ? iri->orig_url : url, &new_url);
+ if (!iri->utf8_encode)
+ new_url = NULL;
+ else
+ {
+ xfree (iri->orig_url);
+ iri->orig_url = xstrdup (url);
+ url_encoded = reencode_escapes (new_url);
+ if (url_encoded != new_url)
+ xfree (new_url);
+ percent_encode = false;
+ }
+ }
+
+ if (percent_encode)
+ url_encoded = reencode_escapes (url);
+
+ p = url_encoded;
+ p += strlen (supported_schemes[scheme].leading_string);
+ uname_b = p;
+ p = url_skip_credentials (p);
+ uname_e = p;
+
+ /* scheme://user:pass@host[:port]... */
+ /* ^ */
+
+ /* We attempt to break down the URL into the components path,
+ params, query, and fragment. They are ordered like this:
+
+ scheme://host[:port][/path][;params][?query][#fragment] */
+
+ path_b = path_e = NULL;
+ params_b = params_e = NULL;
+ query_b = query_e = NULL;
+ fragment_b = fragment_e = NULL;
+
+ /* Initialize separators for optional parts of URL, depending on the
+ scheme. For example, FTP has params, and HTTP and HTTPS have
+ query string and fragment. */
+ seps = init_seps (scheme);
+
+ host_b = p;
+
+ if (*p == '[')
+ {
+ /* Handle IPv6 address inside square brackets. Ideally we'd
+ just look for the terminating ']', but rfc2732 mandates
+ rejecting invalid IPv6 addresses. */
+
+ /* The address begins after '['. */
+ host_b = p + 1;
+ host_e = strchr (host_b, ']');
+
+ if (!host_e)
+ {
+ error_code = PE_UNTERMINATED_IPV6_ADDRESS;
+ goto error;
+ }
+
+#ifdef ENABLE_IPV6
+ /* Check if the IPv6 address is valid. */
+ if (!is_valid_ipv6_address(host_b, host_e))
+ {
+ error_code = PE_INVALID_IPV6_ADDRESS;
+ goto error;
+ }
+
+ /* Continue parsing after the closing ']'. */
+ p = host_e + 1;
+#else
+ error_code = PE_IPV6_NOT_SUPPORTED;
+ goto error;
+#endif
+
+ /* The closing bracket must be followed by a separator or by the
+ null char. */
+ /* http://[::1]... */
+ /* ^ */
+ if (!strchr (seps, *p))
+ {
+ /* Trailing garbage after []-delimited IPv6 address. */
+ error_code = PE_INVALID_HOST_NAME;
+ goto error;
+ }
+ }
+ else
+ {
+ p = strpbrk_or_eos (p, seps);
+ host_e = p;
+ }
+ ++seps; /* advance to '/' */
+
+ if (host_b == host_e)
+ {
+ error_code = PE_INVALID_HOST_NAME;
+ goto error;
+ }
+
+ port = scheme_default_port (scheme);
+ if (*p == ':')
+ {
+ const char *port_b, *port_e, *pp;
+
+ /* scheme://host:port/tralala */
+ /* ^ */
+ ++p;
+ port_b = p;
+ p = strpbrk_or_eos (p, seps);
+ port_e = p;
+
+ /* Allow empty port, as per rfc2396. */
+ if (port_b != port_e)
+ for (port = 0, pp = port_b; pp < port_e; pp++)
+ {
+ if (!c_isdigit (*pp))
+ {
+ /* http://host:12randomgarbage/blah */
+ /* ^ */
+ error_code = PE_BAD_PORT_NUMBER;
+ goto error;
+ }
+ port = 10 * port + (*pp - '0');
+ /* Check for too large port numbers here, before we have
+ a chance to overflow on bogus port values. */
+ if (port > 0xffff)
+ {
+ error_code = PE_BAD_PORT_NUMBER;
+ goto error;
+ }
+ }
+ }
+ /* Advance to the first separator *after* '/' (either ';' or '?',
+ depending on the scheme). */
+ ++seps;
+
+ /* Get the optional parts of URL, each part being delimited by
+ current location and the position of the next separator. */
+#define GET_URL_PART(sepchar, var) do { \
+ if (*p == sepchar) \
+ var##_b = ++p, var##_e = p = strpbrk_or_eos (p, seps); \
+ ++seps; \
+} while (0)
+
+ GET_URL_PART ('/', path);
+ if (supported_schemes[scheme].flags & scm_has_params)
+ GET_URL_PART (';', params);
+ if (supported_schemes[scheme].flags & scm_has_query)
+ GET_URL_PART ('?', query);
+ if (supported_schemes[scheme].flags & scm_has_fragment)
+ GET_URL_PART ('#', fragment);
+
+#undef GET_URL_PART
+ assert (*p == 0);
+
+ if (uname_b != uname_e)
+ {
+ /* http://user:pass@host */
+ /* ^ ^ */
+ /* uname_b uname_e */
+ if (!parse_credentials (uname_b, uname_e - 1, &user, &passwd))
+ {
+ error_code = PE_INVALID_USER_NAME;
+ goto error;
+ }
+ }
+
+ u = xnew0 (struct url);
+ u->scheme = scheme;
+ u->host = strdupdelim (host_b, host_e);
+ u->port = port;
+ u->user = user;
+ u->passwd = passwd;
+
+ u->path = strdupdelim (path_b, path_e);
+ path_modified = path_simplify (scheme, u->path);
+ split_path (u->path, &u->dir, &u->file);
+
+ host_modified = lowercase_str (u->host);
+
+ /* Decode %HH sequences in host name. This is important not so much
+ to support %HH sequences in host names (which other browser
+ don't), but to support binary characters (which will have been
+ converted to %HH by reencode_escapes). */
+ if (strchr (u->host, '%'))
+ {
+ url_unescape (u->host);
+ host_modified = true;
+
+ /* check for invalid control characters in host name */
+ for (p = u->host; *p; p++)
+ {
+ if (c_iscntrl(*p))
+ {
+ url_free(u);
+ error_code = PE_INVALID_HOST_NAME;
+ goto error;
+ }
+ }
+
+ /* Apply IDNA regardless of iri->utf8_encode status */
+ if (opt.enable_iri && iri)
+ {
+ char *new = idn_encode (iri, u->host);
+ if (new)
+ {
+ xfree (u->host);
+ u->host = new;
+ host_modified = true;
+ }
+ }
+ }
+
+ if (params_b)
+ u->params = strdupdelim (params_b, params_e);
+ if (query_b)
+ u->query = strdupdelim (query_b, query_e);
+ if (fragment_b)
+ u->fragment = strdupdelim (fragment_b, fragment_e);
+
+ if (opt.enable_iri || path_modified || u->fragment || host_modified || path_b == path_e)
+ {
+ /* If we suspect that a transformation has rendered what
+ url_string might return different from URL_ENCODED, rebuild
+ u->url using url_string. */
+ u->url = url_string (u, URL_AUTH_SHOW);
+
+ if (url_encoded != url)
+ xfree (url_encoded);
+ }
+ else
+ {
+ if (url_encoded == url)
+ u->url = xstrdup (url);
+ else
+ u->url = (char *) url_encoded;
+ }
+
+ return u;
+
+ error:
+ /* Cleanup in case of error: */
+ if (url_encoded && url_encoded != url)
+ xfree (url_encoded);
+
+ /* Transmit the error code to the caller, if the caller wants to
+ know. */
+ if (error)
+ *error = error_code;
+ return NULL;
+}
+
+/* Return the error message string from ERROR_CODE, which should have
+ been retrieved from url_parse. The error message is translated. */
+
+char *
+url_error (const char *url, int error_code)
+{
+ assert (error_code >= 0 && ((size_t) error_code) < countof (parse_errors));
+
+ if (error_code == PE_UNSUPPORTED_SCHEME)
+ {
+ char *error, *p;
+ char *scheme = xstrdup (url);
+ assert (url_has_scheme (url));
+
+ if ((p = strchr (scheme, ':')))
+ *p = '\0';
+ if (!c_strcasecmp (scheme, "https"))
+ error = aprintf (_("HTTPS support not compiled in"));
+ else
+ error = aprintf (_(parse_errors[error_code]), quote (scheme));
+ xfree (scheme);
+
+ return error;
+ }
+ else
+ return xstrdup (_(parse_errors[error_code]));
+}
+
+/* Split PATH into DIR and FILE. PATH comes from the URL and is
+ expected to be URL-escaped.
+
+ The path is split into directory (the part up to the last slash)
+ and file (the part after the last slash), which are subsequently
+ unescaped. Examples:
+
+ PATH DIR FILE
+ "foo/bar/baz" "foo/bar" "baz"
+ "foo/bar/" "foo/bar" ""
+ "foo" "" "foo"
+ "foo/bar/baz%2fqux" "foo/bar" "baz/qux" (!)
+
+ DIR and FILE are freshly allocated. */
+
+static void
+split_path (const char *path, char **dir, char **file)
+{
+ char *last_slash = strrchr (path, '/');
+ if (!last_slash)
+ {
+ *dir = xstrdup ("");
+ *file = xstrdup (path);
+ }
+ else
+ {
+ *dir = strdupdelim (path, last_slash);
+ *file = xstrdup (last_slash + 1);
+ }
+ url_unescape (*dir);
+ url_unescape (*file);
+}
+
+/* Note: URL's "full path" is the path with the query string and
+ params appended. The "fragment" (#foo) is intentionally ignored,
+ but that might be changed. For example, if the original URL was
+ "http://host:port/foo/bar/baz;bullshit?querystring#uselessfragment",
+ the full path will be "/foo/bar/baz;bullshit?querystring". */
+
+/* Return the length of the full path, without the terminating
+ zero. */
+
+static int
+full_path_length (const struct url *url)
+{
+ int len = 0;
+
+#define FROB(el) if (url->el) len += 1 + strlen (url->el)
+
+ FROB (path);
+ FROB (params);
+ FROB (query);
+
+#undef FROB
+
+ return len;
+}
+
+/* Write out the full path. */
+
+static void
+full_path_write (const struct url *url, char *where)
+{
+#define FROB(el, chr) do { \
+ char *f_el = url->el; \
+ if (f_el) { \
+ int l = strlen (f_el); \
+ *where++ = chr; \
+ memcpy (where, f_el, l); \
+ where += l; \
+ } \
+} while (0)
+
+ FROB (path, '/');
+ FROB (params, ';');
+ FROB (query, '?');
+
+#undef FROB
+}
+
+/* Public function for getting the "full path". E.g. if u->path is
+ "foo/bar" and u->query is "param=value", full_path will be
+ "/foo/bar?param=value". */
+
+char *
+url_full_path (const struct url *url)
+{
+ int length = full_path_length (url);
+ char *full_path = xmalloc (length + 1);
+
+ full_path_write (url, full_path);
+ full_path[length] = '\0';
+
+ return full_path;
+}
+
+/* Unescape CHR in an otherwise escaped STR. Used to selectively
+ escaping of certain characters, such as "/" and ":". Returns a
+ count of unescaped chars. */
+
+static void
+unescape_single_char (char *str, char chr)
+{
+ const char c1 = XNUM_TO_DIGIT (chr >> 4);
+ const char c2 = XNUM_TO_DIGIT (chr & 0xf);
+ char *h = str; /* hare */
+ char *t = str; /* tortoise */
+ for (; *h; h++, t++)
+ {
+ if (h[0] == '%' && h[1] == c1 && h[2] == c2)
+ {
+ *t = chr;
+ h += 2;
+ }
+ else
+ *t = *h;
+ }
+ *t = '\0';
+}
+
+/* Escape unsafe and reserved characters, except for the slash
+ characters. */
+
+static char *
+url_escape_dir (const char *dir)
+{
+ char *newdir = url_escape_1 (dir, urlchr_unsafe | urlchr_reserved, 1);
+ if (newdir == dir)
+ return (char *)dir;
+
+ unescape_single_char (newdir, '/');
+ return newdir;
+}
+
+/* Sync u->path and u->url with u->dir and u->file. Called after
+ u->file or u->dir have been changed, typically by the FTP code. */
+
+static void
+sync_path (struct url *u)
+{
+ char *newpath, *efile, *edir;
+
+ xfree (u->path);
+
+ /* u->dir and u->file are not escaped. URL-escape them before
+ reassembling them into u->path. That way, if they contain
+ separators like '?' or even if u->file contains slashes, the
+ path will be correctly assembled. (u->file can contain slashes
+ if the URL specifies it with %2f, or if an FTP server returns
+ it.) */
+ edir = url_escape_dir (u->dir);
+ efile = url_escape_1 (u->file, urlchr_unsafe | urlchr_reserved, 1);
+
+ if (!*edir)
+ newpath = xstrdup (efile);
+ else
+ {
+ int dirlen = strlen (edir);
+ int filelen = strlen (efile);
+
+ /* Copy "DIR/FILE" to newpath. */
+ char *p = newpath = xmalloc (dirlen + 1 + filelen + 1);
+ memcpy (p, edir, dirlen);
+ p += dirlen;
+ *p++ = '/';
+ memcpy (p, efile, filelen);
+ p += filelen;
+ *p = '\0';
+ }
+
+ u->path = newpath;
+
+ if (edir != u->dir)
+ xfree (edir);
+ if (efile != u->file)
+ xfree (efile);
+
+ /* Regenerate u->url as well. */
+ xfree (u->url);
+ u->url = url_string (u, URL_AUTH_SHOW);
+}
+
+/* Mutators. Code in ftp.c insists on changing u->dir and u->file.
+ This way we can sync u->path and u->url when they get changed. */
+
+void
+url_set_dir (struct url *url, const char *newdir)
+{
+ xfree (url->dir);
+ url->dir = xstrdup (newdir);
+ sync_path (url);
+}
+
+void
+url_set_file (struct url *url, const char *newfile)
+{
+ xfree (url->file);
+ url->file = xstrdup (newfile);
+ sync_path (url);
+}
+
+void
+url_free (struct url *url)
+{
+ if (url)
+ {
+ xfree (url->host);
+
+ xfree (url->path);
+ xfree (url->url);
+
+ xfree (url->params);
+ xfree (url->query);
+ xfree (url->fragment);
+ xfree (url->user);
+ xfree (url->passwd);
+
+ xfree (url->dir);
+ xfree (url->file);
+
+ xfree (url);
+ }
+}
+
+/* Create all the necessary directories for PATH (a file). Calls
+ make_directory internally. */
+int
+mkalldirs (const char *path)
+{
+ const char *p;
+ char *t;
+ struct stat st;
+ int res;
+
+ p = strrchr(path, '/');
+ p = p == NULL ? path : p;
+
+ /* Don't create if it's just a file. */
+ if ((p == path) && (*p != '/'))
+ return 0;
+ t = strdupdelim (path, p);
+
+ /* Check whether the directory exists. */
+ if ((stat (t, &st) == 0))
+ {
+ if (S_ISDIR (st.st_mode))
+ {
+ xfree (t);
+ return 0;
+ }
+ else
+ {
+ /* If the dir exists as a file name, remove it first. This
+ is *only* for Wget to work with buggy old CERN http
+ servers. Here is the scenario: When Wget tries to
+ retrieve a directory without a slash, e.g.
+ http://foo/bar (bar being a directory), CERN server will
+ not redirect it too http://foo/bar/ -- it will generate a
+ directory listing containing links to bar/file1,
+ bar/file2, etc. Wget will lose because it saves this
+ HTML listing to a file `bar', so it cannot create the
+ directory. To work around this, if the file of the same
+ name exists, we just remove it and create the directory
+ anyway. */
+ DEBUGP (("Removing %s because of directory danger!\n", t));
+ if (unlink (t))
+ logprintf (LOG_NOTQUIET, "Failed to unlink %s (%d): %s\n",
+ t, errno, strerror(errno));
+ }
+ }
+ res = make_directory (t);
+ if (res != 0)
+ logprintf (LOG_NOTQUIET, "%s: %s\n", t, strerror (errno));
+ xfree (t);
+ return res;
+}
+
+/* Functions for constructing the file name out of URL components. */
+
+/* A growable string structure, used by url_file_name and friends.
+ This should perhaps be moved to utils.c.
+
+ The idea is to have a convenient and efficient way to construct a
+ string by having various functions append data to it. Instead of
+ passing the obligatory BASEVAR, SIZEVAR and TAILPOS to all the
+ functions in questions, we pass the pointer to this struct.
+
+ Functions that write to the members in this struct must make sure
+ that base remains null terminated by calling append_null().
+ */
+
+struct growable {
+ char *base;
+ int size; /* memory allocated */
+ int tail; /* string length */
+};
+
+/* Ensure that the string can accept APPEND_COUNT more characters past
+ the current TAIL position. If necessary, this will grow the string
+ and update its allocated size. If the string is already large
+ enough to take TAIL+APPEND_COUNT characters, this does nothing. */
+#define GROW(g, append_size) do { \
+ struct growable *G_ = g; \
+ DO_REALLOC (G_->base, G_->size, G_->tail + append_size, char); \
+} while (0)
+
+/* Return the tail position of the string. */
+#define TAIL(r) ((r)->base + (r)->tail)
+
+/* Move the tail position by APPEND_COUNT characters. */
+#define TAIL_INCR(r, append_count) ((r)->tail += append_count)
+
+
+/* Append NULL to DEST. */
+static void
+append_null (struct growable *dest)
+{
+ GROW (dest, 1);
+ *TAIL (dest) = 0;
+}
+
+/* Append CH to DEST. */
+static void
+append_char (char ch, struct growable *dest)
+{
+ if (ch)
+ {
+ GROW (dest, 1);
+ *TAIL (dest) = ch;
+ TAIL_INCR (dest, 1);
+ }
+
+ append_null (dest);
+}
+
+/* Append the string STR to DEST. */
+static void
+append_string (const char *str, struct growable *dest)
+{
+ int l = strlen (str);
+
+ if (l)
+ {
+ GROW (dest, l);
+ memcpy (TAIL (dest), str, l);
+ TAIL_INCR (dest, l);
+ }
+
+ append_null (dest);
+}
+
+
+enum {
+ filechr_not_unix = 1, /* unusable on Unix, / and \0 */
+ filechr_not_vms = 2, /* unusable on VMS (ODS5), 0x00-0x1F * ? */
+ filechr_not_windows = 4, /* unusable on Windows, one of \|/<>?:*" */
+ filechr_control = 8 /* a control character, e.g. 0-31 */
+};
+
+#define FILE_CHAR_TEST(c, mask) \
+ ((opt.restrict_files_nonascii && !c_isascii ((unsigned char)(c))) || \
+ (filechr_table[(unsigned char)(c)] & (mask)))
+
+/* Shorthands for the table: */
+#define U filechr_not_unix
+#define V filechr_not_vms
+#define W filechr_not_windows
+#define C filechr_control
+
+#define UVWC U|V|W|C
+#define UW U|W
+#define VC V|C
+#define VW V|W
+
+/* Table of characters unsafe under various conditions (see above).
+
+ Arguably we could also claim `%' to be unsafe, since we use it as
+ the escape character. If we ever want to be able to reliably
+ translate file name back to URL, this would become important
+ crucial. Right now, it's better to be minimal in escaping. */
+
+static const unsigned char filechr_table[256] =
+{
+UVWC, VC, VC, VC, VC, VC, VC, VC, /* NUL SOH STX ETX EOT ENQ ACK BEL */
+ VC, VC, VC, VC, VC, VC, VC, VC, /* BS HT LF VT FF CR SO SI */
+ VC, VC, VC, VC, VC, VC, VC, VC, /* DLE DC1 DC2 DC3 DC4 NAK SYN ETB */
+ VC, VC, VC, VC, VC, VC, VC, VC, /* CAN EM SUB ESC FS GS RS US */
+ 0, 0, W, 0, 0, 0, 0, 0, /* SP ! " # $ % & ' */
+ 0, 0, VW, 0, 0, 0, 0, UW, /* ( ) * + , - . / */
+ 0, 0, 0, 0, 0, 0, 0, 0, /* 0 1 2 3 4 5 6 7 */
+ 0, 0, W, 0, W, 0, W, VW, /* 8 9 : ; < = > ? */
+ 0, 0, 0, 0, 0, 0, 0, 0, /* @ A B C D E F G */
+ 0, 0, 0, 0, 0, 0, 0, 0, /* H I J K L M N O */
+ 0, 0, 0, 0, 0, 0, 0, 0, /* P Q R S T U V W */
+ 0, 0, 0, 0, W, 0, 0, 0, /* X Y Z [ \ ] ^ _ */
+ 0, 0, 0, 0, 0, 0, 0, 0, /* ` a b c d e f g */
+ 0, 0, 0, 0, 0, 0, 0, 0, /* h i j k l m n o */
+ 0, 0, 0, 0, 0, 0, 0, 0, /* p q r s t u v w */
+ 0, 0, 0, 0, W, 0, 0, C, /* x y z { | } ~ DEL */
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 128-143 */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 144-159 */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+};
+#undef U
+#undef V
+#undef W
+#undef C
+#undef UW
+#undef UVWC
+#undef VC
+#undef VW
+
+/* FN_PORT_SEP is the separator between host and port in file names
+ for non-standard port numbers. On Unix this is normally ':', as in
+ "www.xemacs.org:4001/index.html". Under Windows, we set it to +
+ because Windows can't handle ':' in file names. */
+#define FN_PORT_SEP (opt.restrict_files_os != restrict_windows ? ':' : '+')
+
+/* FN_QUERY_SEP is the separator between the file name and the URL
+ query, normally '?'. Because VMS and Windows cannot handle '?' in a
+ file name, we use '@' instead there. */
+#define FN_QUERY_SEP \
+ (((opt.restrict_files_os != restrict_vms) && \
+ (opt.restrict_files_os != restrict_windows)) ? '?' : '@')
+#define FN_QUERY_SEP_STR \
+ (((opt.restrict_files_os != restrict_vms) && \
+ (opt.restrict_files_os != restrict_windows)) ? "?" : "@")
+
+/* Quote path element, characters in [b, e), as file name, and append
+ the quoted string to DEST. Each character is quoted as per
+ file_unsafe_char and the corresponding table.
+
+ If ESCAPED is true, the path element is considered to be
+ URL-escaped and will be unescaped prior to inspection. */
+
+static void
+append_uri_pathel (const char *b, const char *e, bool escaped,
+ struct growable *dest)
+{
+ const char *p;
+ int quoted, outlen;
+
+ int mask;
+ if (opt.restrict_files_os == restrict_unix)
+ mask = filechr_not_unix;
+ else if (opt.restrict_files_os == restrict_vms)
+ mask = filechr_not_vms;
+ else
+ mask = filechr_not_windows;
+ if (opt.restrict_files_ctrl)
+ mask |= filechr_control;
+
+ /* Copy [b, e) to PATHEL and URL-unescape it. */
+ if (escaped)
+ {
+ char *unescaped;
+ BOUNDED_TO_ALLOCA (b, e, unescaped);
+ url_unescape (unescaped);
+ b = unescaped;
+ e = unescaped + strlen (unescaped);
+ }
+
+ /* Defang ".." when found as component of path. Remember that path
+ comes from the URL and might contain malicious input. */
+ if (e - b == 2 && b[0] == '.' && b[1] == '.')
+ {
+ b = "%2E%2E";
+ e = b + 6;
+ }
+
+ /* Walk the PATHEL string and check how many characters we'll need
+ to quote. */
+ quoted = 0;
+ for (p = b; p < e; p++)
+ if (FILE_CHAR_TEST (*p, mask))
+ ++quoted;
+
+ /* Calculate the length of the output string. e-b is the input
+ string length. Each quoted char introduces two additional
+ characters in the string, hence 2*quoted. */
+ outlen = (e - b) + (2 * quoted);
+ GROW (dest, outlen);
+
+ if (!quoted)
+ {
+ /* If there's nothing to quote, we can simply append the string
+ without processing it again. */
+ memcpy (TAIL (dest), b, outlen);
+ }
+ else
+ {
+ char *q = TAIL (dest);
+ for (p = b; p < e; p++)
+ {
+ if (!FILE_CHAR_TEST (*p, mask))
+ *q++ = *p;
+ else
+ {
+ unsigned char ch = *p;
+ *q++ = '%';
+ *q++ = XNUM_TO_DIGIT (ch >> 4);
+ *q++ = XNUM_TO_DIGIT (ch & 0xf);
+ }
+ }
+ assert (q - TAIL (dest) == outlen);
+ }
+
+ /* Perform inline case transformation if required. */
+ if (opt.restrict_files_case == restrict_lowercase
+ || opt.restrict_files_case == restrict_uppercase)
+ {
+ char *q;
+ for (q = TAIL (dest); q < TAIL (dest) + outlen; ++q)
+ {
+ if (opt.restrict_files_case == restrict_lowercase)
+ *q = c_tolower (*q);
+ else
+ *q = c_toupper (*q);
+ }
+ }
+
+ TAIL_INCR (dest, outlen);
+ append_null (dest);
+}
+
+#ifdef HAVE_ICONV
+static char *
+convert_fname (char *fname)
+{
+ char *converted_fname;
+ const char *from_encoding = opt.encoding_remote;
+ const char *to_encoding = opt.locale;
+ iconv_t cd;
+ size_t len, done, inlen, outlen;
+ char *s;
+ const char *orig_fname;
+
+ /* Defaults for remote and local encodings. */
+ if (!from_encoding)
+ from_encoding = "UTF-8";
+ if (!to_encoding)
+ to_encoding = nl_langinfo (CODESET);
+
+ cd = iconv_open (to_encoding, from_encoding);
+ if (cd == (iconv_t) (-1))
+ {
+ logprintf (LOG_VERBOSE, _ ("Conversion from %s to %s isn't supported\n"),
+ quote (from_encoding), quote (to_encoding));
+ return fname;
+ }
+
+ orig_fname = fname;
+ inlen = strlen (fname);
+ len = outlen = inlen * 2;
+ converted_fname = s = xmalloc (outlen + 1);
+ done = 0;
+
+ for (;;)
+ {
+ errno = 0;
+ if (iconv (cd, (ICONV_CONST char **) &fname, &inlen, &s, &outlen) == 0
+ && iconv (cd, NULL, NULL, &s, &outlen) == 0)
+ {
+ *(converted_fname + len - outlen - done) = '\0';
+ iconv_close (cd);
+ DEBUGP (("Converted file name '%s' (%s) -> '%s' (%s)\n",
+ orig_fname, from_encoding, converted_fname, to_encoding));
+ xfree (orig_fname);
+ return converted_fname;
+ }
+
+ /* Incomplete or invalid multibyte sequence */
+ if (errno == EINVAL || errno == EILSEQ || errno == 0)
+ {
+ if (errno)
+ logprintf (LOG_VERBOSE,
+ _ ("Incomplete or invalid multibyte sequence encountered\n"));
+ else
+ logprintf (LOG_VERBOSE,
+ _ ("Unconvertable multibyte sequence encountered\n"));
+ xfree (converted_fname);
+ converted_fname = (char *) orig_fname;
+ break;
+ }
+ else if (errno == E2BIG) /* Output buffer full */
+ {
+ done = len;
+ len = outlen = done + inlen * 2;
+ converted_fname = xrealloc (converted_fname, outlen + 1);
+ s = converted_fname + done;
+ }
+ else /* Weird, we got an unspecified error */
+ {
+ logprintf (LOG_VERBOSE, _ ("Unhandled errno %d\n"), errno);
+ xfree (converted_fname);
+ converted_fname = (char *) orig_fname;
+ break;
+ }
+ }
+ DEBUGP (("Failed to convert file name '%s' (%s) -> '?' (%s)\n",
+ orig_fname, from_encoding, to_encoding));
+
+ iconv_close (cd);
+
+ return converted_fname;
+}
+#else
+static char *
+convert_fname (char *fname)
+{
+ return fname;
+}
+#endif
+
+/* Append to DEST the directory structure that corresponds the
+ directory part of URL's path. For example, if the URL is
+ http://server/dir1/dir2/file, this appends "/dir1/dir2".
+
+ Each path element ("dir1" and "dir2" in the above example) is
+ examined, url-unescaped, and re-escaped as file name element.
+
+ Additionally, it cuts as many directories from the path as
+ specified by opt.cut_dirs. For example, if opt.cut_dirs is 1, it
+ will produce "bar" for the above example. For 2 or more, it will
+ produce "".
+
+ Each component of the path is quoted for use as file name. */
+
+static void
+append_dir_structure (const struct url *u, struct growable *dest)
+{
+ char *pathel, *next;
+ int cut = opt.cut_dirs;
+
+ /* Go through the path components, de-URL-quote them, and quote them
+ (if necessary) as file names. */
+
+ pathel = u->path;
+ for (; (next = strchr (pathel, '/')) != NULL; pathel = next + 1)
+ {
+ if (cut-- > 0)
+ continue;
+ if (pathel == next)
+ /* Ignore empty pathels. */
+ continue;
+
+ if (dest->tail)
+ append_char ('/', dest);
+ append_uri_pathel (pathel, next, true, dest);
+ }
+}
+
+/* Return a unique file name that matches the given URL as well as
+ possible. Does not create directories on the file system. */
+
+char *
+url_file_name (const struct url *u, char *replaced_filename)
+{
+ struct growable fnres; /* stands for "file name result" */
+ struct growable temp_fnres;
+
+ const char *u_file;
+ char *fname, *unique, *fname_len_check;
+ const char *index_filename = "index.html"; /* The default index file is index.html */
+ size_t max_length;
+
+ fnres.base = NULL;
+ fnres.size = 0;
+ fnres.tail = 0;
+
+ temp_fnres.base = NULL;
+ temp_fnres.size = 0;
+ temp_fnres.tail = 0;
+
+ /* If an alternative index file was defined, change index_filename */
+ if (opt.default_page)
+ index_filename = opt.default_page;
+
+
+ /* Start with the directory prefix, if specified. */
+ if (opt.dir_prefix)
+ append_string (opt.dir_prefix, &fnres);
+
+ /* If "dirstruct" is turned on (typically the case with -r), add
+ the host and port (unless those have been turned off) and
+ directory structure. */
+ /* All safe remote chars are unescaped and stored in temp_fnres,
+ then converted to local and appended to fnres.
+ Internationalized URL/IDN will produce punycode to lookup IP from DNS:
+ https://en.wikipedia.org/wiki/URL
+ https://en.wikipedia.org/wiki/Internationalized_domain_name
+ Non-ASCII code chars in the path:
+ https://en.wikipedia.org/wiki/List_of_Unicode_characters
+ https://en.wikipedia.org/wiki/List_of_writing_systems */
+ if (opt.dirstruct)
+ {
+ if (opt.protocol_directories)
+ {
+ if (temp_fnres.tail)
+ append_char ('/', &temp_fnres);
+ append_string (supported_schemes[u->scheme].name, &temp_fnres);
+ }
+ if (opt.add_hostdir)
+ {
+ if (temp_fnres.tail)
+ append_char ('/', &temp_fnres);
+ if (0 != strcmp (u->host, ".."))
+ append_string (u->host, &temp_fnres);
+ else
+ /* Host name can come from the network; malicious DNS may
+ allow ".." to be resolved, causing us to write to
+ "../<file>". Defang such host names. */
+ append_string ("%2E%2E", &temp_fnres);
+ if (u->port != scheme_default_port (u->scheme))
+ {
+ char portstr[24];
+ number_to_string (portstr, u->port);
+ append_char (FN_PORT_SEP, &temp_fnres);
+ append_string (portstr, &temp_fnres);
+ }
+ }
+
+ append_dir_structure (u, &temp_fnres);
+ }
+
+ if (!replaced_filename)
+ {
+ /* Create the filename. */
+ u_file = *u->file ? u->file : index_filename;
+
+ /* Append "?query" to the file name, even if empty,
+ * and create fname_len_check. */
+ if (u->query)
+ fname_len_check = concat_strings (u_file, FN_QUERY_SEP_STR, u->query, NULL);
+ else
+ fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));
+ }
+ else
+ {
+ u_file = replaced_filename;
+ fname_len_check = strdupdelim (u_file, u_file + strlen (u_file));
+ }
+
+ if (temp_fnres.tail)
+ append_char ('/', &temp_fnres);
+
+ append_uri_pathel (fname_len_check,
+ fname_len_check + strlen (fname_len_check), true, &temp_fnres);
+
+ /* Zero-terminate the temporary file name. */
+ append_char ('\0', &temp_fnres);
+
+ /* convert all remote chars before length check and appending to local path */
+ fname = convert_fname (temp_fnres.base);
+ temp_fnres.base = NULL;
+ temp_fnres.size = 0;
+ temp_fnres.tail = 0;
+ append_string (fname, &temp_fnres);
+ xfree (fname);
+
+ /* Check that the length of the file name is acceptable. */
+#ifdef WINDOWS
+ if (MAX_PATH > (fnres.tail + CHOMP_BUFFER + 2))
+ {
+ max_length = MAX_PATH - (fnres.tail + CHOMP_BUFFER + 2);
+ /* FIXME: In Windows a filename is usually limited to 255 characters.
+ To really be accurate you could call GetVolumeInformation() to get
+ lpMaximumComponentLength
+ */
+ if (max_length > 255)
+ {
+ max_length = 255;
+ }
+ }
+ else
+ {
+ max_length = 0;
+ }
+#else
+ max_length = get_max_length (fnres.base, fnres.tail, _PC_NAME_MAX) - CHOMP_BUFFER;
+#endif
+ if (max_length > 0 && strlen (temp_fnres.base) > max_length)
+ {
+ logprintf (LOG_NOTQUIET, "The name is too long, %lu chars total.\n",
+ (unsigned long) strlen (temp_fnres.base));
+ logprintf (LOG_NOTQUIET, "Trying to shorten...\n");
+
+ /* Shorten the file name. */
+ temp_fnres.base[max_length] = '\0';
+
+ logprintf (LOG_NOTQUIET, "New name is %s.\n", temp_fnres.base);
+ }
+
+ xfree (fname_len_check);
+
+ /* The filename has already been 'cleaned' by append_uri_pathel() above. So,
+ * just append it. */
+ if (fnres.tail)
+ append_char ('/', &fnres);
+ append_string (temp_fnres.base, &fnres);
+
+ fname = fnres.base;
+
+ /* Make a final check that the path length is acceptable? */
+ /* TODO: check fnres.base for path length problem */
+
+ xfree (temp_fnres.base);
+
+ /* Check the cases in which the unique extensions are not used:
+ 1) Clobbering is turned off (-nc).
+ 2) Retrieval with regetting.
+ 3) Timestamping is used.
+ 4) Hierarchy is built.
+ 5) Backups are specified.
+
+ The exception is the case when file does exist and is a
+ directory (see `mkalldirs' for explanation). */
+
+ if (ALLOW_CLOBBER
+ && !(file_exists_p (fname, NULL) && !file_non_directory_p (fname)))
+ {
+ unique = fname;
+ }
+ else
+ {
+ unique = unique_name (fname, true);
+ if (unique != fname)
+ xfree (fname);
+ }
+
+/* On VMS, alter the name as required. */
+#ifdef __VMS
+ {
+ char *unique2;
+
+ unique2 = ods_conform( unique);
+ if (unique2 != unique)
+ {
+ xfree (unique);
+ unique = unique2;
+ }
+ }
+#endif /* def __VMS */
+
+ return unique;
+}
+
+/* Resolve "." and ".." elements of PATH by destructively modifying
+ PATH and return true if PATH has been modified, false otherwise.
+
+ The algorithm is in spirit similar to the one described in rfc1808,
+ although implemented differently, in one pass. To recap, path
+ elements containing only "." are removed, and ".." is taken to mean
+ "back up one element". Single leading and trailing slashes are
+ preserved.
+
+ For example, "a/b/c/./../d/.." will yield "a/b/". More exhaustive
+ test examples are provided below. If you change anything in this
+ function, run test_path_simplify to make sure you haven't broken a
+ test case. */
+
+static bool
+path_simplify (enum url_scheme scheme, char *path)
+{
+ char *h = path; /* hare */
+ char *t = path; /* tortoise */
+ char *beg = path;
+ char *end = strchr (path, '\0');
+
+ while (h < end)
+ {
+ /* Hare should be at the beginning of a path element. */
+
+ if (h[0] == '.' && (h[1] == '/' || h[1] == '\0'))
+ {
+ /* Ignore "./". */
+ h += 2;
+ }
+ else if (h[0] == '.' && h[1] == '.' && (h[2] == '/' || h[2] == '\0'))
+ {
+ /* Handle "../" by retreating the tortoise by one path
+ element -- but not past beginning. */
+ if (t > beg)
+ {
+ /* Move backwards until T hits the beginning of the
+ previous path element or the beginning of path. */
+ for (--t; t > beg && t[-1] != '/'; t--)
+ ;
+ }
+ else if (scheme == SCHEME_FTP
+#ifdef HAVE_SSL
+ || scheme == SCHEME_FTPS
+#endif
+ )
+ {
+ /* If we're at the beginning, copy the "../" literally
+ and move the beginning so a later ".." doesn't remove
+ it. This violates RFC 3986; but we do it for FTP
+ anyway because there is otherwise no way to get at a
+ parent directory, when the FTP server drops us in a
+ non-root directory (which is not uncommon). */
+ beg = t + 3;
+ goto regular;
+ }
+ h += 3;
+ }
+ else
+ {
+ regular:
+ /* A regular path element. If H hasn't advanced past T,
+ simply skip to the next path element. Otherwise, copy
+ the path element until the next slash. */
+ if (t == h)
+ {
+ /* Skip the path element, including the slash. */
+ while (h < end && *h != '/')
+ t++, h++;
+ if (h < end)
+ t++, h++;
+ }
+ else
+ {
+ /* Copy the path element, including the final slash. */
+ while (h < end && *h != '/')
+ *t++ = *h++;
+ if (h < end)
+ *t++ = *h++;
+ }
+ }
+ }
+
+ if (t != h)
+ *t = '\0';
+
+ return t != h;
+}
+
+/* Return the length of URL's path. Path is considered to be
+ terminated by one or more of the ?query or ;params or #fragment,
+ depending on the scheme. */
+
+static const char *
+path_end (const char *url)
+{
+ enum url_scheme scheme = url_scheme (url);
+ const char *seps;
+ if (scheme == SCHEME_INVALID)
+ scheme = SCHEME_HTTP; /* use http semantics for rel links */
+ /* +2 to ignore the first two separators ':' and '/' */
+ seps = init_seps (scheme) + 2;
+ return strpbrk_or_eos (url, seps);
+}
+
+/* Find the last occurrence of character C in the range [b, e), or
+ NULL, if none are present. */
+#define find_last_char(b, e, c) memrchr ((b), (c), (e) - (b))
+
+/* Merge BASE with LINK and return the resulting URI.
+
+ Either of the URIs may be absolute or relative, complete with the
+ host name, or path only. This tries to reasonably handle all
+ foreseeable cases. It only employs minimal URL parsing, without
+ knowledge of the specifics of schemes.
+
+ I briefly considered making this function call path_simplify after
+ the merging process, as rfc1738 seems to suggest. This is a bad
+ idea for several reasons: 1) it complexifies the code, and 2)
+ url_parse has to simplify path anyway, so it's wasteful to boot. */
+
+char *
+uri_merge (const char *base, const char *link)
+{
+ int linklength;
+ const char *end;
+ char *merge;
+
+ if (url_has_scheme (link))
+ return xstrdup (link);
+
+ /* We may not examine BASE past END. */
+ end = path_end (base);
+ linklength = strlen (link);
+
+ if (!*link)
+ {
+ /* Empty LINK points back to BASE, query string and all. */
+ return xstrdup (base);
+ }
+ else if (*link == '?')
+ {
+ /* LINK points to the same location, but changes the query
+ string. Examples: */
+ /* uri_merge("path", "?new") -> "path?new" */
+ /* uri_merge("path?foo", "?new") -> "path?new" */
+ /* uri_merge("path?foo#bar", "?new") -> "path?new" */
+ /* uri_merge("path#foo", "?new") -> "path?new" */
+ int baselength = end - base;
+ merge = xmalloc (baselength + linklength + 1);
+ memcpy (merge, base, baselength);
+ memcpy (merge + baselength, link, linklength);
+ merge[baselength + linklength] = '\0';
+ }
+ else if (*link == '#')
+ {
+ /* uri_merge("path", "#new") -> "path#new" */
+ /* uri_merge("path#foo", "#new") -> "path#new" */
+ /* uri_merge("path?foo", "#new") -> "path?foo#new" */
+ /* uri_merge("path?foo#bar", "#new") -> "path?foo#new" */
+ int baselength;
+ const char *end1 = strchr (base, '#');
+ if (!end1)
+ end1 = base + strlen (base);
+ baselength = end1 - base;
+ merge = xmalloc (baselength + linklength + 1);
+ memcpy (merge, base, baselength);
+ memcpy (merge + baselength, link, linklength);
+ merge[baselength + linklength] = '\0';
+ }
+ else if (*link == '/' && *(link + 1) == '/')
+ {
+ /* LINK begins with "//" and so is a net path: we need to
+ replace everything after (and including) the double slash
+ with LINK. */
+
+ /* uri_merge("foo", "//new/bar") -> "//new/bar" */
+ /* uri_merge("//old/foo", "//new/bar") -> "//new/bar" */
+ /* uri_merge("http://old/foo", "//new/bar") -> "http://new/bar" */
+
+ int span;
+ const char *slash;
+ const char *start_insert;
+
+ /* Look for first slash. */
+ slash = memchr (base, '/', end - base);
+ /* If found slash and it is a double slash, then replace
+ from this point, else default to replacing from the
+ beginning. */
+ if (slash && *(slash + 1) == '/')
+ start_insert = slash;
+ else
+ start_insert = base;
+
+ span = start_insert - base;
+ merge = xmalloc (span + linklength + 1);
+ if (span)
+ memcpy (merge, base, span);
+ memcpy (merge + span, link, linklength);
+ merge[span + linklength] = '\0';
+ }
+ else if (*link == '/')
+ {
+ /* LINK is an absolute path: we need to replace everything
+ after (and including) the FIRST slash with LINK.
+
+ So, if BASE is "http://host/whatever/foo/bar", and LINK is
+ "/qux/xyzzy", our result should be
+ "http://host/qux/xyzzy". */
+ int span;
+ const char *slash;
+ const char *start_insert = NULL; /* for gcc to shut up. */
+ const char *pos = base;
+ bool seen_slash_slash = false;
+ /* We're looking for the first slash, but want to ignore
+ double slash. */
+ again:
+ slash = memchr (pos, '/', end - pos);
+ if (slash && !seen_slash_slash)
+ if (*(slash + 1) == '/')
+ {
+ pos = slash + 2;
+ seen_slash_slash = true;
+ goto again;
+ }
+
+ /* At this point, SLASH is the location of the first / after
+ "//", or the first slash altogether. START_INSERT is the
+ pointer to the location where LINK will be inserted. When
+ examining the last two examples, keep in mind that LINK
+ begins with '/'. */
+
+ if (!slash && !seen_slash_slash)
+ /* example: "foo" */
+ /* ^ */
+ start_insert = base;
+ else if (!slash && seen_slash_slash)
+ /* example: "http://foo" */
+ /* ^ */
+ start_insert = end;
+ else if (slash && !seen_slash_slash)
+ /* example: "foo/bar" */
+ /* ^ */
+ start_insert = base;
+ else if (slash && seen_slash_slash)
+ /* example: "http://something/" */
+ /* ^ */
+ start_insert = slash;
+
+ span = start_insert - base;
+ merge = xmalloc (span + linklength + 1);
+ if (span)
+ memcpy (merge, base, span);
+ memcpy (merge + span, link, linklength);
+ merge[span + linklength] = '\0';
+ }
+ else
+ {
+ /* LINK is a relative URL: we need to replace everything
+ after last slash (possibly empty) with LINK.
+
+ So, if BASE is "whatever/foo/bar", and LINK is "qux/xyzzy",
+ our result should be "whatever/foo/qux/xyzzy". */
+ bool need_explicit_slash = false;
+ int span;
+ const char *start_insert;
+ const char *last_slash = find_last_char (base, end, '/');
+ if (!last_slash)
+ {
+ /* No slash found at all. Replace what we have with LINK. */
+ start_insert = base;
+ }
+ else if (last_slash && last_slash >= base + 2
+ && last_slash[-2] == ':' && last_slash[-1] == '/')
+ {
+ /* example: http://host" */
+ /* ^ */
+ start_insert = end + 1;
+ need_explicit_slash = true;
+ }
+ else
+ {
+ /* example: "whatever/foo/bar" */
+ /* ^ */
+ start_insert = last_slash + 1;
+ }
+
+ span = start_insert - base;
+ merge = xmalloc (span + linklength + 1);
+ if (span)
+ memcpy (merge, base, span);
+ if (need_explicit_slash)
+ merge[span - 1] = '/';
+ memcpy (merge + span, link, linklength);
+ merge[span + linklength] = '\0';
+ }
+
+ return merge;
+}
+
+#define APPEND(p, s) do { \
+ int len = strlen (s); \
+ memcpy (p, s, len); \
+ p += len; \
+} while (0)
+
+/* Use this instead of password when the actual password is supposed
+ to be hidden. We intentionally use a generic string without giving
+ away the number of characters in the password, like previous
+ versions did. */
+#define HIDDEN_PASSWORD "*password*"
+
+/* Recreate the URL string from the data in URL.
+
+ If HIDE is true (as it is when we're calling this on a URL we plan
+ to print, but not when calling it to canonicalize a URL for use
+ within the program), password will be hidden. Unsafe characters in
+ the URL will be quoted. */
+
+char *
+url_string (const struct url *url, enum url_auth_mode auth_mode)
+{
+ int size;
+ char *result, *p;
+ char *quoted_host, *quoted_user = NULL, *quoted_passwd = NULL;
+
+ int scheme_port = supported_schemes[url->scheme].default_port;
+ const char *scheme_str = supported_schemes[url->scheme].leading_string;
+ int fplen = full_path_length (url);
+
+ bool brackets_around_host;
+
+ assert (scheme_str != NULL);
+
+ /* Make sure the user name and password are quoted. */
+ if (url->user)
+ {
+ if (auth_mode != URL_AUTH_HIDE)
+ {
+ quoted_user = url_escape_allow_passthrough (url->user);
+ if (url->passwd)
+ {
+ if (auth_mode == URL_AUTH_HIDE_PASSWD)
+ quoted_passwd = (char *) HIDDEN_PASSWORD;
+ else
+ quoted_passwd = url_escape_allow_passthrough (url->passwd);
+ }
+ }
+ }
+
+ /* In the unlikely event that the host name contains non-printable
+ characters, quote it for displaying to the user. */
+ quoted_host = url_escape_allow_passthrough (url->host);
+
+ /* Undo the quoting of colons that URL escaping performs. IPv6
+ addresses may legally contain colons, and in that case must be
+ placed in square brackets. */
+ if (quoted_host != url->host)
+ unescape_single_char (quoted_host, ':');
+ brackets_around_host = strchr (quoted_host, ':') != NULL;
+
+ size = (strlen (scheme_str)
+ + strlen (quoted_host)
+ + (brackets_around_host ? 2 : 0)
+ + fplen
+ + 1);
+ if (url->port != scheme_port)
+ size += 1 + numdigit (url->port);
+ if (quoted_user)
+ {
+ size += 1 + strlen (quoted_user);
+ if (quoted_passwd)
+ size += 1 + strlen (quoted_passwd);
+ }
+
+ p = result = xmalloc (size);
+
+ APPEND (p, scheme_str);
+ if (quoted_user)
+ {
+ APPEND (p, quoted_user);
+ if (quoted_passwd)
+ {
+ *p++ = ':';
+ APPEND (p, quoted_passwd);
+ }
+ *p++ = '@';
+ }
+
+ if (brackets_around_host)
+ *p++ = '[';
+ APPEND (p, quoted_host);
+ if (brackets_around_host)
+ *p++ = ']';
+ if (url->port != scheme_port)
+ {
+ *p++ = ':';
+ p = number_to_string (p, url->port);
+ }
+
+ full_path_write (url, p);
+ p += fplen;
+ *p++ = '\0';
+
+ assert (p - result == size);
+
+ if (quoted_user && quoted_user != url->user)
+ xfree (quoted_user);
+ if (quoted_passwd && auth_mode == URL_AUTH_SHOW
+ && quoted_passwd != url->passwd)
+ xfree (quoted_passwd);
+ if (quoted_host != url->host)
+ xfree (quoted_host);
+
+ return result;
+}
+
+/* Return true if scheme a is similar to scheme b.
+
+ Schemes are similar if they are equal. If SSL is supported, schemes
+ are also similar if one is http (SCHEME_HTTP) and the other is https
+ (SCHEME_HTTPS). */
+bool
+schemes_are_similar_p (enum url_scheme a, enum url_scheme b)
+{
+ if (a == b)
+ return true;
+#ifdef HAVE_SSL
+ if ((a == SCHEME_HTTP && b == SCHEME_HTTPS)
+ || (a == SCHEME_HTTPS && b == SCHEME_HTTP))
+ return true;
+#endif
+ return false;
+}
+
+static int
+getchar_from_escaped_string (const char *str, char *c)
+{
+ const char *p = str;
+
+ assert (str && *str);
+ assert (c);
+
+ if (p[0] == '%')
+ {
+ if (!c_isxdigit(p[1]) || !c_isxdigit(p[2]))
+ {
+ *c = '%';
+ return 1;
+ }
+ else
+ {
+ if (p[2] == 0)
+ return 0; /* error: invalid string */
+
+ *c = X2DIGITS_TO_NUM (p[1], p[2]);
+ if (URL_RESERVED_CHAR(*c))
+ {
+ *c = '%';
+ return 1;
+ }
+ else
+ return 3;
+ }
+ }
+ else
+ {
+ *c = p[0];
+ }
+
+ return 1;
+}
+
+bool
+are_urls_equal (const char *u1, const char *u2)
+{
+ const char *p, *q;
+ int pp, qq;
+ char ch1, ch2;
+ assert(u1 && u2);
+
+ p = u1;
+ q = u2;
+
+ while (*p && *q
+ && (pp = getchar_from_escaped_string (p, &ch1))
+ && (qq = getchar_from_escaped_string (q, &ch2))
+ && (c_tolower(ch1) == c_tolower(ch2)))
+ {
+ p += pp;
+ q += qq;
+ }
+
+ return (*p == 0 && *q == 0 ? true : false);
+}
+
+#ifdef TESTING
+/* Debugging and testing support for path_simplify. */
+
+#if 0
+/* Debug: run path_simplify on PATH and return the result in a new
+ string. Useful for calling from the debugger. */
+static char *
+ps (char *path)
+{
+ char *copy = xstrdup (path);
+ path_simplify (copy);
+ return copy;
+}
+#endif
+
+static const char *
+run_test (const char *test, const char *expected_result, enum url_scheme scheme,
+ bool expected_change)
+{
+ char *test_copy = xstrdup (test);
+ bool modified = path_simplify (scheme, test_copy);
+
+ if (0 != strcmp (test_copy, expected_result))
+ {
+ printf ("Failed path_simplify(\"%s\"): expected \"%s\", got \"%s\".\n",
+ test, expected_result, test_copy);
+ mu_assert ("", 0);
+ }
+ if (modified != expected_change)
+ {
+ if (expected_change)
+ printf ("Expected modification with path_simplify(\"%s\").\n",
+ test);
+ else
+ printf ("Expected no modification with path_simplify(\"%s\").\n",
+ test);
+ }
+ xfree (test_copy);
+ mu_assert ("", modified == expected_change);
+ return NULL;
+}
+
+const char *
+test_path_simplify (void)
+{
+ static const struct {
+ const char *test, *result;
+ enum url_scheme scheme;
+ bool should_modify;
+ } tests[] = {
+ { "", "", SCHEME_HTTP, false },
+ { ".", "", SCHEME_HTTP, true },
+ { "./", "", SCHEME_HTTP, true },
+ { "..", "", SCHEME_HTTP, true },
+ { "../", "", SCHEME_HTTP, true },
+ { "..", "..", SCHEME_FTP, false },
+ { "../", "../", SCHEME_FTP, false },
+ { "foo", "foo", SCHEME_HTTP, false },
+ { "foo/bar", "foo/bar", SCHEME_HTTP, false },
+ { "foo///bar", "foo///bar", SCHEME_HTTP, false },
+ { "foo/.", "foo/", SCHEME_HTTP, true },
+ { "foo/./", "foo/", SCHEME_HTTP, true },
+ { "foo./", "foo./", SCHEME_HTTP, false },
+ { "foo/../bar", "bar", SCHEME_HTTP, true },
+ { "foo/../bar/", "bar/", SCHEME_HTTP, true },
+ { "foo/bar/..", "foo/", SCHEME_HTTP, true },
+ { "foo/bar/../x", "foo/x", SCHEME_HTTP, true },
+ { "foo/bar/../x/", "foo/x/", SCHEME_HTTP, true },
+ { "foo/..", "", SCHEME_HTTP, true },
+ { "foo/../..", "", SCHEME_HTTP, true },
+ { "foo/../../..", "", SCHEME_HTTP, true },
+ { "foo/../../bar/../../baz", "baz", SCHEME_HTTP, true },
+ { "foo/../..", "..", SCHEME_FTP, true },
+ { "foo/../../..", "../..", SCHEME_FTP, true },
+ { "foo/../../bar/../../baz", "../../baz", SCHEME_FTP, true },
+ { "a/b/../../c", "c", SCHEME_HTTP, true },
+ { "./a/../b", "b", SCHEME_HTTP, true }
+ };
+ unsigned i;
+
+ for (i = 0; i < countof (tests); i++)
+ {
+ const char *message;
+ const char *test = tests[i].test;
+ const char *expected_result = tests[i].result;
+ enum url_scheme scheme = tests[i].scheme;
+ bool expected_change = tests[i].should_modify;
+
+ message = run_test (test, expected_result, scheme, expected_change);
+ if (message) return message;
+ }
+ return NULL;
+}
+
+const char *
+test_append_uri_pathel(void)
+{
+ unsigned i;
+ static const struct {
+ const char *original_url;
+ const char *input;
+ bool escaped;
+ const char *expected_result;
+ } test_array[] = {
+ { "http://www.yoyodyne.com/path/", "somepage.html", false, "http://www.yoyodyne.com/path/somepage.html" },
+ };
+
+ for (i = 0; i < countof(test_array); ++i)
+ {
+ struct growable dest;
+ const char *p = test_array[i].input;
+
+ memset (&dest, 0, sizeof (dest));
+
+ append_string (test_array[i].original_url, &dest);
+ append_uri_pathel (p, p + strlen(p), test_array[i].escaped, &dest);
+
+ mu_assert ("test_append_uri_pathel: wrong result",
+ strcmp (dest.base, test_array[i].expected_result) == 0);
+ xfree (dest.base);
+ }
+
+ return NULL;
+}
+
+const char *
+test_are_urls_equal(void)
+{
+ unsigned i;
+ static const struct {
+ const char *url1;
+ const char *url2;
+ bool expected_result;
+ } test_array[] = {
+ { "http://www.adomain.com/apath/", "http://www.adomain.com/apath/", true },
+ { "http://www.adomain.com/apath/", "http://www.adomain.com/anotherpath/", false },
+ { "http://www.adomain.com/apath/", "http://www.anotherdomain.com/path/", false },
+ { "http://www.adomain.com/~path/", "http://www.adomain.com/%7epath/", true },
+ { "http://www.adomain.com/longer-path/", "http://www.adomain.com/path/", false },
+ { "http://www.adomain.com/path%2f", "http://www.adomain.com/path/", false },
+ };
+
+ for (i = 0; i < countof(test_array); ++i)
+ {
+ mu_assert ("test_are_urls_equal: wrong result",
+ are_urls_equal (test_array[i].url1, test_array[i].url2) == test_array[i].expected_result);
+ }
+
+ return NULL;
+}
+
+#endif /* TESTING */
+
+/*
+ * vim: et ts=2 sw=2
+ */