summaryrefslogtreecommitdiffstats
path: root/src/http.c
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/http.c5500
1 files changed, 5500 insertions, 0 deletions
diff --git a/src/http.c b/src/http.c
new file mode 100644
index 0000000..f61c99a
--- /dev/null
+++ b/src/http.c
@@ -0,0 +1,5500 @@
+/* HTTP support.
+ Copyright (C) 1996-2012, 2014-2015, 2018-2022 Free Software
+ Foundation, Inc.
+
+This file is part of GNU Wget.
+
+GNU Wget is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+GNU Wget is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Wget. If not, see <http://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work. */
+
+#include "wget.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <assert.h>
+#include <errno.h>
+#include <time.h>
+#include <locale.h>
+#include <fcntl.h>
+
+#include "hash.h"
+#include "http.h"
+#include "hsts.h"
+#include "utils.h"
+#include "url.h"
+#include "host.h"
+#include "retr.h"
+#include "connect.h"
+#include "netrc.h"
+#ifdef HAVE_SSL
+# include "ssl.h"
+#endif
+#ifdef ENABLE_NTLM
+# include "http-ntlm.h"
+#endif
+#include "cookies.h"
+#include "md5.h"
+#include "convert.h"
+#include "spider.h"
+#include "warc.h"
+#include "c-strcase.h"
+#include "version.h"
+#include "xstrndup.h"
+#ifdef HAVE_METALINK
+# include "metalink.h"
+#endif
+#ifdef ENABLE_XATTR
+#include "xattr.h"
+#endif
+
+#ifdef TESTING
+#include "../tests/unit-tests.h"
+#endif
+
+#ifdef __VMS
+# include "vms.h"
+#endif /* def __VMS */
+
+
+/* Forward decls. */
+struct http_stat;
+static char *create_authorization_line (const char *, const char *,
+ const char *, const char *,
+ const char *, bool *, uerr_t *);
+static char *basic_authentication_encode (const char *, const char *);
+static bool known_authentication_scheme_p (const char *, const char *);
+static void ensure_extension (struct http_stat *, const char *, int *);
+static void load_cookies (void);
+
+static bool cookies_loaded_p;
+static struct cookie_jar *wget_cookie_jar;
+
+#define TEXTHTML_S "text/html"
+#define TEXTXHTML_S "application/xhtml+xml"
+#define TEXTCSS_S "text/css"
+
+/* Some status code validation macros: */
+#define H_10X(x) (((x) >= 100) && ((x) < 200))
+#define H_20X(x) (((x) >= 200) && ((x) < 300))
+#define H_PARTIAL(x) ((x) == HTTP_STATUS_PARTIAL_CONTENTS)
+#define H_REDIRECTED(x) ((x) == HTTP_STATUS_MOVED_PERMANENTLY \
+ || (x) == HTTP_STATUS_MOVED_TEMPORARILY \
+ || (x) == HTTP_STATUS_SEE_OTHER \
+ || (x) == HTTP_STATUS_TEMPORARY_REDIRECT \
+ || (x) == HTTP_STATUS_PERMANENT_REDIRECT)
+
+/* HTTP/1.0 status codes from RFC1945, provided for reference. */
+/* Successful 2xx. */
+#define HTTP_STATUS_OK 200
+#define HTTP_STATUS_CREATED 201
+#define HTTP_STATUS_ACCEPTED 202
+#define HTTP_STATUS_NO_CONTENT 204
+#define HTTP_STATUS_PARTIAL_CONTENTS 206
+
+/* Redirection 3xx. */
+#define HTTP_STATUS_MULTIPLE_CHOICES 300
+#define HTTP_STATUS_MOVED_PERMANENTLY 301
+#define HTTP_STATUS_MOVED_TEMPORARILY 302
+#define HTTP_STATUS_SEE_OTHER 303 /* from HTTP/1.1 */
+#define HTTP_STATUS_NOT_MODIFIED 304
+#define HTTP_STATUS_TEMPORARY_REDIRECT 307 /* from HTTP/1.1 */
+#define HTTP_STATUS_PERMANENT_REDIRECT 308 /* from HTTP/1.1 */
+
+/* Client error 4xx. */
+#define HTTP_STATUS_BAD_REQUEST 400
+#define HTTP_STATUS_UNAUTHORIZED 401
+#define HTTP_STATUS_FORBIDDEN 403
+#define HTTP_STATUS_NOT_FOUND 404
+#define HTTP_STATUS_RANGE_NOT_SATISFIABLE 416
+
+/* Server errors 5xx. */
+#define HTTP_STATUS_INTERNAL 500
+#define HTTP_STATUS_NOT_IMPLEMENTED 501
+#define HTTP_STATUS_BAD_GATEWAY 502
+#define HTTP_STATUS_UNAVAILABLE 503
+#define HTTP_STATUS_GATEWAY_TIMEOUT 504
+
+enum rp {
+ rel_none, rel_name, rel_value, rel_both
+};
+
+struct request {
+ const char *method;
+ char *arg;
+
+ struct request_header {
+ char *name, *value;
+ enum rp release_policy;
+ } *headers;
+ int hcount, hcapacity;
+};
+
+
+/* Create a new, empty request. Set the request's method and its
+ arguments. METHOD should be a literal string (or it should outlive
+ the request) because it will not be freed. ARG will be freed by
+ request_free. */
+
+static struct request *
+request_new (const char *method, char *arg)
+{
+ struct request *req = xnew0 (struct request);
+ req->hcapacity = 8;
+ req->headers = xnew_array (struct request_header, req->hcapacity);
+ req->method = method;
+ req->arg = arg;
+ return req;
+}
+
+/* Return the method string passed with the last call to
+ request_set_method. */
+
+static const char *
+request_method (const struct request *req)
+{
+ return req->method;
+}
+
+/* Free one header according to the release policy specified with
+ request_set_header. */
+
+static void
+release_header (struct request_header *hdr)
+{
+ switch (hdr->release_policy)
+ {
+ case rel_none:
+ break;
+ case rel_name:
+ xfree (hdr->name);
+ break;
+ case rel_value:
+ xfree (hdr->value);
+ break;
+ case rel_both:
+ xfree (hdr->name);
+ xfree (hdr->value);
+ break;
+ }
+}
+
+/* Set the request named NAME to VALUE. Specifically, this means that
+ a "NAME: VALUE\r\n" header line will be used in the request. If a
+ header with the same name previously existed in the request, its
+ value will be replaced by this one. A NULL value means do nothing.
+
+ RELEASE_POLICY determines whether NAME and VALUE should be released
+ (freed) with request_free. Allowed values are:
+
+ - rel_none - don't free NAME or VALUE
+ - rel_name - free NAME when done
+ - rel_value - free VALUE when done
+ - rel_both - free both NAME and VALUE when done
+
+ Setting release policy is useful when arguments come from different
+ sources. For example:
+
+ // Don't free literal strings!
+ request_set_header (req, "Pragma", "no-cache", rel_none);
+
+ // Don't free a global variable, we'll need it later.
+ request_set_header (req, "Referer", opt.referer, rel_none);
+
+ // Value freshly allocated, free it when done.
+ request_set_header (req, "Range",
+ aprintf ("bytes=%s-", number_to_static_string (hs->restval)),
+ rel_value);
+ */
+
+static void
+request_set_header (struct request *req, const char *name, const char *value,
+ enum rp release_policy)
+{
+ struct request_header *hdr;
+ int i;
+
+ if (!value)
+ {
+ /* A NULL value is a no-op; if freeing the name is requested,
+ free it now to avoid leaks. */
+ if (release_policy == rel_name || release_policy == rel_both)
+ xfree (name);
+ return;
+ }
+
+ for (i = 0; i < req->hcount; i++)
+ {
+ hdr = &req->headers[i];
+ if (0 == c_strcasecmp (name, hdr->name))
+ {
+ /* Replace existing header. */
+ release_header (hdr);
+ hdr->name = (void *)name;
+ hdr->value = (void *)value;
+ hdr->release_policy = release_policy;
+ return;
+ }
+ }
+
+ /* Install new header. */
+
+ if (req->hcount >= req->hcapacity)
+ {
+ req->hcapacity <<= 1;
+ req->headers = xrealloc (req->headers, req->hcapacity * sizeof (*hdr));
+ }
+ hdr = &req->headers[req->hcount++];
+ hdr->name = (void *)name;
+ hdr->value = (void *)value;
+ hdr->release_policy = release_policy;
+}
+
+/* Like request_set_header, but sets the whole header line, as
+ provided by the user using the `--header' option. For example,
+ request_set_user_header (req, "Foo: bar") works just like
+ request_set_header (req, "Foo", "bar"). */
+
+static void
+request_set_user_header (struct request *req, const char *header)
+{
+ const char *name, *p;
+
+ if (!(p = strchr (header, ':')))
+ return;
+
+ name = xstrndup(header, p - header);
+
+ ++p;
+ while (c_isspace (*p))
+ ++p;
+
+ request_set_header (req, name, p, rel_name);
+}
+
+/* Remove the header with specified name from REQ. Returns true if
+ the header was actually removed, false otherwise. */
+
+static bool
+request_remove_header (struct request *req, const char *name)
+{
+ int i;
+ for (i = 0; i < req->hcount; i++)
+ {
+ struct request_header *hdr = &req->headers[i];
+ if (0 == c_strcasecmp (name, hdr->name))
+ {
+ release_header (hdr);
+ /* Move the remaining headers by one. */
+ if (i < req->hcount - 1)
+ memmove (hdr, hdr + 1, (req->hcount - i - 1) * sizeof (*hdr));
+ --req->hcount;
+ return true;
+ }
+ }
+ return false;
+}
+
+#define APPEND(p, str) do { \
+ int A_len = strlen (str); \
+ memcpy (p, str, A_len); \
+ p += A_len; \
+} while (0)
+
+/* Construct the request and write it to FD using fd_write.
+ If warc_tmp is set to a file pointer, the request string will
+ also be written to that file. */
+
+static int
+request_send (const struct request *req, int fd, FILE *warc_tmp)
+{
+ char *request_string, *p;
+ int i, size, write_error;
+
+ /* Count the request size. */
+ size = 0;
+
+ /* METHOD " " ARG " " "HTTP/1.0" "\r\n" */
+ size += strlen (req->method) + 1 + strlen (req->arg) + 1 + 8 + 2;
+
+ for (i = 0; i < req->hcount; i++)
+ {
+ struct request_header *hdr = &req->headers[i];
+ /* NAME ": " VALUE "\r\n" */
+ size += strlen (hdr->name) + 2 + strlen (hdr->value) + 2;
+ }
+
+ /* "\r\n\0" */
+ size += 3;
+
+ p = request_string = xmalloc (size);
+
+ /* Generate the request. */
+
+ APPEND (p, req->method); *p++ = ' ';
+ APPEND (p, req->arg); *p++ = ' ';
+ memcpy (p, "HTTP/1.1\r\n", 10); p += 10;
+
+ for (i = 0; i < req->hcount; i++)
+ {
+ struct request_header *hdr = &req->headers[i];
+ APPEND (p, hdr->name);
+ *p++ = ':', *p++ = ' ';
+ APPEND (p, hdr->value);
+ *p++ = '\r', *p++ = '\n';
+ }
+
+ *p++ = '\r', *p++ = '\n', *p++ = '\0';
+ assert (p - request_string == size);
+
+#undef APPEND
+
+ DEBUGP (("\n---request begin---\n%s---request end---\n", request_string));
+
+ /* Send the request to the server. */
+
+ write_error = fd_write (fd, request_string, size - 1, -1);
+ if (write_error < 0)
+ logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"),
+ fd_errstr (fd));
+ else if (warc_tmp != NULL)
+ {
+ /* Write a copy of the data to the WARC record. */
+ int warc_tmp_written = fwrite (request_string, 1, size - 1, warc_tmp);
+ if (warc_tmp_written != size - 1)
+ write_error = -2;
+ }
+ xfree (request_string);
+ return write_error;
+}
+
+/* Release the resources used by REQ.
+ It is safe to call it with a valid pointer to a NULL pointer.
+ It is not safe to call it with an invalid or NULL pointer. */
+
+static void
+request_free (struct request **req_ref)
+{
+ int i;
+ struct request *req = *req_ref;
+
+ if (!req)
+ return;
+
+ xfree (req->arg);
+ for (i = 0; i < req->hcount; i++)
+ release_header (&req->headers[i]);
+ xfree (req->headers);
+ xfree (req);
+ *req_ref = NULL;
+}
+
+static struct hash_table *basic_authed_hosts;
+
+/* Find out if this host has issued a Basic challenge yet; if so, give
+ * it the username, password. A temporary measure until we can get
+ * proper authentication in place. */
+
+static bool
+maybe_send_basic_creds (const char *hostname, const char *user,
+ const char *passwd, struct request *req)
+{
+ bool do_challenge = false;
+
+ if (opt.auth_without_challenge)
+ {
+ DEBUGP (("Auth-without-challenge set, sending Basic credentials.\n"));
+ do_challenge = true;
+ }
+ else if (basic_authed_hosts
+ && hash_table_contains (basic_authed_hosts, hostname))
+ {
+ DEBUGP (("Found %s in basic_authed_hosts.\n", quote (hostname)));
+ do_challenge = true;
+ }
+ else
+ {
+ DEBUGP (("Host %s has not issued a general basic challenge.\n",
+ quote (hostname)));
+ }
+ if (do_challenge)
+ {
+ request_set_header (req, "Authorization",
+ basic_authentication_encode (user, passwd),
+ rel_value);
+ }
+ return do_challenge;
+}
+
+static void
+register_basic_auth_host (const char *hostname)
+{
+ if (!basic_authed_hosts)
+ {
+ basic_authed_hosts = make_nocase_string_hash_table (1);
+ }
+ if (!hash_table_contains (basic_authed_hosts, hostname))
+ {
+ hash_table_put (basic_authed_hosts, xstrdup (hostname), NULL);
+ DEBUGP (("Inserted %s into basic_authed_hosts\n", quote (hostname)));
+ }
+}
+
+/* Send the contents of FILE_NAME to SOCK. Make sure that exactly
+ PROMISED_SIZE bytes are sent over the wire -- if the file is
+ longer, read only that much; if the file is shorter, report an error.
+ If warc_tmp is set to a file pointer, the post data will
+ also be written to that file. */
+
+static int
+body_file_send (int sock, const char *file_name, wgint promised_size, FILE *warc_tmp)
+{
+ static char chunk[8192];
+ wgint written = 0;
+ int write_error;
+ FILE *fp;
+
+ DEBUGP (("[writing BODY file %s ... ", file_name));
+
+ fp = fopen (file_name, "rb");
+ if (!fp)
+ return -1;
+ while (!feof (fp) && written < promised_size)
+ {
+ int towrite;
+ int length = fread (chunk, 1, sizeof (chunk), fp);
+ if (length == 0)
+ break;
+ towrite = MIN (promised_size - written, length);
+ write_error = fd_write (sock, chunk, towrite, -1);
+ if (write_error < 0)
+ {
+ fclose (fp);
+ return -1;
+ }
+ if (warc_tmp != NULL)
+ {
+ /* Write a copy of the data to the WARC record. */
+ int warc_tmp_written = fwrite (chunk, 1, towrite, warc_tmp);
+ if (warc_tmp_written != towrite)
+ {
+ fclose (fp);
+ return -2;
+ }
+ }
+ written += towrite;
+ }
+ fclose (fp);
+
+ /* If we've written less than was promised, report a (probably
+ nonsensical) error rather than break the promise. */
+ if (written < promised_size)
+ {
+ errno = EINVAL;
+ return -1;
+ }
+
+ assert (written == promised_size);
+ DEBUGP (("done]\n"));
+ return 0;
+}
+
+/* Determine whether [START, PEEKED + PEEKLEN) contains an empty line.
+ If so, return the pointer to the position after the line, otherwise
+ return NULL. This is used as callback to fd_read_hunk. The data
+ between START and PEEKED has been read and cannot be "unread"; the
+ data after PEEKED has only been peeked. */
+
+static const char *
+response_head_terminator (const char *start, const char *peeked, int peeklen)
+{
+ const char *p, *end;
+
+ /* If at first peek, verify whether HUNK starts with "HTTP". If
+ not, this is a HTTP/0.9 request and we must bail out without
+ reading anything. */
+ if (start == peeked && 0 != memcmp (start, "HTTP", MIN (peeklen, 4)))
+ return start;
+
+ /* Look for "\n[\r]\n", and return the following position if found.
+ Start two chars before the current to cover the possibility that
+ part of the terminator (e.g. "\n\r") arrived in the previous
+ batch. */
+ p = peeked - start < 2 ? start : peeked - 2;
+ end = peeked + peeklen;
+
+ /* Check for \n\r\n or \n\n anywhere in [p, end-2). */
+ for (; p < end - 2; p++)
+ if (*p == '\n')
+ {
+ if (p[1] == '\r' && p[2] == '\n')
+ return p + 3;
+ else if (p[1] == '\n')
+ return p + 2;
+ }
+ /* p==end-2: check for \n\n directly preceding END. */
+ if (peeklen >= 2 && p[0] == '\n' && p[1] == '\n')
+ return p + 2;
+
+ return NULL;
+}
+
+/* The maximum size of a single HTTP response we care to read. Rather
+ than being a limit of the reader implementation, this limit
+ prevents Wget from slurping all available memory upon encountering
+ malicious or buggy server output, thus protecting the user. Define
+ it to 0 to remove the limit. */
+
+#define HTTP_RESPONSE_MAX_SIZE 65536
+
+/* Read the HTTP request head from FD and return it. The error
+ conditions are the same as with fd_read_hunk.
+
+ To support HTTP/0.9 responses, this function tries to make sure
+ that the data begins with "HTTP". If this is not the case, no data
+ is read and an empty request is returned, so that the remaining
+ data can be treated as body. */
+
+static char *
+read_http_response_head (int fd)
+{
+ return fd_read_hunk (fd, response_head_terminator, 512,
+ HTTP_RESPONSE_MAX_SIZE);
+}
+
+struct response {
+ /* The response data. */
+ const char *data;
+
+ /* The array of pointers that indicate where each header starts.
+ For example, given this HTTP response:
+
+ HTTP/1.0 200 Ok
+ Description: some
+ text
+ Etag: x
+
+ The headers are located like this:
+
+ "HTTP/1.0 200 Ok\r\nDescription: some\r\n text\r\nEtag: x\r\n\r\n"
+ ^ ^ ^ ^
+ headers[0] headers[1] headers[2] headers[3]
+
+ I.e. headers[0] points to the beginning of the request,
+ headers[1] points to the end of the first header and the
+ beginning of the second one, etc. */
+
+ const char **headers;
+};
+
+/* Create a new response object from the text of the HTTP response,
+ available in HEAD. That text is automatically split into
+ constituent header lines for fast retrieval using
+ resp_header_*. */
+
+static struct response *
+resp_new (char *head)
+{
+ char *hdr;
+ int count, size;
+
+ struct response *resp = xnew0 (struct response);
+ resp->data = head;
+
+ if (*head == '\0')
+ {
+ /* Empty head means that we're dealing with a headerless
+ (HTTP/0.9) response. In that case, don't set HEADERS at
+ all. */
+ return resp;
+ }
+
+ /* Split HEAD into header lines, so that resp_header_* functions
+ don't need to do this over and over again. */
+
+ size = count = 0;
+ hdr = head;
+ while (1)
+ {
+ DO_REALLOC (resp->headers, size, count + 1, const char *);
+ resp->headers[count++] = hdr;
+
+ /* Break upon encountering an empty line. */
+ if (!hdr[0] || (hdr[0] == '\r' && hdr[1] == '\n') || hdr[0] == '\n')
+ break;
+
+ /* Find the end of HDR, including continuations. */
+ for (;;)
+ {
+ char *end = strchr (hdr, '\n');
+
+ if (!end)
+ {
+ hdr += strlen (hdr);
+ break;
+ }
+
+ hdr = end + 1;
+
+ if (*hdr != ' ' && *hdr != '\t')
+ break;
+
+ // continuation, transform \r and \n into spaces
+ *end = ' ';
+ if (end > head && end[-1] == '\r')
+ end[-1] = ' ';
+ }
+ }
+ DO_REALLOC (resp->headers, size, count + 1, const char *);
+ resp->headers[count] = NULL;
+
+ return resp;
+}
+
+/* Locate the header named NAME in the request data, starting with
+ position START. This allows the code to loop through the request
+ data, filtering for all requests of a given name. Returns the
+ found position, or -1 for failure. The code that uses this
+ function typically looks like this:
+
+ for (pos = 0; (pos = resp_header_locate (...)) != -1; pos++)
+ ... do something with header ...
+
+ If you only care about one header, use resp_header_get instead of
+ this function. */
+
+static int
+resp_header_locate (const struct response *resp, const char *name, int start,
+ const char **begptr, const char **endptr)
+{
+ int i;
+ const char **headers = resp->headers;
+ int name_len;
+
+ if (!headers || !headers[1])
+ return -1;
+
+ name_len = strlen (name);
+ if (start > 0)
+ i = start;
+ else
+ i = 1;
+
+ for (; headers[i + 1]; i++)
+ {
+ const char *b = headers[i];
+ const char *e = headers[i + 1];
+ if (e - b > name_len
+ && b[name_len] == ':'
+ && 0 == c_strncasecmp (b, name, name_len))
+ {
+ b += name_len + 1;
+ while (b < e && c_isspace (*b))
+ ++b;
+ while (b < e && c_isspace (e[-1]))
+ --e;
+ *begptr = b;
+ *endptr = e;
+ return i;
+ }
+ }
+ return -1;
+}
+
+/* Find and retrieve the header named NAME in the request data. If
+ found, set *BEGPTR to its starting, and *ENDPTR to its ending
+ position, and return true. Otherwise return false.
+
+ This function is used as a building block for resp_header_copy
+ and resp_header_strdup. */
+
+static bool
+resp_header_get (const struct response *resp, const char *name,
+ const char **begptr, const char **endptr)
+{
+ int pos = resp_header_locate (resp, name, 0, begptr, endptr);
+ return pos != -1;
+}
+
+/* Copy the response header named NAME to buffer BUF, no longer than
+ BUFSIZE (BUFSIZE includes the terminating 0). If the header
+ exists, true is returned, false otherwise. If there should be no
+ limit on the size of the header, use resp_header_strdup instead.
+
+ If BUFSIZE is 0, no data is copied, but the boolean indication of
+ whether the header is present is still returned. */
+
+static bool
+resp_header_copy (const struct response *resp, const char *name,
+ char *buf, int bufsize)
+{
+ const char *b, *e;
+ if (!resp_header_get (resp, name, &b, &e))
+ return false;
+ if (bufsize)
+ {
+ int len = MIN (e - b, bufsize - 1);
+ memcpy (buf, b, len);
+ buf[len] = '\0';
+ }
+ return true;
+}
+
+/* Return the value of header named NAME in RESP, allocated with
+ malloc. If such a header does not exist in RESP, return NULL. */
+
+static char *
+resp_header_strdup (const struct response *resp, const char *name)
+{
+ const char *b, *e;
+ if (!resp_header_get (resp, name, &b, &e))
+ return NULL;
+ return strdupdelim (b, e);
+}
+
+/* Parse the HTTP status line, which is of format:
+
+ HTTP-Version SP Status-Code SP Reason-Phrase
+
+ The function returns the status-code, or -1 if the status line
+ appears malformed. The pointer to "reason-phrase" message is
+ returned in *MESSAGE. */
+
+static int
+resp_status (const struct response *resp, char **message)
+{
+ int status;
+ const char *p, *end;
+
+ if (!resp->headers)
+ {
+ /* For a HTTP/0.9 response, assume status 200. */
+ if (message)
+ *message = xstrdup (_("No headers, assuming HTTP/0.9"));
+ return 200;
+ }
+
+ p = resp->headers[0];
+ end = resp->headers[1];
+
+ if (!end)
+ return -1;
+
+ /* "HTTP" */
+ if (end - p < 4 || 0 != strncmp (p, "HTTP", 4))
+ return -1;
+ p += 4;
+
+ /* Match the HTTP version. This is optional because Gnutella
+ servers have been reported to not specify HTTP version. */
+ if (p < end && *p == '/')
+ {
+ ++p;
+ while (p < end && c_isdigit (*p))
+ ++p;
+ if (p < end && *p == '.')
+ ++p;
+ while (p < end && c_isdigit (*p))
+ ++p;
+ }
+
+ while (p < end && c_isspace (*p))
+ ++p;
+ if (end - p < 3 || !c_isdigit (p[0]) || !c_isdigit (p[1]) || !c_isdigit (p[2]))
+ return -1;
+
+ status = 100 * (p[0] - '0') + 10 * (p[1] - '0') + (p[2] - '0');
+ p += 3;
+
+ if (message)
+ {
+ while (p < end && c_isspace (*p))
+ ++p;
+ while (p < end && c_isspace (end[-1]))
+ --end;
+ *message = strdupdelim (p, end);
+ }
+
+ return status;
+}
+
+/* Release the resources used by RESP.
+ It is safe to call it with a valid pointer to a NULL pointer.
+ It is not safe to call it with a invalid or NULL pointer. */
+
+static void
+resp_free (struct response **resp_ref)
+{
+ struct response *resp = *resp_ref;
+
+ if (!resp)
+ return;
+
+ xfree (resp->headers);
+ xfree (resp);
+
+ *resp_ref = NULL;
+}
+
+/* Print a single line of response, the characters [b, e). We tried
+ getting away with
+ logprintf (LOG_VERBOSE, "%s%.*s\n", prefix, (int) (e - b), b);
+ but that failed to escape the non-printable characters and, in fact,
+ caused crashes in UTF-8 locales. */
+
+static void
+print_response_line (const char *prefix, const char *b, const char *e)
+{
+ char buf[1024], *copy;
+ size_t len = e - b;
+
+ if (len < sizeof (buf))
+ copy = buf;
+ else
+ copy = xmalloc(len + 1);
+
+ memcpy(copy, b, len);
+ copy[len] = 0;
+
+ logprintf (LOG_ALWAYS, "%s%s\n", prefix,
+ quotearg_style (escape_quoting_style, copy));
+
+ if (copy != buf)
+ xfree (copy);
+}
+
+/* Print the server response, line by line, omitting the trailing CRLF
+ from individual header lines, and prefixed with PREFIX. */
+
+static void
+print_server_response (const struct response *resp, const char *prefix)
+{
+ int i;
+ if (!resp->headers)
+ return;
+ for (i = 0; resp->headers[i + 1]; i++)
+ {
+ const char *b = resp->headers[i];
+ const char *e = resp->headers[i + 1];
+ /* Skip CRLF */
+ if (b < e && e[-1] == '\n')
+ --e;
+ if (b < e && e[-1] == '\r')
+ --e;
+ print_response_line (prefix, b, e);
+ }
+}
+
+/* Parse the `Content-Range' header and extract the information it
+ contains. Returns true if successful, false otherwise. */
+static bool
+parse_content_range (const char *hdr, wgint *first_byte_ptr,
+ wgint *last_byte_ptr, wgint *entity_length_ptr)
+{
+ wgint num;
+
+ /* Ancient versions of Netscape proxy server, presumably predating
+ rfc2068, sent out `Content-Range' without the "bytes"
+ specifier. */
+ if (0 == strncasecmp (hdr, "bytes", 5))
+ {
+ hdr += 5;
+ /* "JavaWebServer/1.1.1" sends "bytes: x-y/z", contrary to the
+ HTTP spec. */
+ if (*hdr == ':')
+ ++hdr;
+ while (c_isspace (*hdr))
+ ++hdr;
+ if (!*hdr)
+ return false;
+ }
+ if (!c_isdigit (*hdr))
+ return false;
+ for (num = 0; c_isdigit (*hdr); hdr++)
+ num = 10 * num + (*hdr - '0');
+ if (*hdr != '-' || !c_isdigit (*(hdr + 1)))
+ return false;
+ *first_byte_ptr = num;
+ ++hdr;
+ for (num = 0; c_isdigit (*hdr); hdr++)
+ num = 10 * num + (*hdr - '0');
+ if (*hdr != '/')
+ return false;
+ *last_byte_ptr = num;
+ if (!(c_isdigit (*(hdr + 1)) || *(hdr + 1) == '*'))
+ return false;
+ if (*last_byte_ptr < *first_byte_ptr)
+ return false;
+ ++hdr;
+ if (*hdr == '*')
+ num = -1;
+ else
+ for (num = 0; c_isdigit (*hdr); hdr++)
+ num = 10 * num + (*hdr - '0');
+ *entity_length_ptr = num;
+ if ((*entity_length_ptr <= *last_byte_ptr) && *entity_length_ptr != -1)
+ return false;
+ return true;
+}
+
+/* Read the body of the request, but don't store it anywhere and don't
+ display a progress gauge. This is useful for reading the bodies of
+ administrative responses to which we will soon issue another
+ request. The response is not useful to the user, but reading it
+ allows us to continue using the same connection to the server.
+
+ If reading fails, false is returned, true otherwise. In debug
+ mode, the body is displayed for debugging purposes. */
+
+static bool
+skip_short_body (int fd, wgint contlen, bool chunked)
+{
+ enum {
+ SKIP_SIZE = 512, /* size of the download buffer */
+ SKIP_THRESHOLD = 4096 /* the largest size we read */
+ };
+ wgint remaining_chunk_size = 0;
+ char dlbuf[SKIP_SIZE + 1];
+ dlbuf[SKIP_SIZE] = '\0'; /* so DEBUGP can safely print it */
+
+ /* If the body is too large, it makes more sense to simply close the
+ connection than to try to read the body. */
+ if (contlen > SKIP_THRESHOLD)
+ return false;
+
+ while (contlen > 0 || chunked)
+ {
+ int ret;
+ if (chunked)
+ {
+ if (remaining_chunk_size == 0)
+ {
+ char *line = fd_read_line (fd);
+ char *endl;
+ if (line == NULL)
+ break;
+
+ remaining_chunk_size = strtol (line, &endl, 16);
+ xfree (line);
+
+ if (remaining_chunk_size < 0)
+ return false;
+
+ if (remaining_chunk_size == 0)
+ {
+ line = fd_read_line (fd);
+ xfree (line);
+ break;
+ }
+ }
+
+ contlen = MIN (remaining_chunk_size, SKIP_SIZE);
+ }
+
+ DEBUGP (("Skipping %s bytes of body: [", number_to_static_string (contlen)));
+
+ ret = fd_read (fd, dlbuf, MIN (contlen, SKIP_SIZE), -1);
+ if (ret <= 0)
+ {
+ /* Don't normally report the error since this is an
+ optimization that should be invisible to the user. */
+ DEBUGP (("] aborting (%s).\n",
+ ret < 0 ? fd_errstr (fd) : "EOF received"));
+ return false;
+ }
+ contlen -= ret;
+
+ if (chunked)
+ {
+ remaining_chunk_size -= ret;
+ if (remaining_chunk_size == 0)
+ {
+ char *line = fd_read_line (fd);
+ if (line == NULL)
+ return false;
+ else
+ xfree (line);
+ }
+ }
+
+ /* Safe even if %.*s bogusly expects terminating \0 because
+ we've zero-terminated dlbuf above. */
+ DEBUGP (("%.*s", ret, dlbuf));
+ }
+
+ DEBUGP (("] done.\n"));
+ return true;
+}
+
+#define NOT_RFC2231 0
+#define RFC2231_NOENCODING 1
+#define RFC2231_ENCODING 2
+
+/* extract_param extracts the parameter name into NAME.
+ However, if the parameter name is in RFC2231 format then
+ this function adjusts NAME by stripping of the trailing
+ characters that are not part of the name but are present to
+ indicate the presence of encoding information in the value
+ or a fragment of a long parameter value
+*/
+static int
+modify_param_name (param_token *name)
+{
+ const char *delim1 = memchr (name->b, '*', name->e - name->b);
+ const char *delim2 = memrchr (name->b, '*', name->e - name->b);
+
+ int result;
+
+ if (delim1 == NULL)
+ {
+ result = NOT_RFC2231;
+ }
+ else if (delim1 == delim2)
+ {
+ if ((name->e - 1) == delim1)
+ {
+ result = RFC2231_ENCODING;
+ }
+ else
+ {
+ result = RFC2231_NOENCODING;
+ }
+ name->e = delim1;
+ }
+ else
+ {
+ name->e = delim1;
+ result = RFC2231_ENCODING;
+ }
+ return result;
+}
+
+/* extract_param extract the parameter value into VALUE.
+ Like modify_param_name this function modifies VALUE by
+ stripping off the encoding information from the actual value
+*/
+static void
+modify_param_value (param_token *value, int encoding_type )
+{
+ if (encoding_type == RFC2231_ENCODING)
+ {
+ const char *delim = memrchr (value->b, '\'', value->e - value->b);
+ if (delim != NULL)
+ {
+ value->b = (delim+1);
+ }
+ }
+}
+
+/* Extract a parameter from the string (typically an HTTP header) at
+ **SOURCE and advance SOURCE to the next parameter. Return false
+ when there are no more parameters to extract. The name of the
+ parameter is returned in NAME, and the value in VALUE. If the
+ parameter has no value, the token's value is zeroed out.
+
+ For example, if *SOURCE points to the string "attachment;
+ filename=\"foo bar\"", the first call to this function will return
+ the token named "attachment" and no value, and the second call will
+ return the token named "filename" and value "foo bar". The third
+ call will return false, indicating no more valid tokens.
+
+ is_url_encoded is an out parameter. If not NULL, a boolean value will be
+ stored into it, letting the caller know whether or not the extracted value is
+ URL-encoded. The caller can then decode it with url_unescape(), which however
+ performs decoding in-place. URL-encoding is used by RFC 2231 to support
+ non-US-ASCII characters in HTTP header values. */
+
+bool
+extract_param (const char **source, param_token *name, param_token *value,
+ char separator, bool *is_url_encoded)
+{
+ const char *p = *source;
+ int param_type;
+ if (is_url_encoded)
+ *is_url_encoded = false; /* initializing the out parameter */
+
+ while (c_isspace (*p)) ++p;
+ if (!*p)
+ {
+ *source = p;
+ return false; /* no error; nothing more to extract */
+ }
+
+ /* Extract name. */
+ name->b = p;
+ while (*p && !c_isspace (*p) && *p != '=' && *p != separator) ++p;
+ name->e = p;
+ if (name->b == name->e)
+ return false; /* empty name: error */
+ while (c_isspace (*p)) ++p;
+ if (*p == separator || !*p) /* no value */
+ {
+ xzero (*value);
+ if (*p == separator) ++p;
+ *source = p;
+ return true;
+ }
+ if (*p != '=')
+ return false; /* error */
+
+ /* *p is '=', extract value */
+ ++p;
+ while (c_isspace (*p)) ++p;
+ if (*p == '"') /* quoted */
+ {
+ value->b = ++p;
+ while (*p && *p != '"') ++p;
+ if (!*p)
+ return false;
+ value->e = p++;
+ /* Currently at closing quote; find the end of param. */
+ while (c_isspace (*p)) ++p;
+ while (*p && *p != separator) ++p;
+ if (*p == separator)
+ ++p;
+ else if (*p)
+ /* garbage after closed quote, e.g. foo="bar"baz */
+ return false;
+ }
+ else /* unquoted */
+ {
+ value->b = p;
+ while (*p && *p != separator) ++p;
+ value->e = p;
+ while (value->e != value->b && c_isspace (value->e[-1]))
+ --value->e;
+ if (*p == separator) ++p;
+ }
+ *source = p;
+
+ param_type = modify_param_name (name);
+ if (param_type != NOT_RFC2231)
+ {
+ if (param_type == RFC2231_ENCODING && is_url_encoded)
+ *is_url_encoded = true;
+ modify_param_value (value, param_type);
+ }
+ return true;
+}
+
+#undef NOT_RFC2231
+#undef RFC2231_NOENCODING
+#undef RFC2231_ENCODING
+
+/* Appends the string represented by VALUE to FILENAME */
+
+static void
+append_value_to_filename (char **filename, param_token const * const value,
+ bool is_url_encoded)
+{
+ int original_length = strlen (*filename);
+ int new_length = strlen (*filename) + (value->e - value->b);
+ *filename = xrealloc (*filename, new_length+1);
+ memcpy (*filename + original_length, value->b, (value->e - value->b));
+ (*filename)[new_length] = '\0';
+ if (is_url_encoded)
+ url_unescape (*filename + original_length);
+}
+
+/* Parse the contents of the `Content-Disposition' header, extracting
+ the information useful to Wget. Content-Disposition is a header
+ borrowed from MIME; when used in HTTP, it typically serves for
+ specifying the desired file name of the resource. For example:
+
+ Content-Disposition: attachment; filename="flora.jpg"
+
+ Wget will skip the tokens it doesn't care about, such as
+ "attachment" in the previous example; it will also skip other
+ unrecognized params. If the header is syntactically correct and
+ contains a file name, a copy of the file name is stored in
+ *filename and true is returned. Otherwise, the function returns
+ false.
+
+ The file name is stripped of directory components and must not be
+ empty.
+
+ Historically, this function returned filename prefixed with opt.dir_prefix,
+ now that logic is handled by the caller, new code should pay attention,
+ changed by crq, Sep 2010.
+
+*/
+static bool
+parse_content_disposition (const char *hdr, char **filename)
+{
+ param_token name, value;
+ bool is_url_encoded = false;
+
+ char *encodedFilename = NULL;
+ char *unencodedFilename = NULL;
+ for ( ; extract_param (&hdr, &name, &value, ';', &is_url_encoded);
+ is_url_encoded = false)
+ {
+ int isFilename = BOUNDED_EQUAL_NO_CASE (name.b, name.e, "filename");
+ if ( isFilename && value.b != NULL)
+ {
+ /* Make the file name begin at the last slash or backslash. */
+ bool isEncodedFilename;
+ char **outFilename;
+ const char *last_slash = memrchr (value.b, '/', value.e - value.b);
+ const char *last_bs = memrchr (value.b, '\\', value.e - value.b);
+ if (last_slash && last_bs)
+ value.b = 1 + MAX (last_slash, last_bs);
+ else if (last_slash || last_bs)
+ value.b = 1 + (last_slash ? last_slash : last_bs);
+ if (value.b == value.e)
+ continue;
+
+ /* Check if the name is "filename*" as specified in RFC 6266.
+ * Since "filename" could be broken up as "filename*N" (RFC 2231),
+ * a check is needed to make sure this is not the case */
+ isEncodedFilename = *name.e == '*' && !c_isdigit (*(name.e + 1));
+ outFilename = isEncodedFilename ? &encodedFilename
+ : &unencodedFilename;
+ if (*outFilename)
+ append_value_to_filename (outFilename, &value, is_url_encoded);
+ else
+ {
+ *outFilename = strdupdelim (value.b, value.e);
+ if (is_url_encoded)
+ url_unescape (*outFilename);
+ }
+ }
+ }
+ if (encodedFilename)
+ {
+ xfree (unencodedFilename);
+ *filename = encodedFilename;
+ }
+ else
+ {
+ xfree (encodedFilename);
+ *filename = unencodedFilename;
+ }
+ if (*filename)
+ return true;
+ else
+ return false;
+}
+
+#ifdef HAVE_HSTS
+static bool
+parse_strict_transport_security (const char *header, time_t *max_age, bool *include_subdomains)
+{
+ param_token name, value;
+ const char *c_max_age = NULL;
+ bool is = false; /* includeSubDomains */
+ bool is_url_encoded = false;
+ bool success = false;
+
+ if (header)
+ {
+ /* Process the STS header. Keys should be matched case-insensitively. */
+ for (; extract_param (&header, &name, &value, ';', &is_url_encoded); is_url_encoded = false)
+ {
+ if (BOUNDED_EQUAL_NO_CASE (name.b, name.e, "max-age"))
+ {
+ xfree (c_max_age);
+ c_max_age = strdupdelim (value.b, value.e);
+ }
+ else if (BOUNDED_EQUAL_NO_CASE (name.b, name.e, "includeSubDomains"))
+ is = true;
+ }
+
+ /* pass the parsed values over */
+ if (c_max_age)
+ {
+ /* If the string value goes out of a long's bounds, strtol() will return LONG_MIN or LONG_MAX.
+ * In theory, the HSTS engine should be able to handle it.
+ * Also, time_t is normally defined as a long, so this should not break.
+ */
+ if (max_age)
+ *max_age = (time_t) strtol (c_max_age, NULL, 10);
+ if (include_subdomains)
+ *include_subdomains = is;
+
+ DEBUGP (("Parsed Strict-Transport-Security max-age = %s, includeSubDomains = %s\n",
+ c_max_age, (is ? "true" : "false")));
+
+ xfree (c_max_age);
+ success = true;
+ }
+ else
+ {
+ /* something weird happened */
+ logprintf (LOG_VERBOSE, "Could not parse Strict-Transport-Security header\n");
+ success = false;
+ }
+ }
+
+ return success;
+}
+#endif
+
+/* Persistent connections. Currently, we cache the most recently used
+ connection as persistent, provided that the HTTP server agrees to
+ make it such. The persistence data is stored in the variables
+ below. Ideally, it should be possible to cache an arbitrary fixed
+ number of these connections. */
+
+/* Whether a persistent connection is active. */
+static bool pconn_active;
+
+static struct {
+ /* The socket of the connection. */
+ int socket;
+
+ /* Host and port of the currently active persistent connection. */
+ char *host;
+ int port;
+
+ /* Whether a ssl handshake has occurred on this connection. */
+ bool ssl;
+
+ /* Whether the connection was authorized. This is only done by
+ NTLM, which authorizes *connections* rather than individual
+ requests. (That practice is peculiar for HTTP, but it is a
+ useful optimization.) */
+ bool authorized;
+
+#ifdef ENABLE_NTLM
+ /* NTLM data of the current connection. */
+ struct ntlmdata ntlm;
+#endif
+} pconn;
+
+/* Mark the persistent connection as invalid and free the resources it
+ uses. This is used by the CLOSE_* macros after they forcefully
+ close a registered persistent connection. */
+
+static void
+invalidate_persistent (void)
+{
+ DEBUGP (("Disabling further reuse of socket %d.\n", pconn.socket));
+ pconn_active = false;
+ fd_close (pconn.socket);
+ xfree (pconn.host);
+ xzero (pconn);
+}
+
+/* Register FD, which should be a TCP/IP connection to HOST:PORT, as
+ persistent. This will enable someone to use the same connection
+ later. In the context of HTTP, this must be called only AFTER the
+ response has been received and the server has promised that the
+ connection will remain alive.
+
+ If a previous connection was persistent, it is closed. */
+
+static void
+register_persistent (const char *host, int port, int fd, bool ssl)
+{
+ if (pconn_active)
+ {
+ if (pconn.socket == fd)
+ {
+ /* The connection FD is already registered. */
+ return;
+ }
+ else
+ {
+ /* The old persistent connection is still active; close it
+ first. This situation arises whenever a persistent
+ connection exists, but we then connect to a different
+ host, and try to register a persistent connection to that
+ one. */
+ invalidate_persistent ();
+ }
+ }
+
+ pconn_active = true;
+ pconn.socket = fd;
+ pconn.host = xstrdup (host);
+ pconn.port = port;
+ pconn.ssl = ssl;
+ pconn.authorized = false;
+
+ DEBUGP (("Registered socket %d for persistent reuse.\n", fd));
+}
+
+/* Return true if a persistent connection is available for connecting
+ to HOST:PORT. */
+
+static bool
+persistent_available_p (const char *host, int port, bool ssl,
+ bool *host_lookup_failed)
+{
+ /* First, check whether a persistent connection is active at all. */
+ if (!pconn_active)
+ return false;
+
+ /* If we want SSL and the last connection wasn't or vice versa,
+ don't use it. Checking for host and port is not enough because
+ HTTP and HTTPS can apparently coexist on the same port. */
+ if (ssl != pconn.ssl)
+ return false;
+
+ /* If we're not connecting to the same port, we're not interested. */
+ if (port != pconn.port)
+ return false;
+
+ /* If the host is the same, we're in business. If not, there is
+ still hope -- read below. */
+ if (0 != strcasecmp (host, pconn.host))
+ {
+ /* Check if pconn.socket is talking to HOST under another name.
+ This happens often when both sites are virtual hosts
+ distinguished only by name and served by the same network
+ interface, and hence the same web server (possibly set up by
+ the ISP and serving many different web sites). This
+ admittedly unconventional optimization does not contradict
+ HTTP and works well with popular server software. */
+
+ bool found;
+ ip_address ip;
+ struct address_list *al;
+
+ if (ssl)
+ /* Don't try to talk to two different SSL sites over the same
+ secure connection! (Besides, it's not clear that
+ name-based virtual hosting is even possible with SSL.) */
+ return false;
+
+ /* If pconn.socket's peer is one of the IP addresses HOST
+ resolves to, pconn.socket is for all intents and purposes
+ already talking to HOST. */
+
+ if (!socket_ip_address (pconn.socket, &ip, ENDPOINT_PEER))
+ {
+ /* Can't get the peer's address -- something must be very
+ wrong with the connection. */
+ invalidate_persistent ();
+ return false;
+ }
+ al = lookup_host (host, 0);
+ if (!al)
+ {
+ *host_lookup_failed = true;
+ return false;
+ }
+
+ found = address_list_contains (al, &ip);
+ address_list_release (al);
+
+ if (!found)
+ return false;
+
+ /* The persistent connection's peer address was found among the
+ addresses HOST resolved to; therefore, pconn.sock is in fact
+ already talking to HOST -- no need to reconnect. */
+ }
+
+ /* Finally, check whether the connection is still open. This is
+ important because most servers implement liberal (short) timeout
+ on persistent connections. Wget can of course always reconnect
+ if the connection doesn't work out, but it's nicer to know in
+ advance. This test is a logical followup of the first test, but
+ is "expensive" and therefore placed at the end of the list.
+
+ (Current implementation of test_socket_open has a nice side
+ effect that it treats sockets with pending data as "closed".
+ This is exactly what we want: if a broken server sends message
+ body in response to HEAD, or if it sends more than conent-length
+ data, we won't reuse the corrupted connection.) */
+
+ if (!test_socket_open (pconn.socket))
+ {
+ /* Oops, the socket is no longer open. Now that we know that,
+ let's invalidate the persistent connection before returning
+ 0. */
+ invalidate_persistent ();
+ return false;
+ }
+
+ return true;
+}
+
+/* The idea behind these two CLOSE macros is to distinguish between
+ two cases: one when the job we've been doing is finished, and we
+ want to close the connection and leave, and two when something is
+ seriously wrong and we're closing the connection as part of
+ cleanup.
+
+ In case of keep_alive, CLOSE_FINISH should leave the connection
+ open, while CLOSE_INVALIDATE should still close it.
+
+ Note that the semantics of the flag `keep_alive' is "this
+ connection *will* be reused (the server has promised not to close
+ the connection once we're done)", while the semantics of
+ `pc_active_p && (fd) == pc_last_fd' is "we're *now* using an
+ active, registered connection". */
+
+#define CLOSE_FINISH(fd) do { \
+ if (!keep_alive) \
+ { \
+ if (pconn_active && (fd) == pconn.socket) \
+ invalidate_persistent (); \
+ else \
+ fd_close (fd); \
+ fd = -1; \
+ } \
+} while (0)
+
+#define CLOSE_INVALIDATE(fd) do { \
+ if (pconn_active && (fd) == pconn.socket) \
+ invalidate_persistent (); \
+ else \
+ fd_close (fd); \
+ fd = -1; \
+} while (0)
+
+typedef enum
+{
+ ENC_INVALID = -1, /* invalid encoding */
+ ENC_NONE = 0, /* no special encoding */
+ ENC_GZIP, /* gzip compression */
+ ENC_DEFLATE, /* deflate compression */
+ ENC_COMPRESS, /* compress compression */
+ ENC_BROTLI /* brotli compression */
+} encoding_t;
+
+struct http_stat
+{
+ wgint len; /* received length */
+ wgint contlen; /* expected length */
+ wgint restval; /* the restart value */
+ int res; /* the result of last read */
+ char *rderrmsg; /* error message from read error */
+ char *newloc; /* new location (redirection) */
+ char *remote_time; /* remote time-stamp string */
+ char *error; /* textual HTTP error */
+ int statcode; /* status code */
+ char *message; /* status message */
+ wgint rd_size; /* amount of data read from socket */
+ double dltime; /* time it took to download the data */
+ const char *referer; /* value of the referer header. */
+ char *local_file; /* local file name. */
+ bool existence_checked; /* true if we already checked for a file's
+ existence after having begun to download
+ (needed in gethttp for when connection is
+ interrupted/restarted. */
+ bool timestamp_checked; /* true if pre-download time-stamping checks
+ * have already been performed */
+ char *orig_file_name; /* name of file to compare for time-stamping
+ * (might be != local_file if -K is set) */
+ wgint orig_file_size; /* size of file to compare for time-stamping */
+ time_t orig_file_tstamp; /* time-stamp of file to compare for
+ * time-stamping */
+#ifdef HAVE_METALINK
+ metalink_t *metalink;
+#endif
+
+ encoding_t local_encoding; /* the encoding of the local file */
+ encoding_t remote_encoding; /* the encoding of the remote file */
+
+ bool temporary; /* downloading a temporary file */
+};
+
+static void
+free_hstat (struct http_stat *hs)
+{
+ xfree (hs->newloc);
+ xfree (hs->remote_time);
+ xfree (hs->error);
+ xfree (hs->rderrmsg);
+ xfree (hs->local_file);
+ xfree (hs->orig_file_name);
+ xfree (hs->message);
+#ifdef HAVE_METALINK
+ metalink_delete (hs->metalink);
+ hs->metalink = NULL;
+#endif
+}
+
+static void
+get_file_flags (const char *filename, int *dt)
+{
+ logprintf (LOG_VERBOSE, _("\
+File %s already there; not retrieving.\n\n"), quote (filename));
+ /* If the file is there, we suppose it's retrieved OK. */
+ *dt |= RETROKF;
+
+ /* #### Bogusness alert. */
+ /* If its suffix is "html" or "htm" or similar, assume text/html. */
+ if (has_html_suffix_p (filename))
+ *dt |= TEXTHTML;
+}
+
+/* Download the response body from the socket and writes it to
+ an output file. The headers have already been read from the
+ socket. If WARC is enabled, the response body will also be
+ written to a WARC response record.
+
+ hs, contlen, contrange, chunked_transfer_encoding and url are
+ parameters from the gethttp method. fp is a pointer to the
+ output file.
+
+ url, warc_timestamp_str, warc_request_uuid, warc_ip, type
+ and statcode will be saved in the headers of the WARC record.
+ The head parameter contains the HTTP headers of the response.
+
+ If fp is NULL and WARC is enabled, the response body will be
+ written only to the WARC file. If WARC is disabled and fp
+ is a file pointer, the data will be written to the file.
+ If fp is a file pointer and WARC is enabled, the body will
+ be written to both destinations.
+
+ Returns the error code. */
+static int
+read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen,
+ wgint contrange, bool chunked_transfer_encoding,
+ char *url, char *warc_timestamp_str, char *warc_request_uuid,
+ ip_address *warc_ip, char *type, int statcode, char *head)
+{
+ int warc_payload_offset = 0;
+ FILE *warc_tmp = NULL;
+ int warcerr = 0;
+ int flags = 0;
+
+ if (opt.warc_filename != NULL)
+ {
+ /* Open a temporary file where we can write the response before we
+ add it to the WARC record. */
+ warc_tmp = warc_tempfile ();
+ if (warc_tmp == NULL)
+ warcerr = WARC_TMP_FOPENERR;
+
+ if (warcerr == 0)
+ {
+ /* We should keep the response headers for the WARC record. */
+ int head_len = strlen (head);
+ int warc_tmp_written = fwrite (head, 1, head_len, warc_tmp);
+ if (warc_tmp_written != head_len)
+ warcerr = WARC_TMP_FWRITEERR;
+ warc_payload_offset = head_len;
+ }
+
+ if (warcerr != 0)
+ {
+ if (warc_tmp != NULL)
+ fclose (warc_tmp);
+ return warcerr;
+ }
+ }
+
+ if (fp != NULL)
+ {
+ /* This confuses the timestamping code that checks for file size.
+ #### The timestamping code should be smarter about file size. */
+ if (opt.save_headers && hs->restval == 0)
+ fwrite (head, 1, strlen (head), fp);
+ }
+
+ /* Read the response body. */
+ if (contlen != -1)
+ /* If content-length is present, read that much; otherwise, read
+ until EOF. The HTTP spec doesn't require the server to
+ actually close the connection when it's done sending data. */
+ flags |= rb_read_exactly;
+ if (fp != NULL && hs->restval > 0 && contrange == 0)
+ /* If the server ignored our range request, instruct fd_read_body
+ to skip the first RESTVAL bytes of body. */
+ flags |= rb_skip_startpos;
+ if (chunked_transfer_encoding)
+ flags |= rb_chunked_transfer_encoding;
+
+ if (hs->remote_encoding == ENC_GZIP)
+ flags |= rb_compressed_gzip;
+
+ hs->len = hs->restval;
+ hs->rd_size = 0;
+ /* Download the response body and write it to fp.
+ If we are working on a WARC file, we simultaneously write the
+ response body to warc_tmp. */
+ hs->res = fd_read_body (hs->local_file, sock, fp, contlen != -1 ? contlen : 0,
+ hs->restval, &hs->rd_size, &hs->len, &hs->dltime,
+ flags, warc_tmp);
+ if (hs->res >= 0)
+ {
+ if (warc_tmp != NULL)
+ {
+ /* Create a response record and write it to the WARC file.
+ Note: per the WARC standard, the request and response should share
+ the same date header. We re-use the timestamp of the request.
+ The response record should also refer to the uuid of the request. */
+ bool r = warc_write_response_record (url, warc_timestamp_str,
+ warc_request_uuid, warc_ip,
+ warc_tmp, warc_payload_offset,
+ type, statcode, hs->newloc);
+
+ /* warc_write_response_record has closed warc_tmp. */
+
+ if (! r)
+ return WARC_ERR;
+ }
+
+ return RETRFINISHED;
+ }
+
+ if (warc_tmp != NULL)
+ fclose (warc_tmp);
+
+ if (hs->res == -2)
+ {
+ /* Error while writing to fd. */
+ return FWRITEERR;
+ }
+ else if (hs->res == -3)
+ {
+ /* Error while writing to warc_tmp. */
+ return WARC_TMP_FWRITEERR;
+ }
+ else
+ {
+ /* A read error! */
+ xfree (hs->rderrmsg);
+ hs->rderrmsg = xstrdup (fd_errstr (sock));
+ return RETRFINISHED;
+ }
+}
+
+#define BEGINS_WITH(line, string_constant) \
+ (!c_strncasecmp (line, string_constant, sizeof (string_constant) - 1) \
+ && (c_isspace (line[sizeof (string_constant) - 1]) \
+ || !line[sizeof (string_constant) - 1]))
+
+#define SET_USER_AGENT(req) do { \
+ if (!opt.useragent) \
+ request_set_header (req, "User-Agent", \
+ aprintf ("Wget/%s", \
+ version_string), \
+ rel_value); \
+ else if (*opt.useragent) \
+ request_set_header (req, "User-Agent", opt.useragent, rel_none); \
+} while (0)
+
+/*
+ Convert time_t to one of valid HTTP date formats
+ ie. rfc1123-date.
+
+ HTTP-date = rfc1123-date | rfc850-date | asctime-date
+ rfc1123-date = wkday "," SP date1 SP time SP "GMT"
+ rfc850-date = weekday "," SP date2 SP time SP "GMT"
+ asctime-date = wkday SP date3 SP time SP 4DIGIT
+ date1 = 2DIGIT SP month SP 4DIGIT
+ ; day month year (e.g., 02 Jun 1982)
+ date2 = 2DIGIT "-" month "-" 2DIGIT
+ ; day-month-year (e.g., 02-Jun-82)
+ date3 = month SP ( 2DIGIT | ( SP 1DIGIT ))
+ ; month day (e.g., Jun 2)
+ time = 2DIGIT ":" 2DIGIT ":" 2DIGIT
+ ; 00:00:00 - 23:59:59
+ wkday = "Mon" | "Tue" | "Wed"
+ | "Thu" | "Fri" | "Sat" | "Sun"
+ weekday = "Monday" | "Tuesday" | "Wednesday"
+ | "Thursday" | "Friday" | "Saturday" | "Sunday"
+ month = "Jan" | "Feb" | "Mar" | "Apr"
+ | "May" | "Jun" | "Jul" | "Aug"
+ | "Sep" | "Oct" | "Nov" | "Dec"
+
+ source: RFC2616 */
+static uerr_t
+time_to_rfc1123 (time_t time, char *buf, size_t bufsize)
+{
+ static const char *wkday[] = { "Sun", "Mon", "Tue", "Wed",
+ "Thu", "Fri", "Sat" };
+ static const char *month[] = { "Jan", "Feb", "Mar", "Apr",
+ "May", "Jun", "Jul", "Aug",
+ "Sep", "Oct", "Nov", "Dec" };
+ /* rfc1123 example: Thu, 01 Jan 1998 22:12:57 GMT */
+ static const char *time_format = "%s, %02d %s %04d %02d:%02d:%02d GMT";
+
+ struct tm *gtm = gmtime (&time);
+ if (!gtm)
+ {
+ logprintf (LOG_NOTQUIET,
+ _("gmtime failed. This is probably a bug.\n"));
+ return TIMECONV_ERR;
+ }
+
+ snprintf (buf, bufsize, time_format, wkday[gtm->tm_wday],
+ gtm->tm_mday, month[gtm->tm_mon],
+ gtm->tm_year + 1900, gtm->tm_hour,
+ gtm->tm_min, gtm->tm_sec);
+
+ return RETROK;
+}
+
+static struct request *
+initialize_request (const struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
+ bool inhibit_keep_alive, bool *basic_auth_finished,
+ wgint *body_data_size, char **user, char **passwd, uerr_t *ret)
+{
+ bool head_only = !!(*dt & HEAD_ONLY);
+ struct request *req;
+
+ /* Prepare the request to send. */
+ {
+ char *meth_arg;
+ const char *meth = "GET";
+ if (head_only)
+ meth = "HEAD";
+ else if (opt.method)
+ meth = opt.method;
+ /* Use the full path, i.e. one that includes the leading slash and
+ the query string. E.g. if u->path is "foo/bar" and u->query is
+ "param=value", full_path will be "/foo/bar?param=value". */
+ if (proxy
+#ifdef HAVE_SSL
+ /* When using SSL over proxy, CONNECT establishes a direct
+ connection to the HTTPS server. Therefore use the same
+ argument as when talking to the server directly. */
+ && u->scheme != SCHEME_HTTPS
+#endif
+ )
+ meth_arg = xstrdup (u->url);
+ else
+ meth_arg = url_full_path (u);
+ req = request_new (meth, meth_arg);
+ }
+
+ /* Generate the Host header, HOST:PORT. Take into account that:
+
+ - Broken server-side software often doesn't recognize the PORT
+ argument, so we must generate "Host: www.server.com" instead of
+ "Host: www.server.com:80" (and likewise for https port).
+
+ - IPv6 addresses contain ":", so "Host: 3ffe:8100:200:2::2:1234"
+ becomes ambiguous and needs to be rewritten as "Host:
+ [3ffe:8100:200:2::2]:1234". */
+ {
+ /* Formats arranged for hfmt[add_port][add_squares]. */
+ static const char *hfmt[][2] = {
+ { "%s", "[%s]" }, { "%s:%d", "[%s]:%d" }
+ };
+ int add_port = u->port != scheme_default_port (u->scheme);
+ int add_squares = strchr (u->host, ':') != NULL;
+ request_set_header (req, "Host",
+ aprintf (hfmt[add_port][add_squares], u->host, u->port),
+ rel_value);
+ }
+
+ request_set_header (req, "Referer", hs->referer, rel_none);
+ if (*dt & SEND_NOCACHE)
+ {
+ /* Cache-Control MUST be obeyed by all HTTP/1.1 caching mechanisms... */
+ request_set_header (req, "Cache-Control", "no-cache", rel_none);
+
+ /* ... but some HTTP/1.0 caches doesn't implement Cache-Control. */
+ request_set_header (req, "Pragma", "no-cache", rel_none);
+ }
+ if (*dt & IF_MODIFIED_SINCE)
+ {
+ char strtime[32];
+ uerr_t err = time_to_rfc1123 (hs->orig_file_tstamp, strtime, countof (strtime));
+
+ if (err != RETROK)
+ {
+ logputs (LOG_VERBOSE, _("Cannot convert timestamp to http format. "
+ "Falling back to time 0 as last modification "
+ "time.\n"));
+ strcpy (strtime, "Thu, 01 Jan 1970 00:00:00 GMT");
+ }
+ request_set_header (req, "If-Modified-Since", xstrdup (strtime), rel_value);
+ }
+ if (hs->restval)
+ request_set_header (req, "Range",
+ aprintf ("bytes=%s-",
+ number_to_static_string (hs->restval)),
+ rel_value);
+ SET_USER_AGENT (req);
+ request_set_header (req, "Accept", "*/*", rel_none);
+#ifdef HAVE_LIBZ
+ if (opt.compression != compression_none)
+ request_set_header (req, "Accept-Encoding", "gzip", rel_none);
+ else
+#endif
+ request_set_header (req, "Accept-Encoding", "identity", rel_none);
+
+ /* Find the username with priority */
+ if (u->user)
+ *user = u->user;
+ else if (opt.user && (opt.use_askpass || opt.ask_passwd))
+ *user = opt.user;
+ else if (opt.http_user)
+ *user = opt.http_user;
+ else if (opt.user)
+ *user = opt.user;
+ else
+ *user = NULL;
+
+ /* Find the password with priority */
+ if (u->passwd)
+ *passwd = u->passwd;
+ else if (opt.passwd && (opt.use_askpass || opt.ask_passwd))
+ *passwd = opt.passwd;
+ else if (opt.http_passwd)
+ *passwd = opt.http_passwd;
+ else if (opt.passwd)
+ *passwd = opt.passwd;
+ else
+ *passwd = NULL;
+
+ /* Check for ~/.netrc if none of the above match */
+ if (opt.netrc && (!*user || !*passwd))
+ search_netrc (u->host, (const char **) user, (const char **) passwd, 0, NULL);
+
+ /* We only do "site-wide" authentication with "global" user/password
+ * values unless --auth-no-challenge has been requested; URL user/password
+ * info overrides. */
+ if (*user && *passwd && (!u->user || opt.auth_without_challenge))
+ {
+ /* If this is a host for which we've already received a Basic
+ * challenge, we'll go ahead and send Basic authentication creds. */
+ *basic_auth_finished = maybe_send_basic_creds (u->host, *user, *passwd, req);
+ }
+
+ if (inhibit_keep_alive)
+ request_set_header (req, "Connection", "Close", rel_none);
+ else
+ {
+ request_set_header (req, "Connection", "Keep-Alive", rel_none);
+ if (proxy)
+ request_set_header (req, "Proxy-Connection", "Keep-Alive", rel_none);
+ }
+
+ if (opt.method)
+ {
+
+ if (opt.body_data || opt.body_file)
+ {
+ request_set_header (req, "Content-Type",
+ "application/x-www-form-urlencoded", rel_none);
+
+ if (opt.body_data)
+ *body_data_size = strlen (opt.body_data);
+ else
+ {
+ *body_data_size = file_size (opt.body_file);
+ if (*body_data_size == -1)
+ {
+ logprintf (LOG_NOTQUIET, _("BODY data file %s missing: %s\n"),
+ quote (opt.body_file), strerror (errno));
+ request_free (&req);
+ *ret = FILEBADFILE;
+ return NULL;
+ }
+ }
+ request_set_header (req, "Content-Length",
+ xstrdup (number_to_static_string (*body_data_size)),
+ rel_value);
+ }
+ else if (c_strcasecmp (opt.method, "post") == 0
+ || c_strcasecmp (opt.method, "put") == 0
+ || c_strcasecmp (opt.method, "patch") == 0)
+ request_set_header (req, "Content-Length", "0", rel_none);
+ }
+ return req;
+}
+
+static void
+initialize_proxy_configuration (const struct url *u, struct request *req,
+ struct url *proxy, char **proxyauth)
+{
+ char *proxy_user, *proxy_passwd;
+ /* For normal username and password, URL components override
+ command-line/wgetrc parameters. With proxy
+ authentication, it's the reverse, because proxy URLs are
+ normally the "permanent" ones, so command-line args
+ should take precedence. */
+ if (opt.proxy_user && opt.proxy_passwd)
+ {
+ proxy_user = opt.proxy_user;
+ proxy_passwd = opt.proxy_passwd;
+ }
+ else
+ {
+ proxy_user = proxy->user;
+ proxy_passwd = proxy->passwd;
+ }
+ /* #### This does not appear right. Can't the proxy request,
+ say, `Digest' authentication? */
+ if (proxy_user && proxy_passwd)
+ *proxyauth = basic_authentication_encode (proxy_user, proxy_passwd);
+
+ /* Proxy authorization over SSL is handled below. */
+#ifdef HAVE_SSL
+ if (u->scheme != SCHEME_HTTPS)
+#endif
+ request_set_header (req, "Proxy-Authorization", *proxyauth, rel_value);
+}
+
+static uerr_t
+establish_connection (const struct url *u, const struct url **conn_ref,
+ struct http_stat *hs, struct url *proxy,
+ char **proxyauth,
+ struct request **req_ref, bool *using_ssl,
+ bool inhibit_keep_alive,
+ int *sock_ref)
+{
+ bool host_lookup_failed = false;
+ int sock = *sock_ref;
+ struct request *req = *req_ref;
+ const struct url *conn = *conn_ref;
+ struct response *resp;
+ int write_error;
+ int statcode;
+
+ if (! inhibit_keep_alive)
+ {
+ /* Look for a persistent connection to target host, unless a
+ proxy is used. The exception is when SSL is in use, in which
+ case the proxy is nothing but a passthrough to the target
+ host, registered as a connection to the latter. */
+ const struct url *relevant = conn;
+#ifdef HAVE_SSL
+ if (u->scheme == SCHEME_HTTPS)
+ relevant = u;
+#endif
+
+ if (persistent_available_p (relevant->host, relevant->port,
+#ifdef HAVE_SSL
+ relevant->scheme == SCHEME_HTTPS,
+#else
+ 0,
+#endif
+ &host_lookup_failed))
+ {
+ int family = socket_family (pconn.socket, ENDPOINT_PEER);
+ sock = pconn.socket;
+ *using_ssl = pconn.ssl;
+#if ENABLE_IPV6
+ if (family == AF_INET6)
+ logprintf (LOG_VERBOSE, _("Reusing existing connection to [%s]:%d.\n"),
+ quotearg_style (escape_quoting_style, pconn.host),
+ pconn.port);
+ else
+#endif
+ logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"),
+ quotearg_style (escape_quoting_style, pconn.host),
+ pconn.port);
+ DEBUGP (("Reusing fd %d.\n", sock));
+ if (pconn.authorized)
+ /* If the connection is already authorized, the "Basic"
+ authorization added by code above is unnecessary and
+ only hurts us. */
+ request_remove_header (req, "Authorization");
+ }
+ else if (host_lookup_failed)
+ {
+ logprintf(LOG_NOTQUIET,
+ _("%s: unable to resolve host address %s\n"),
+ exec_name, quote (relevant->host));
+ return HOSTERR;
+ }
+ else if (sock != -1)
+ {
+ sock = -1;
+ }
+ }
+
+ if (sock < 0)
+ {
+ sock = connect_to_host (conn->host, conn->port);
+ if (sock == E_HOST)
+ return HOSTERR;
+ else if (sock < 0)
+ return (retryable_socket_connect_error (errno)
+ ? CONERROR : CONIMPOSSIBLE);
+
+#ifdef HAVE_SSL
+ if (proxy && u->scheme == SCHEME_HTTPS)
+ {
+ char *head;
+ char *message;
+ /* When requesting SSL URLs through proxies, use the
+ CONNECT method to request passthrough. */
+ struct request *connreq = request_new ("CONNECT",
+ aprintf ("%s:%d", u->host, u->port));
+ SET_USER_AGENT (connreq);
+ if (proxyauth)
+ {
+ request_set_header (connreq, "Proxy-Authorization",
+ *proxyauth, rel_value);
+ /* Now that PROXYAUTH is part of the CONNECT request,
+ zero it out so we don't send proxy authorization with
+ the regular request below. */
+ *proxyauth = NULL;
+ }
+ request_set_header (connreq, "Host",
+ aprintf ("%s:%d", u->host, u->port),
+ rel_value);
+
+ write_error = request_send (connreq, sock, 0);
+ request_free (&connreq);
+ if (write_error < 0)
+ {
+ CLOSE_INVALIDATE (sock);
+ return WRITEFAILED;
+ }
+
+ head = read_http_response_head (sock);
+ if (!head)
+ {
+ logprintf (LOG_VERBOSE, _("Failed reading proxy response: %s\n"),
+ fd_errstr (sock));
+ CLOSE_INVALIDATE (sock);
+ return HERR;
+ }
+ message = NULL;
+ if (!*head)
+ {
+ xfree (head);
+ goto failed_tunnel;
+ }
+ DEBUGP (("proxy responded with: [%s]\n", head));
+
+ resp = resp_new (head);
+ statcode = resp_status (resp, &message);
+ if (statcode < 0)
+ {
+ char *tms = datetime_str (time (NULL));
+ logprintf (LOG_VERBOSE, "%d\n", statcode);
+ logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"), tms, statcode,
+ quotearg_style (escape_quoting_style,
+ _("Malformed status line")));
+ xfree (head);
+ return HERR;
+ }
+ xfree (hs->message);
+ hs->message = xstrdup (message);
+ resp_free (&resp);
+ xfree (head);
+ if (statcode != 200)
+ {
+ failed_tunnel:
+ logprintf (LOG_NOTQUIET, _("Proxy tunneling failed: %s"),
+ message ? quotearg_style (escape_quoting_style, message) : "?");
+ xfree (message);
+ return CONSSLERR;
+ }
+ xfree (message);
+
+ /* SOCK is now *really* connected to u->host, so update CONN
+ to reflect this. That way register_persistent will
+ register SOCK as being connected to u->host:u->port. */
+ conn = u;
+ }
+
+ if (conn->scheme == SCHEME_HTTPS)
+ {
+ if (!ssl_connect_wget (sock, u->host, NULL))
+ {
+ CLOSE_INVALIDATE (sock);
+ return CONSSLERR;
+ }
+ else if (!ssl_check_certificate (sock, u->host))
+ {
+ CLOSE_INVALIDATE (sock);
+ return VERIFCERTERR;
+ }
+ *using_ssl = true;
+ }
+#endif /* HAVE_SSL */
+ }
+ *conn_ref = conn;
+ *req_ref = req;
+ *sock_ref = sock;
+ return RETROK;
+}
+
+static uerr_t
+set_file_timestamp (struct http_stat *hs)
+{
+ bool local_dot_orig_file_exists = false;
+ char *local_filename = NULL;
+ struct stat st;
+ char buf[1024];
+
+ if (opt.backup_converted)
+ /* If -K is specified, we'll act on the assumption that it was specified
+ last time these files were downloaded as well, and instead of just
+ comparing local file X against server file X, we'll compare local
+ file X.orig (if extant, else X) against server file X. If -K
+ _wasn't_ specified last time, or the server contains files called
+ *.orig, -N will be back to not operating correctly with -k. */
+ {
+ size_t filename_len = strlen (hs->local_file);
+ char *filename_plus_orig_suffix;
+
+ if (filename_len + sizeof (ORIG_SFX) > sizeof (buf))
+ filename_plus_orig_suffix = xmalloc (filename_len + sizeof (ORIG_SFX));
+ else
+ filename_plus_orig_suffix = buf;
+
+ /* Would a single s[n]printf() call be faster? --dan
+
+ Definitely not. sprintf() is horribly slow. It's a
+ different question whether the difference between the two
+ affects a program. Usually I'd say "no", but at one
+ point I profiled Wget, and found that a measurable and
+ non-negligible amount of time was lost calling sprintf()
+ in url.c. Replacing sprintf with inline calls to
+ strcpy() and number_to_string() made a difference.
+ --hniksic */
+ memcpy (filename_plus_orig_suffix, hs->local_file, filename_len);
+ memcpy (filename_plus_orig_suffix + filename_len,
+ ORIG_SFX, sizeof (ORIG_SFX));
+
+ /* Try to stat() the .orig file. */
+ if (stat (filename_plus_orig_suffix, &st) == 0)
+ {
+ local_dot_orig_file_exists = true;
+ local_filename = filename_plus_orig_suffix;
+ }
+ }
+
+ if (!local_dot_orig_file_exists)
+ /* Couldn't stat() <file>.orig, so try to stat() <file>. */
+ if (stat (hs->local_file, &st) == 0)
+ {
+ if (local_filename != buf)
+ xfree (local_filename);
+ local_filename = hs->local_file;
+ }
+
+ if (local_filename != NULL)
+ /* There was a local file, so we'll check later to see if the version
+ the server has is the same version we already have, allowing us to
+ skip a download. */
+ {
+ if (local_filename == buf || local_filename == hs->local_file)
+ hs->orig_file_name = xstrdup (local_filename); // on stack or a copy, make a heap copy
+ else
+ hs->orig_file_name = local_filename; // was previously malloc'ed
+ hs->orig_file_size = st.st_size;
+ hs->orig_file_tstamp = st.st_mtime;
+#ifdef WINDOWS
+ /* Modification time granularity is 2 seconds for Windows, so
+ increase local time by 1 second for later comparison. */
+ ++hs->orig_file_tstamp;
+#endif
+ hs->timestamp_checked = true;
+ }
+
+ return RETROK;
+}
+
+static uerr_t
+check_file_output (const struct url *u, struct http_stat *hs,
+ struct response *resp, char *hdrval, size_t hdrsize)
+{
+ /* Determine the local filename if needed. Notice that if -O is used
+ * hstat.local_file is set by http_loop to the argument of -O. */
+ if (!hs->local_file)
+ {
+ char *local_file = NULL;
+
+ /* Honor Content-Disposition whether possible. */
+ if (!opt.content_disposition
+ || !resp_header_copy (resp, "Content-Disposition",
+ hdrval, hdrsize)
+ || !parse_content_disposition (hdrval, &local_file))
+ {
+ /* The Content-Disposition header is missing or broken.
+ * Choose unique file name according to given URL. */
+ hs->local_file = url_file_name (u, NULL);
+ }
+ else
+ {
+ DEBUGP (("Parsed filename from Content-Disposition: %s\n",
+ local_file));
+ hs->local_file = url_file_name (u, local_file);
+ }
+
+ xfree (local_file);
+ }
+
+ hs->temporary = opt.delete_after || opt.spider || !acceptable (hs->local_file);
+ if (hs->temporary)
+ {
+ char *tmp = aprintf ("%s.tmp", hs->local_file);
+ xfree (hs->local_file);
+ hs->local_file = tmp;
+ }
+
+ /* TODO: perform this check only once. */
+ if (!hs->existence_checked && file_exists_p (hs->local_file, NULL))
+ {
+ if (opt.noclobber && !opt.output_document)
+ {
+ /* If opt.noclobber is turned on and file already exists, do not
+ retrieve the file. But if the output_document was given, then this
+ test was already done and the file didn't exist. Hence the !opt.output_document */
+ return RETRUNNEEDED;
+ }
+ else if (!ALLOW_CLOBBER)
+ {
+ char *unique = unique_name_passthrough (hs->local_file);
+ if (unique != hs->local_file)
+ xfree (hs->local_file);
+ hs->local_file = unique;
+ }
+ }
+ hs->existence_checked = true;
+
+ /* Support timestamping */
+ if (opt.timestamping && !hs->timestamp_checked)
+ {
+ uerr_t timestamp_err = set_file_timestamp (hs);
+ if (timestamp_err != RETROK)
+ return timestamp_err;
+ }
+ return RETROK;
+}
+
+static uerr_t
+check_auth (const struct url *u, char *user, char *passwd, struct response *resp,
+ struct request *req, bool *ntlm_seen_ref, bool *retry,
+ bool *basic_auth_finished_ref, bool *auth_finished_ref)
+{
+ uerr_t auth_err = RETROK;
+ bool basic_auth_finished = *basic_auth_finished_ref;
+ bool auth_finished = *auth_finished_ref;
+ bool ntlm_seen = *ntlm_seen_ref;
+ char buf[256], *tmp = NULL;
+
+ *retry = false;
+
+ if (!auth_finished && (user && passwd))
+ {
+ /* IIS sends multiple copies of WWW-Authenticate, one with
+ the value "negotiate", and other(s) with data. Loop over
+ all the occurrences and pick the one we recognize. */
+ int wapos;
+ const char *www_authenticate = NULL;
+ const char *wabeg, *waend;
+ const char *digest = NULL, *basic = NULL, *ntlm = NULL;
+
+ for (wapos = 0; !ntlm
+ && (wapos = resp_header_locate (resp, "WWW-Authenticate", wapos,
+ &wabeg, &waend)) != -1;
+ ++wapos)
+ {
+ param_token name, value;
+ size_t len = waend - wabeg;
+
+ if (tmp != buf)
+ xfree (tmp);
+
+ if (len < sizeof (buf))
+ tmp = buf;
+ else
+ tmp = xmalloc (len + 1);
+
+ memcpy (tmp, wabeg, len);
+ tmp[len] = 0;
+
+ www_authenticate = tmp;
+
+ for (;!ntlm;)
+ {
+ /* extract the auth-scheme */
+ while (c_isspace (*www_authenticate)) www_authenticate++;
+ name.e = name.b = www_authenticate;
+ while (*name.e && !c_isspace (*name.e)) name.e++;
+
+ if (name.b == name.e)
+ break;
+
+ DEBUGP (("Auth scheme found '%.*s'\n", (int) (name.e - name.b), name.b));
+
+ if (known_authentication_scheme_p (name.b, name.e))
+ {
+ if (BEGINS_WITH (name.b, "NTLM"))
+ {
+ ntlm = name.b;
+ break; /* this is the most secure challenge, stop here */
+ }
+ else if (!digest && BEGINS_WITH (name.b, "Digest"))
+ digest = name.b;
+ else if (!basic && BEGINS_WITH (name.b, "Basic"))
+ basic = name.b;
+ }
+
+ /* now advance over the auth-params */
+ www_authenticate = name.e;
+ DEBUGP (("Auth param list '%s'\n", www_authenticate));
+ while (extract_param (&www_authenticate, &name, &value, ',', NULL) && name.b && value.b)
+ {
+ DEBUGP (("Auth param %.*s=%.*s\n",
+ (int) (name.e - name.b), name.b, (int) (value.e - value.b), value.b));
+ }
+ }
+ }
+
+ if (!basic && !digest && !ntlm)
+ {
+ /* If the authentication header is missing or
+ unrecognized, there's no sense in retrying. */
+ logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
+ }
+ else if (!basic_auth_finished
+ || !basic)
+ {
+ char *pth = url_full_path (u);
+ const char *value;
+ uerr_t *auth_stat;
+ auth_stat = xmalloc (sizeof (uerr_t));
+ *auth_stat = RETROK;
+
+ if (ntlm)
+ www_authenticate = ntlm;
+ else if (digest)
+ www_authenticate = digest;
+ else
+ www_authenticate = basic;
+
+ logprintf (LOG_NOTQUIET, _("Authentication selected: %s\n"), www_authenticate);
+
+ value = create_authorization_line (www_authenticate,
+ user, passwd,
+ request_method (req),
+ pth,
+ &auth_finished,
+ auth_stat);
+
+ auth_err = *auth_stat;
+ xfree (auth_stat);
+ xfree (pth);
+ if (auth_err == RETROK)
+ {
+ request_set_header (req, "Authorization", value, rel_value);
+
+ if (BEGINS_WITH (www_authenticate, "NTLM"))
+ ntlm_seen = true;
+ else if (!u->user && BEGINS_WITH (www_authenticate, "Basic"))
+ {
+ /* Need to register this host as using basic auth,
+ * so we automatically send creds next time. */
+ register_basic_auth_host (u->host);
+ }
+
+ *retry = true;
+ goto cleanup;
+ }
+ else
+ {
+ /* Creating the Authorization header went wrong */
+ xfree (value);
+ }
+ }
+ else
+ {
+ /* We already did Basic auth, and it failed. Gotta
+ * give up. */
+ }
+ }
+
+ cleanup:
+ if (tmp != buf)
+ xfree (tmp);
+ *ntlm_seen_ref = ntlm_seen;
+ *basic_auth_finished_ref = basic_auth_finished;
+ *auth_finished_ref = auth_finished;
+ return auth_err;
+}
+
+static uerr_t
+open_output_stream (struct http_stat *hs, int count, FILE **fp)
+{
+/* 2005-06-17 SMS.
+ For VMS, define common fopen() optional arguments.
+*/
+#ifdef __VMS
+# define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
+# define FOPEN_BIN_FLAG 3
+#else /* def __VMS */
+# define FOPEN_BIN_FLAG true
+#endif /* def __VMS [else] */
+
+ /* Open the local file. */
+ if (!output_stream)
+ {
+ mkalldirs (hs->local_file);
+ if (opt.backups)
+ rotate_backups (hs->local_file);
+ if (hs->restval)
+ {
+#ifdef __VMS
+ int open_id;
+
+ open_id = 21;
+ *fp = fopen (hs->local_file, "ab", FOPEN_OPT_ARGS);
+#else /* def __VMS */
+ *fp = fopen (hs->local_file, "ab");
+#endif /* def __VMS [else] */
+ }
+ else if (ALLOW_CLOBBER || count > 0)
+ {
+ if (opt.unlink_requested && file_exists_p (hs->local_file, NULL))
+ {
+ if (unlink (hs->local_file) < 0)
+ {
+ logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file,
+ strerror (errno));
+ return UNLINKERR;
+ }
+ }
+
+#ifdef __VMS
+ int open_id;
+
+ open_id = 22;
+ *fp = fopen (hs->local_file, "wb", FOPEN_OPT_ARGS);
+#else /* def __VMS */
+ if (hs->temporary)
+ {
+ *fp = fdopen (open (hs->local_file, O_BINARY | O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR), "wb");
+ }
+ else
+ {
+ *fp = fopen (hs->local_file, "wb");
+ }
+
+#endif /* def __VMS [else] */
+ }
+ else
+ {
+ *fp = fopen_excl (hs->local_file, FOPEN_BIN_FLAG);
+ if (!*fp && errno == EEXIST)
+ {
+ /* We cannot just invent a new name and use it (which is
+ what functions like unique_create typically do)
+ because we told the user we'd use this name.
+ Instead, return and retry the download. */
+ logprintf (LOG_NOTQUIET,
+ _("%s has sprung into existence.\n"),
+ hs->local_file);
+ return FOPEN_EXCL_ERR;
+ }
+ }
+ if (!*fp)
+ {
+ logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file, strerror (errno));
+ return FOPENERR;
+ }
+ }
+ else
+ *fp = output_stream;
+
+ /* Print fetch message, if opt.verbose. */
+ logprintf (LOG_VERBOSE, _("Saving to: %s\n"),
+ HYPHENP (hs->local_file) ? quote ("STDOUT") : quote (hs->local_file));
+
+ return RETROK;
+}
+
+/* Set proper type flags based on type string. */
+static void
+set_content_type (int *dt, const char *type)
+{
+ /* If content-type is not given, assume text/html. This is because
+ of the multitude of broken CGI's that "forget" to generate the
+ content-type. */
+ if (!type ||
+ 0 == c_strcasecmp (type, TEXTHTML_S) ||
+ 0 == c_strcasecmp (type, TEXTXHTML_S))
+ *dt |= TEXTHTML;
+ else
+ *dt &= ~TEXTHTML;
+
+ if (type &&
+ 0 == c_strcasecmp (type, TEXTCSS_S))
+ *dt |= TEXTCSS;
+ else
+ *dt &= ~TEXTCSS;
+}
+
+#ifdef HAVE_METALINK
+/* Will return proper metalink_t structure if enough data was found in
+ http response resp. Otherwise returns NULL.
+ Two exit points: one for success and one for failure. */
+static metalink_t *
+metalink_from_http (const struct response *resp, const struct http_stat *hs,
+ const struct url *u)
+{
+ metalink_t *metalink = NULL;
+ metalink_file_t *mfile = xnew0 (metalink_file_t);
+ const char *val_beg, *val_end;
+ int res_count = 0, meta_count = 0, hash_count = 0, sig_count = 0, i;
+
+ DEBUGP (("Checking for Metalink in HTTP response\n"));
+
+ /* Initialize metalink file for our simple use case. */
+ if (hs->local_file)
+ mfile->name = xstrdup (hs->local_file);
+ else
+ mfile->name = url_file_name (u, NULL);
+
+ /* Begin with 1-element array (for 0-termination). */
+ mfile->checksums = xnew0 (metalink_checksum_t *);
+ mfile->resources = xnew0 (metalink_resource_t *);
+ mfile->metaurls = xnew0 (metalink_metaurl_t *);
+
+ /* Process the Content-Type header. */
+ if (resp_header_locate (resp, "Content-Type", 0, &val_beg, &val_end) != -1)
+ {
+ metalink_metaurl_t murl = {0};
+
+ const char *type_beg, *type_end;
+ char *typestr = NULL;
+ char *namestr = NULL;
+ size_t type_len;
+
+ DEBUGP (("Processing Content-Type header...\n"));
+
+ /* Find beginning of type. */
+ type_beg = val_beg;
+ while (type_beg < val_end && c_isspace (*type_beg))
+ type_beg++;
+
+ /* Find end of type. */
+ type_end = type_beg + 1;
+ while (type_end < val_end &&
+ *type_end != ';' &&
+ *type_end != ' ' &&
+ *type_end != '\r' &&
+ *type_end != '\n')
+ type_end++;
+
+ if (type_beg >= val_end || type_end > val_end)
+ {
+ DEBUGP (("Invalid Content-Type header. Ignoring.\n"));
+ goto skip_content_type;
+ }
+
+ type_len = type_end - type_beg;
+ typestr = xstrndup (type_beg, type_len);
+
+ DEBUGP (("Content-Type: %s\n", typestr));
+
+ if (strcmp (typestr, "application/metalink4+xml"))
+ {
+ xfree (typestr);
+ goto skip_content_type;
+ }
+
+ /*
+ Valid ranges for the "pri" attribute are from
+ 1 to 999999. Mirror servers with a lower value of the "pri"
+ attribute have a higher priority, while mirrors with an undefined
+ "pri" attribute are considered to have a value of 999999, which is
+ the lowest priority.
+
+ rfc6249 section 3.1
+ */
+ murl.priority = DEFAULT_PRI;
+
+ murl.mediatype = typestr;
+ typestr = NULL;
+
+ if (opt.content_disposition
+ && resp_header_locate (resp, "Content-Disposition", 0, &val_beg, &val_end) != -1)
+ {
+ find_key_value (val_beg, val_end, "filename", &namestr);
+ murl.name = namestr;
+ namestr = NULL;
+ }
+
+ murl.url = xstrdup (u->url);
+
+ DEBUGP (("URL=%s\n", murl.url));
+ DEBUGP (("MEDIATYPE=%s\n", murl.mediatype));
+ DEBUGP (("NAME=%s\n", murl.name ? murl.name : ""));
+ DEBUGP (("PRIORITY=%d\n", murl.priority));
+
+ /* 1 slot from new resource, 1 slot for null-termination. */
+ mfile->metaurls = xrealloc (mfile->metaurls,
+ sizeof (metalink_metaurl_t *) * (meta_count + 2));
+ mfile->metaurls[meta_count] = xnew0 (metalink_metaurl_t);
+ *mfile->metaurls[meta_count] = murl;
+ meta_count++;
+ }
+skip_content_type:
+
+ /* Find all Link headers. */
+ for (i = 0;
+ (i = resp_header_locate (resp, "Link", i, &val_beg, &val_end)) != -1;
+ i++)
+ {
+ char *rel = NULL, *reltype = NULL;
+ char *urlstr = NULL;
+ const char *url_beg, *url_end, *attrs_beg;
+ size_t url_len;
+
+ /* Sample Metalink Link headers:
+
+ Link: <http://www2.example.com/dir1/dir2/dir3/dir4/dir5/example.ext>;
+ rel=duplicate; pri=1; pref; geo=gb; depth=4
+
+ Link: <http://example.com/example.ext.asc>; rel=describedby;
+ type="application/pgp-signature"
+ */
+
+ /* Find beginning of URL. */
+ url_beg = val_beg;
+ while (url_beg < val_end - 1 && c_isspace (*url_beg))
+ url_beg++;
+
+ /* Find end of URL. */
+ /* The convention here is that end ptr points to one element after
+ end of string. In this case, it should be pointing to the '>', which
+ is one element after end of actual URL. Therefore, it should never point
+ to val_end, which is one element after entire header value string. */
+ url_end = url_beg + 1;
+ while (url_end < val_end - 1 && *url_end != '>')
+ url_end++;
+
+ if (url_beg >= val_end || url_end >= val_end ||
+ *url_beg != '<' || *url_end != '>')
+ {
+ DEBUGP (("This is not a valid Link header. Ignoring.\n"));
+ continue;
+ }
+
+ /* Skip <. */
+ url_beg++;
+ url_len = url_end - url_beg;
+
+ /* URL found. Now handle the attributes. */
+ attrs_beg = url_end + 1;
+
+ /* First we need to find out what type of link it is. Currently, we
+ support rel=duplicate and rel=describedby. */
+ if (!find_key_value (attrs_beg, val_end, "rel", &rel))
+ {
+ DEBUGP (("No rel value in Link header, skipping.\n"));
+ continue;
+ }
+
+ urlstr = xstrndup (url_beg, url_len);
+ DEBUGP (("URL=%s\n", urlstr));
+ DEBUGP (("rel=%s\n", rel));
+
+ if (!strcmp (rel, "describedby"))
+ find_key_value (attrs_beg, val_end, "type", &reltype);
+
+ /* Handle signatures.
+ Libmetalink only supports one signature per file. Therefore we stop
+ as soon as we successfully get first supported signature. */
+ if (sig_count == 0 &&
+ reltype && !strcmp (reltype, "application/pgp-signature"))
+ {
+ /* Download the signature to a temporary file. */
+ FILE *_output_stream = output_stream;
+ bool _output_stream_regular = output_stream_regular;
+
+ output_stream = tmpfile ();
+ if (output_stream)
+ {
+ struct iri *iri = iri_new ();
+ struct url *url;
+ int url_err;
+
+ set_uri_encoding (iri, opt.locale, true);
+ url = url_parse (urlstr, &url_err, iri, false);
+
+ if (!url)
+ {
+ char *error = url_error (urlstr, url_err);
+ logprintf (LOG_NOTQUIET, _("When downloading signature:\n"
+ "%s: %s.\n"), urlstr, error);
+ xfree (error);
+ iri_free (iri);
+ }
+ else
+ {
+ /* Avoid recursive Metalink from HTTP headers. */
+ bool _metalink_http = opt.metalink_over_http;
+ uerr_t retr_err;
+
+ opt.metalink_over_http = false;
+ retr_err = retrieve_url (url, urlstr, NULL, NULL,
+ NULL, NULL, false, iri, false);
+ opt.metalink_over_http = _metalink_http;
+
+ url_free (url);
+ iri_free (iri);
+
+ if (retr_err == RETROK)
+ {
+ /* Signature is in the temporary file. Read it into
+ metalink resource structure. */
+ metalink_signature_t msig;
+ size_t siglen;
+
+ fseek (output_stream, 0, SEEK_END);
+ siglen = ftell (output_stream);
+ fseek (output_stream, 0, SEEK_SET);
+
+ DEBUGP (("siglen=%lu\n", siglen));
+
+ msig.signature = xmalloc (siglen + 1);
+ if (fread (msig.signature, siglen, 1, output_stream) != 1)
+ {
+ logputs (LOG_NOTQUIET,
+ _("Unable to read signature content from "
+ "temporary file. Skipping.\n"));
+ xfree (msig.signature);
+ }
+ else
+ {
+ msig.signature[siglen] = '\0'; /* Just in case. */
+ msig.mediatype = xstrdup ("application/pgp-signature");
+
+ DEBUGP (("Signature (%s):\n%s\n",
+ msig.mediatype, msig.signature));
+
+ mfile->signature = xnew (metalink_signature_t);
+ *mfile->signature = msig;
+
+ sig_count++;
+ }
+ }
+ }
+ fclose (output_stream);
+ }
+ else
+ {
+ logputs (LOG_NOTQUIET, _("Could not create temporary file. "
+ "Skipping signature download.\n"));
+ }
+ output_stream_regular = _output_stream_regular;
+ output_stream = _output_stream;
+ } /* Iterate over signatures. */
+
+ /* Handle Metalink resources. */
+ else if (!strcmp (rel, "duplicate"))
+ {
+ metalink_resource_t mres = {0};
+ char *pristr;
+
+ /*
+ Valid ranges for the "pri" attribute are from
+ 1 to 999999. Mirror servers with a lower value of the "pri"
+ attribute have a higher priority, while mirrors with an undefined
+ "pri" attribute are considered to have a value of 999999, which is
+ the lowest priority.
+
+ rfc6249 section 3.1
+ */
+ mres.priority = DEFAULT_PRI;
+ if (find_key_value (url_end, val_end, "pri", &pristr))
+ {
+ long pri;
+ char *end_pristr;
+ /* Do not care for errno since 0 is error in this case. */
+ pri = strtol (pristr, &end_pristr, 10);
+ if (end_pristr != pristr + strlen (pristr) ||
+ !VALID_PRI_RANGE (pri))
+ {
+ /* This is against the specification, so let's inform the user. */
+ logprintf (LOG_NOTQUIET,
+ _("Invalid pri value. Assuming %d.\n"),
+ DEFAULT_PRI);
+ }
+ else
+ mres.priority = pri;
+ xfree (pristr);
+ }
+
+ switch (url_scheme (urlstr))
+ {
+ case SCHEME_HTTP:
+ mres.type = xstrdup ("http");
+ break;
+#ifdef HAVE_SSL
+ case SCHEME_HTTPS:
+ mres.type = xstrdup ("https");
+ break;
+ case SCHEME_FTPS:
+ mres.type = xstrdup ("ftps");
+ break;
+#endif
+ case SCHEME_FTP:
+ mres.type = xstrdup ("ftp");
+ break;
+ default:
+ DEBUGP (("Unsupported url scheme in %s. Skipping resource.\n", urlstr));
+ }
+
+ if (mres.type)
+ {
+ DEBUGP (("TYPE=%s\n", mres.type));
+
+ /* At this point we have validated the new resource. */
+
+ find_key_value (url_end, val_end, "geo", &mres.location);
+
+ mres.url = urlstr;
+ urlstr = NULL;
+
+ mres.preference = 0;
+ if (has_key (url_end, val_end, "pref"))
+ {
+ DEBUGP (("This resource has preference\n"));
+ mres.preference = 1;
+ }
+
+ /* 1 slot from new resource, 1 slot for null-termination. */
+ mfile->resources = xrealloc (mfile->resources,
+ sizeof (metalink_resource_t *) * (res_count + 2));
+ mfile->resources[res_count] = xnew0 (metalink_resource_t);
+ *mfile->resources[res_count] = mres;
+ res_count++;
+ }
+ } /* Handle resource link (rel=duplicate). */
+
+ /* Handle Metalink/XML resources. */
+ else if (reltype && !strcmp (reltype, "application/metalink4+xml"))
+ {
+ metalink_metaurl_t murl = {0};
+ char *pristr;
+
+ /*
+ Valid ranges for the "pri" attribute are from
+ 1 to 999999. Mirror servers with a lower value of the "pri"
+ attribute have a higher priority, while mirrors with an undefined
+ "pri" attribute are considered to have a value of 999999, which is
+ the lowest priority.
+
+ rfc6249 section 3.1
+ */
+ murl.priority = DEFAULT_PRI;
+ if (find_key_value (url_end, val_end, "pri", &pristr))
+ {
+ long pri;
+ char *end_pristr;
+ /* Do not care for errno since 0 is error in this case. */
+ pri = strtol (pristr, &end_pristr, 10);
+ if (end_pristr != pristr + strlen (pristr) ||
+ !VALID_PRI_RANGE (pri))
+ {
+ /* This is against the specification, so let's inform the user. */
+ logprintf (LOG_NOTQUIET,
+ _("Invalid pri value. Assuming %d.\n"),
+ DEFAULT_PRI);
+ }
+ else
+ murl.priority = pri;
+ xfree (pristr);
+ }
+
+ murl.mediatype = xstrdup (reltype);
+
+ DEBUGP (("MEDIATYPE=%s\n", murl.mediatype));
+
+ /* At this point we have validated the new resource. */
+
+ find_key_value (url_end, val_end, "name", &murl.name);
+
+ murl.url = urlstr;
+ urlstr = NULL;
+
+ /* 1 slot from new resource, 1 slot for null-termination. */
+ mfile->metaurls = xrealloc (mfile->metaurls,
+ sizeof (metalink_metaurl_t *) * (meta_count + 2));
+ mfile->metaurls[meta_count] = xnew0 (metalink_metaurl_t);
+ *mfile->metaurls[meta_count] = murl;
+ meta_count++;
+ } /* Handle resource link (rel=describedby). */
+ else
+ DEBUGP (("This link header was not used for Metalink\n"));
+
+ xfree (urlstr);
+ xfree (reltype);
+ xfree (rel);
+ } /* Iterate over link headers. */
+
+ /* Null-terminate resources array. */
+ mfile->resources[res_count] = 0;
+ mfile->metaurls[meta_count] = 0;
+
+ if (res_count == 0 && meta_count == 0)
+ {
+ DEBUGP (("No valid metalink references found.\n"));
+ goto fail;
+ }
+
+ /* Find all Digest headers. */
+ for (i = 0;
+ (i = resp_header_locate (resp, "Digest", i, &val_beg, &val_end)) != -1;
+ i++)
+ {
+ const char *dig_pos;
+ char *dig_type, *dig_hash;
+
+ /* Each Digest header can include multiple hashes. Example:
+ Digest: SHA=thvDyvhfIqlvFe+A9MYgxAfm1q5=,unixsum=30637
+ Digest: md5=HUXZLQLMuI/KZ5KDcJPcOA==
+ */
+ for (dig_pos = val_beg;
+ (dig_pos = find_key_values (dig_pos, val_end, &dig_type, &dig_hash));
+ dig_pos++)
+ {
+ /* The hash here is assumed to be base64. We need the hash in hex.
+ Therefore we convert: base64 -> binary -> hex. */
+ const size_t dig_hash_str_len = strlen (dig_hash);
+ char bin_hash[256];
+ ssize_t hash_bin_len;
+
+ // there is no hash with that size
+ if (dig_hash_str_len >= sizeof (bin_hash))
+ {
+ DEBUGP (("Hash too long, ignored.\n"));
+ xfree (dig_type);
+ xfree (dig_hash);
+ continue;
+ }
+
+ hash_bin_len = wget_base64_decode (dig_hash, bin_hash, dig_hash_str_len * 3 / 4 + 1);
+
+ /* Detect malformed base64 input. */
+ if (hash_bin_len < 0)
+ {
+ DEBUGP (("Malformed base64 input, ignored.\n"));
+ xfree (dig_type);
+ xfree (dig_hash);
+ continue;
+ }
+
+ /* One slot for me, one for zero-termination. */
+ mfile->checksums =
+ xrealloc (mfile->checksums,
+ sizeof (metalink_checksum_t *) * (hash_count + 2));
+ mfile->checksums[hash_count] = xnew (metalink_checksum_t);
+ mfile->checksums[hash_count]->type = dig_type;
+
+ mfile->checksums[hash_count]->hash = xmalloc ((size_t)hash_bin_len * 2 + 1);
+ wg_hex_to_string (mfile->checksums[hash_count]->hash, bin_hash, (size_t)hash_bin_len);
+
+ xfree (dig_hash);
+
+ hash_count++;
+ }
+ }
+
+ /* Zero-terminate checksums array. */
+ mfile->checksums[hash_count] = 0;
+
+ /*
+ If Instance Digests are not provided by the Metalink servers, the
+ Link header fields pertaining to this specification MUST be ignored.
+
+ rfc6249 section 6
+ */
+ if (res_count && hash_count == 0)
+ {
+ logputs (LOG_VERBOSE,
+ _("Could not find acceptable digest for Metalink resources.\n"
+ "Ignoring them.\n"));
+ goto fail;
+ }
+
+ /* Metalink data is OK. Now we just need to sort the resources based
+ on their priorities, preference, and perhaps location. */
+ stable_sort (mfile->resources, res_count, sizeof (metalink_resource_t *), metalink_res_cmp);
+ stable_sort (mfile->metaurls, meta_count, sizeof (metalink_metaurl_t *), metalink_meta_cmp);
+
+ /* Restore sensible preference values (in case someone cares to look). */
+ for (i = 0; i < res_count; ++i)
+ mfile->resources[i]->preference = 1000000 - mfile->resources[i]->priority;
+
+ metalink = xnew0 (metalink_t);
+ metalink->files = xmalloc (sizeof (metalink_file_t *) * 2);
+ metalink->files[0] = mfile;
+ metalink->files[1] = 0;
+ metalink->origin = xstrdup (u->url);
+ metalink->version = METALINK_VERSION_4;
+ /* Leave other fields set to 0. */
+
+ return metalink;
+
+fail:
+ /* Free all allocated memory. */
+ if (metalink)
+ metalink_delete (metalink);
+ else
+ metalink_file_delete (mfile);
+ return NULL;
+}
+#endif /* HAVE_METALINK */
+
+/* Retrieve a document through HTTP protocol. It recognizes status
+ code, and correctly handles redirections. It closes the network
+ socket. If it receives an error from the functions below it, it
+ will print it if there is enough information to do so (almost
+ always), returning the error to the caller (i.e. http_loop).
+
+ Various HTTP parameters are stored to hs.
+
+ If PROXY is non-NULL, the connection will be made to the proxy
+ server, and u->url will be requested. */
+static uerr_t
+gethttp (const struct url *u, struct url *original_url, struct http_stat *hs,
+ int *dt, struct url *proxy, struct iri *iri, int count)
+{
+ struct request *req = NULL;
+
+ char *type = NULL;
+ char *user, *passwd;
+ char *proxyauth;
+ int statcode;
+ int write_error;
+ wgint contlen, contrange;
+ const struct url *conn;
+ FILE *fp;
+ int err;
+ uerr_t retval;
+#ifdef HAVE_HSTS
+#ifdef TESTING
+ /* we don't link against main.o when we're testing */
+ hsts_store_t hsts_store = NULL;
+#else
+ extern hsts_store_t hsts_store;
+#endif
+ const char *hsts_params;
+ time_t max_age;
+ bool include_subdomains;
+#endif
+
+ int sock = -1;
+
+ /* Set to 1 when the authorization has already been sent and should
+ not be tried again. */
+ bool auth_finished = false;
+
+ /* Set to 1 when just globally-set Basic authorization has been sent;
+ * should prevent further Basic negotiations, but not other
+ * mechanisms. */
+ bool basic_auth_finished = false;
+
+ /* Whether NTLM authentication is used for this request. */
+ bool ntlm_seen = false;
+
+ /* Whether our connection to the remote host is through SSL. */
+ bool using_ssl = false;
+
+ /* Whether a HEAD request will be issued (as opposed to GET or
+ POST). */
+ bool head_only = !!(*dt & HEAD_ONLY);
+
+ /* Whether conditional get request will be issued. */
+ bool cond_get = !!(*dt & IF_MODIFIED_SINCE);
+
+#ifdef HAVE_METALINK
+ /* Are we looking for metalink info in HTTP headers? */
+ bool metalink = !!(*dt & METALINK_METADATA);
+#endif
+
+ char *head = NULL;
+ struct response *resp = NULL;
+ char hdrval[512];
+ char *message = NULL;
+
+ /* Declare WARC variables. */
+ bool warc_enabled = (opt.warc_filename != NULL);
+ FILE *warc_tmp = NULL;
+ char warc_timestamp_str [21];
+ char warc_request_uuid [48];
+ ip_address warc_ip_buf, *warc_ip = NULL;
+ off_t warc_payload_offset = -1;
+
+ /* Whether this connection will be kept alive after the HTTP request
+ is done. */
+ bool keep_alive;
+
+ /* Is the server using the chunked transfer encoding? */
+ bool chunked_transfer_encoding = false;
+
+ /* Whether keep-alive should be inhibited. */
+ bool inhibit_keep_alive =
+ !opt.http_keep_alive || opt.ignore_length;
+
+ /* Headers sent when using POST. */
+ wgint body_data_size = 0;
+
+#ifdef HAVE_SSL
+ if (u->scheme == SCHEME_HTTPS)
+ {
+ /* Initialize the SSL context. After this has once been done,
+ it becomes a no-op. */
+ if (!ssl_init ())
+ {
+ scheme_disable (SCHEME_HTTPS);
+ logprintf (LOG_NOTQUIET,
+ _("Disabling SSL due to encountered errors.\n"));
+ retval = SSLINITFAILED;
+ goto cleanup;
+ }
+ }
+#endif /* HAVE_SSL */
+
+ /* Initialize certain elements of struct http_stat.
+ * Since this function is called in a loop, we have to xfree certain
+ * members. */
+ hs->len = 0;
+ hs->contlen = -1;
+ hs->res = -1;
+ xfree (hs->rderrmsg);
+ xfree (hs->newloc);
+ xfree (hs->remote_time);
+ xfree (hs->error);
+ xfree (hs->message);
+ hs->local_encoding = ENC_NONE;
+ hs->remote_encoding = ENC_NONE;
+
+ conn = u;
+
+ {
+ uerr_t ret;
+ req = initialize_request (u, hs, dt, proxy, inhibit_keep_alive,
+ &basic_auth_finished, &body_data_size,
+ &user, &passwd, &ret);
+ if (req == NULL)
+ {
+ retval = ret;
+ goto cleanup;
+ }
+ }
+ retry_with_auth:
+ /* We need to come back here when the initial attempt to retrieve
+ without authorization header fails. (Expected to happen at least
+ for the Digest authorization scheme.) */
+
+ if (opt.cookies)
+ request_set_header (req, "Cookie",
+ cookie_header (wget_cookie_jar,
+ u->host, u->port, u->path,
+#ifdef HAVE_SSL
+ u->scheme == SCHEME_HTTPS
+#else
+ 0
+#endif
+ ),
+ rel_value);
+
+ /* Add the user headers. */
+ if (opt.user_headers)
+ {
+ int i;
+ for (i = 0; opt.user_headers[i]; i++)
+ request_set_user_header (req, opt.user_headers[i]);
+ }
+
+ proxyauth = NULL;
+ if (proxy)
+ {
+ conn = proxy;
+ initialize_proxy_configuration (u, req, proxy, &proxyauth);
+ }
+ keep_alive = true;
+
+ /* Establish the connection. */
+ if (inhibit_keep_alive)
+ keep_alive = false;
+
+ {
+ uerr_t conn_err = establish_connection (u, &conn, hs, proxy, &proxyauth, &req,
+ &using_ssl, inhibit_keep_alive, &sock);
+ if (conn_err != RETROK)
+ {
+ retval = conn_err;
+ goto cleanup;
+ }
+ }
+
+ /* Open the temporary file where we will write the request. */
+ if (warc_enabled)
+ {
+ warc_tmp = warc_tempfile ();
+ if (warc_tmp == NULL)
+ {
+ CLOSE_INVALIDATE (sock);
+ retval = WARC_TMP_FOPENERR;
+ goto cleanup;
+ }
+
+ if (! proxy)
+ {
+ warc_ip = &warc_ip_buf;
+ socket_ip_address (sock, warc_ip, ENDPOINT_PEER);
+ }
+ }
+
+ /* Send the request to server. */
+ write_error = request_send (req, sock, warc_tmp);
+
+ if (write_error >= 0)
+ {
+ if (opt.body_data)
+ {
+ DEBUGP (("[BODY data: %s]\n", opt.body_data));
+ write_error = fd_write (sock, opt.body_data, body_data_size, -1);
+ if (write_error >= 0 && warc_tmp != NULL)
+ {
+ int warc_tmp_written;
+
+ /* Remember end of headers / start of payload. */
+ warc_payload_offset = ftello (warc_tmp);
+
+ /* Write a copy of the data to the WARC record. */
+ warc_tmp_written = fwrite (opt.body_data, 1, body_data_size, warc_tmp);
+ if (warc_tmp_written != body_data_size)
+ write_error = -2;
+ }
+ }
+ else if (opt.body_file && body_data_size != 0)
+ {
+ if (warc_tmp != NULL)
+ /* Remember end of headers / start of payload */
+ warc_payload_offset = ftello (warc_tmp);
+
+ write_error = body_file_send (sock, opt.body_file, body_data_size, warc_tmp);
+ }
+ }
+
+ if (write_error < 0)
+ {
+ CLOSE_INVALIDATE (sock);
+
+ if (warc_tmp != NULL)
+ fclose (warc_tmp);
+
+ if (write_error == -2)
+ retval = WARC_TMP_FWRITEERR;
+ else
+ retval = WRITEFAILED;
+ goto cleanup;
+ }
+ logprintf (LOG_VERBOSE, _("%s request sent, awaiting response... "),
+ proxy ? "Proxy" : "HTTP");
+ contlen = -1;
+ contrange = 0;
+ *dt &= ~RETROKF;
+
+
+ if (warc_enabled)
+ {
+ bool warc_result;
+
+ /* Generate a timestamp and uuid for this request. */
+ warc_timestamp (warc_timestamp_str, sizeof (warc_timestamp_str));
+ warc_uuid_str (warc_request_uuid, sizeof (warc_request_uuid));
+
+ /* Create a request record and store it in the WARC file. */
+ warc_result = warc_write_request_record (u->url, warc_timestamp_str,
+ warc_request_uuid, warc_ip,
+ warc_tmp, warc_payload_offset);
+ if (! warc_result)
+ {
+ CLOSE_INVALIDATE (sock);
+ retval = WARC_ERR;
+ goto cleanup;
+ }
+
+ /* warc_write_request_record has also closed warc_tmp. */
+ }
+
+ /* Repeat while we receive a 10x response code. */
+ {
+ bool _repeat;
+
+ do
+ {
+ head = read_http_response_head (sock);
+ if (!head)
+ {
+ if (errno == 0)
+ {
+ logputs (LOG_NOTQUIET, _("No data received.\n"));
+ CLOSE_INVALIDATE (sock);
+ retval = HEOF;
+ }
+ else
+ {
+ logprintf (LOG_NOTQUIET, _("Read error (%s) in headers.\n"),
+ fd_errstr (sock));
+ CLOSE_INVALIDATE (sock);
+ retval = HERR;
+ }
+ goto cleanup;
+ }
+ DEBUGP (("\n---response begin---\n%s---response end---\n", head));
+
+ resp = resp_new (head);
+
+ /* Check for status line. */
+ xfree (message);
+ statcode = resp_status (resp, &message);
+ if (statcode < 0)
+ {
+ char *tms = datetime_str (time (NULL));
+ logprintf (LOG_VERBOSE, "%d\n", statcode);
+ logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"), tms, statcode,
+ quotearg_style (escape_quoting_style,
+ _("Malformed status line")));
+ CLOSE_INVALIDATE (sock);
+ retval = HERR;
+ goto cleanup;
+ }
+
+ if (H_10X (statcode))
+ {
+ xfree (head);
+ resp_free (&resp);
+ _repeat = true;
+ DEBUGP (("Ignoring response\n"));
+ }
+ else
+ {
+ _repeat = false;
+ }
+ }
+ while (_repeat);
+ }
+
+ xfree (hs->message);
+ hs->message = xstrdup (message);
+ if (!opt.server_response)
+ logprintf (LOG_VERBOSE, "%2d %s\n", statcode,
+ message ? quotearg_style (escape_quoting_style, message) : "");
+ else
+ {
+ logprintf (LOG_VERBOSE, "\n");
+ print_server_response (resp, " ");
+ }
+
+ if (!opt.ignore_length
+ && resp_header_copy (resp, "Content-Length", hdrval, sizeof (hdrval)))
+ {
+ wgint parsed;
+ errno = 0;
+ parsed = str_to_wgint (hdrval, NULL, 10);
+ if (parsed == WGINT_MAX && errno == ERANGE)
+ {
+ /* Out of range.
+ #### If Content-Length is out of range, it most likely
+ means that the file is larger than 2G and that we're
+ compiled without LFS. In that case we should probably
+ refuse to even attempt to download the file. */
+ contlen = -1;
+ }
+ else if (parsed < 0)
+ {
+ /* Negative Content-Length; nonsensical, so we can't
+ assume any information about the content to receive. */
+ contlen = -1;
+ }
+ else
+ contlen = parsed;
+ }
+
+ /* Check for keep-alive related responses. */
+ if (!inhibit_keep_alive)
+ {
+ if (resp_header_copy (resp, "Connection", hdrval, sizeof (hdrval)))
+ {
+ if (0 == c_strcasecmp (hdrval, "Close"))
+ keep_alive = false;
+ }
+ }
+
+ chunked_transfer_encoding = false;
+ if (resp_header_copy (resp, "Transfer-Encoding", hdrval, sizeof (hdrval))
+ && 0 == c_strcasecmp (hdrval, "chunked"))
+ chunked_transfer_encoding = true;
+
+ /* Handle (possibly multiple instances of) the Set-Cookie header. */
+ if (opt.cookies)
+ {
+ int scpos;
+ const char *scbeg, *scend;
+ /* The jar should have been created by now. */
+ assert (wget_cookie_jar != NULL);
+ for (scpos = 0;
+ (scpos = resp_header_locate (resp, "Set-Cookie", scpos,
+ &scbeg, &scend)) != -1;
+ ++scpos)
+ {
+ char buf[1024], *set_cookie;
+ size_t len = scend - scbeg;
+
+ if (len < sizeof (buf))
+ set_cookie = buf;
+ else
+ set_cookie = xmalloc (len + 1);
+
+ memcpy (set_cookie, scbeg, len);
+ set_cookie[len] = 0;
+
+ cookie_handle_set_cookie (wget_cookie_jar, u->host, u->port,
+ u->path, set_cookie);
+
+ if (set_cookie != buf)
+ xfree (set_cookie);
+ }
+ }
+
+ if (keep_alive)
+ /* The server has promised that it will not close the connection
+ when we're done. This means that we can register it. */
+ register_persistent (conn->host, conn->port, sock, using_ssl);
+
+#ifdef HAVE_METALINK
+ /* We need to check for the Metalink data in the very first response
+ we get from the server (before redirections, authorization, etc.). */
+ if (metalink)
+ {
+ hs->metalink = metalink_from_http (resp, hs, u);
+ /* Bugfix: hs->local_file is NULL (opt.content_disposition). */
+ if (!hs->local_file && hs->metalink && hs->metalink->origin)
+ hs->local_file = xstrdup (hs->metalink->origin);
+ xfree (hs->message);
+ retval = RETR_WITH_METALINK;
+ CLOSE_FINISH (sock);
+ goto cleanup;
+ }
+#endif
+
+ if (statcode == HTTP_STATUS_UNAUTHORIZED)
+ {
+ /* Authorization is required. */
+ uerr_t auth_err = RETROK;
+ bool retry;
+ /* Normally we are not interested in the response body.
+ But if we are writing a WARC file we are: we like to keep everything. */
+ if (warc_enabled)
+ {
+ int _err;
+ type = resp_header_strdup (resp, "Content-Type");
+ _err = read_response_body (hs, sock, NULL, contlen, 0,
+ chunked_transfer_encoding,
+ u->url, warc_timestamp_str,
+ warc_request_uuid, warc_ip, type,
+ statcode, head);
+ xfree (type);
+
+ if (_err != RETRFINISHED || hs->res < 0)
+ {
+ CLOSE_INVALIDATE (sock);
+ retval = _err;
+ goto cleanup;
+ }
+ else
+ CLOSE_FINISH (sock);
+ }
+ else
+ {
+ /* Since WARC is disabled, we are not interested in the response body. */
+ if (keep_alive && !head_only
+ && skip_short_body (sock, contlen, chunked_transfer_encoding))
+ CLOSE_FINISH (sock);
+ else
+ CLOSE_INVALIDATE (sock);
+ }
+
+ pconn.authorized = false;
+
+ {
+ auth_err = check_auth (u, user, passwd, resp, req,
+ &ntlm_seen, &retry,
+ &basic_auth_finished,
+ &auth_finished);
+ if (auth_err == RETROK && retry)
+ {
+ resp_free (&resp);
+ xfree (message);
+ xfree (head);
+ goto retry_with_auth;
+ }
+ }
+ if (auth_err == RETROK)
+ retval = AUTHFAILED;
+ else
+ retval = auth_err;
+ goto cleanup;
+ }
+ else /* statcode != HTTP_STATUS_UNAUTHORIZED */
+ {
+ /* Kludge: if NTLM is used, mark the TCP connection as authorized. */
+ if (ntlm_seen)
+ pconn.authorized = true;
+ }
+
+ {
+ uerr_t ret = check_file_output (u, hs, resp, hdrval, sizeof hdrval);
+ if (ret != RETROK)
+ {
+ retval = ret;
+ goto cleanup;
+ }
+ }
+
+ hs->statcode = statcode;
+ xfree (hs->error);
+ if (statcode == -1)
+ hs->error = xstrdup (_("Malformed status line"));
+ else if (!message || !*message)
+ hs->error = xstrdup (_("(no description)"));
+ else
+ hs->error = xstrdup (message);
+
+#ifdef HAVE_HSTS
+ if (opt.hsts && hsts_store)
+ {
+ hsts_params = resp_header_strdup (resp, "Strict-Transport-Security");
+ if (parse_strict_transport_security (hsts_params, &max_age, &include_subdomains))
+ {
+ /* process strict transport security */
+ if (hsts_store_entry (hsts_store, u->scheme, u->host, u->port, max_age, include_subdomains))
+ DEBUGP(("Added new HSTS host: %s:%u (max-age: %lu, includeSubdomains: %s)\n",
+ u->host,
+ (unsigned) u->port,
+ (unsigned long) max_age,
+ (include_subdomains ? "true" : "false")));
+ else
+ DEBUGP(("Updated HSTS host: %s:%u (max-age: %lu, includeSubdomains: %s)\n",
+ u->host,
+ (unsigned) u->port,
+ (unsigned long) max_age,
+ (include_subdomains ? "true" : "false")));
+ }
+ xfree (hsts_params);
+ }
+#endif
+
+ type = resp_header_strdup (resp, "Content-Type");
+ if (type)
+ {
+ char *tmp = strchr (type, ';');
+ if (tmp)
+ {
+#ifdef ENABLE_IRI
+ /* sXXXav: only needed if IRI support is enabled */
+ char *tmp2 = tmp + 1;
+#endif
+
+ while (tmp > type && c_isspace (tmp[-1]))
+ --tmp;
+ *tmp = '\0';
+
+#ifdef ENABLE_IRI
+ /* Try to get remote encoding if needed */
+ if (opt.enable_iri && !opt.encoding_remote)
+ {
+ tmp = parse_charset (tmp2);
+ if (tmp)
+ set_content_encoding (iri, tmp);
+ xfree (tmp);
+ }
+#endif
+ }
+ }
+ xfree (hs->newloc);
+ hs->newloc = resp_header_strdup (resp, "Location");
+ xfree (hs->remote_time);
+ hs->remote_time = resp_header_strdup (resp, "Last-Modified");
+ if (!hs->remote_time) // now look for the Wayback Machine's timestamp
+ hs->remote_time = resp_header_strdup (resp, "X-Archive-Orig-last-modified");
+
+ if (resp_header_copy (resp, "Content-Range", hdrval, sizeof (hdrval)))
+ {
+ wgint first_byte_pos, last_byte_pos, entity_length;
+ if (parse_content_range (hdrval, &first_byte_pos, &last_byte_pos,
+ &entity_length))
+ {
+ contrange = first_byte_pos;
+ contlen = last_byte_pos - first_byte_pos + 1;
+ }
+ }
+
+ if (resp_header_copy (resp, "Content-Encoding", hdrval, sizeof (hdrval)))
+ {
+ hs->local_encoding = ENC_INVALID;
+
+ switch (hdrval[0])
+ {
+ case 'b': case 'B':
+ if (0 == c_strcasecmp(hdrval, "br"))
+ hs->local_encoding = ENC_BROTLI;
+ break;
+ case 'c': case 'C':
+ if (0 == c_strcasecmp(hdrval, "compress"))
+ hs->local_encoding = ENC_COMPRESS;
+ break;
+ case 'd': case 'D':
+ if (0 == c_strcasecmp(hdrval, "deflate"))
+ hs->local_encoding = ENC_DEFLATE;
+ break;
+ case 'g': case 'G':
+ if (0 == c_strcasecmp(hdrval, "gzip"))
+ hs->local_encoding = ENC_GZIP;
+ break;
+ case 'i': case 'I':
+ if (0 == c_strcasecmp(hdrval, "identity"))
+ hs->local_encoding = ENC_NONE;
+ break;
+ case 'x': case 'X':
+ if (0 == c_strcasecmp(hdrval, "x-compress"))
+ hs->local_encoding = ENC_COMPRESS;
+ else if (0 == c_strcasecmp(hdrval, "x-gzip"))
+ hs->local_encoding = ENC_GZIP;
+ break;
+ case '\0':
+ hs->local_encoding = ENC_NONE;
+ }
+
+ if (hs->local_encoding == ENC_INVALID)
+ {
+ DEBUGP (("Unrecognized Content-Encoding: %s\n", hdrval));
+ hs->local_encoding = ENC_NONE;
+ }
+#ifdef HAVE_LIBZ
+ else if (hs->local_encoding == ENC_GZIP
+ && opt.compression != compression_none)
+ {
+ const char *p;
+
+ /* Make sure the Content-Type is not gzip before decompressing */
+ if (type)
+ {
+ p = strchr (type, '/');
+ if (p == NULL)
+ {
+ hs->remote_encoding = ENC_GZIP;
+ hs->local_encoding = ENC_NONE;
+ }
+ else
+ {
+ p++;
+ if (c_tolower(p[0]) == 'x' && p[1] == '-')
+ p += 2;
+ if (0 != c_strcasecmp (p, "gzip"))
+ {
+ hs->remote_encoding = ENC_GZIP;
+ hs->local_encoding = ENC_NONE;
+ }
+ }
+ }
+ else
+ {
+ hs->remote_encoding = ENC_GZIP;
+ hs->local_encoding = ENC_NONE;
+ }
+
+ /* don't uncompress if a file ends with '.gz' or '.tgz' */
+ if (hs->remote_encoding == ENC_GZIP
+ && (p = strrchr(u->file, '.'))
+ && (c_strcasecmp(p, ".gz") == 0 || c_strcasecmp(p, ".tgz") == 0))
+ {
+ DEBUGP (("Enabling broken server workaround. Will not decompress this GZip file.\n"));
+ hs->remote_encoding = ENC_NONE;
+ }
+ }
+#endif
+ }
+
+ /* 20x responses are counted among successful by default. */
+ if (H_20X (statcode))
+ *dt |= RETROKF;
+
+ if (statcode == HTTP_STATUS_NO_CONTENT)
+ {
+ /* 204 response has no body (RFC 2616, 4.3) */
+
+ /* In case the caller cares to look... */
+ hs->len = 0;
+ hs->res = 0;
+ hs->restval = 0;
+
+ CLOSE_FINISH (sock);
+
+ retval = RETRFINISHED;
+ goto cleanup;
+ }
+
+ /* Return if redirected. */
+ if (H_REDIRECTED (statcode) || statcode == HTTP_STATUS_MULTIPLE_CHOICES)
+ {
+ /* RFC2068 says that in case of the 300 (multiple choices)
+ response, the server can output a preferred URL through
+ `Location' header; otherwise, the request should be treated
+ like GET. So, if the location is set, it will be a
+ redirection; otherwise, just proceed normally. */
+ if (statcode == HTTP_STATUS_MULTIPLE_CHOICES && !hs->newloc)
+ *dt |= RETROKF;
+ else
+ {
+ logprintf (LOG_VERBOSE,
+ _("Location: %s%s\n"),
+ hs->newloc ? escnonprint_uri (hs->newloc) : _("unspecified"),
+ hs->newloc ? _(" [following]") : "");
+
+ /* In case the caller cares to look... */
+ hs->len = 0;
+ hs->res = 0;
+ hs->restval = 0;
+
+ /* Normally we are not interested in the response body of a redirect.
+ But if we are writing a WARC file we are: we like to keep everything. */
+ if (warc_enabled)
+ {
+ int _err = read_response_body (hs, sock, NULL, contlen, 0,
+ chunked_transfer_encoding,
+ u->url, warc_timestamp_str,
+ warc_request_uuid, warc_ip, type,
+ statcode, head);
+
+ if (_err != RETRFINISHED || hs->res < 0)
+ {
+ CLOSE_INVALIDATE (sock);
+ retval = _err;
+ goto cleanup;
+ }
+ else
+ CLOSE_FINISH (sock);
+ }
+ else
+ {
+ /* Since WARC is disabled, we are not interested in the response body. */
+ if (keep_alive && !head_only
+ && skip_short_body (sock, contlen, chunked_transfer_encoding))
+ CLOSE_FINISH (sock);
+ else
+ CLOSE_INVALIDATE (sock);
+ }
+
+ /* From RFC2616: The status codes 303 and 307 have
+ been added for servers that wish to make unambiguously
+ clear which kind of reaction is expected of the client.
+
+ A 307 should be redirected using the same method,
+ in other words, a POST should be preserved and not
+ converted to a GET in that case.
+
+ With strict adherence to RFC2616, POST requests are not
+ converted to a GET request on 301 Permanent Redirect
+ or 302 Temporary Redirect.
+
+ A switch may be provided later based on the HTTPbis draft
+ that allows clients to convert POST requests to GET
+ requests on 301 and 302 response codes. */
+ switch (statcode)
+ {
+ case HTTP_STATUS_TEMPORARY_REDIRECT:
+ case HTTP_STATUS_PERMANENT_REDIRECT:
+ retval = NEWLOCATION_KEEP_POST;
+ goto cleanup;
+ case HTTP_STATUS_MOVED_PERMANENTLY:
+ if (opt.method && c_strcasecmp (opt.method, "post") != 0)
+ {
+ retval = NEWLOCATION_KEEP_POST;
+ goto cleanup;
+ }
+ break;
+ case HTTP_STATUS_MOVED_TEMPORARILY:
+ if (opt.method && c_strcasecmp (opt.method, "post") != 0)
+ {
+ retval = NEWLOCATION_KEEP_POST;
+ goto cleanup;
+ }
+ break;
+ }
+ retval = NEWLOCATION;
+ goto cleanup;
+ }
+ }
+
+ if (cond_get)
+ {
+ if (statcode == HTTP_STATUS_NOT_MODIFIED)
+ {
+ logprintf (LOG_VERBOSE,
+ _ ("File %s not modified on server. Omitting download.\n\n"),
+ quote (hs->local_file));
+ *dt |= RETROKF;
+ CLOSE_FINISH (sock);
+ retval = RETRUNNEEDED;
+ goto cleanup;
+ }
+ }
+
+ set_content_type (dt, type);
+
+ if (opt.adjust_extension)
+ {
+ const char *encoding_ext = NULL;
+ switch (hs->local_encoding)
+ {
+ case ENC_INVALID:
+ case ENC_NONE:
+ break;
+ case ENC_BROTLI:
+ encoding_ext = ".br";
+ break;
+ case ENC_COMPRESS:
+ encoding_ext = ".Z";
+ break;
+ case ENC_DEFLATE:
+ encoding_ext = ".zlib";
+ break;
+ case ENC_GZIP:
+ encoding_ext = ".gz";
+ break;
+ default:
+ DEBUGP (("No extension found for encoding %d\n",
+ hs->local_encoding));
+ }
+ if (encoding_ext != NULL)
+ {
+ char *file_ext = strrchr (hs->local_file, '.');
+ /* strip Content-Encoding extension (it will be re-added later) */
+ if (file_ext != NULL && 0 == strcasecmp (file_ext, encoding_ext))
+ *file_ext = '\0';
+ }
+ if (*dt & TEXTHTML)
+ /* -E / --adjust-extension / adjust_extension = on was specified,
+ and this is a text/html file. If some case-insensitive
+ variation on ".htm[l]" isn't already the file's suffix,
+ tack on ".html". */
+ {
+ ensure_extension (hs, ".html", dt);
+ }
+ else if (*dt & TEXTCSS)
+ {
+ ensure_extension (hs, ".css", dt);
+ }
+ if (encoding_ext != NULL)
+ {
+ ensure_extension (hs, encoding_ext, dt);
+ }
+ }
+
+ if (cond_get)
+ {
+ /* Handle the case when server ignores If-Modified-Since header. */
+ if (statcode == HTTP_STATUS_OK && hs->remote_time)
+ {
+ time_t tmr = http_atotm (hs->remote_time);
+
+ /* Check if the local file is up-to-date based on Last-Modified header
+ and content length. */
+ if (tmr != (time_t) - 1 && tmr <= hs->orig_file_tstamp
+ && (contlen == -1 || contlen == hs->orig_file_size))
+ {
+ logprintf (LOG_VERBOSE,
+ _("Server ignored If-Modified-Since header for file %s.\n"
+ "You might want to add --no-if-modified-since option."
+ "\n\n"),
+ quote (hs->local_file));
+ *dt |= RETROKF;
+ CLOSE_INVALIDATE (sock);
+ retval = RETRUNNEEDED;
+ goto cleanup;
+ }
+ }
+ }
+
+ if (statcode == HTTP_STATUS_RANGE_NOT_SATISFIABLE
+ || (!opt.timestamping && hs->restval > 0 && statcode == HTTP_STATUS_OK
+ && contrange == 0 && contlen >= 0 && hs->restval >= contlen))
+ {
+ /* If `-c' is in use and the file has been fully downloaded (or
+ the remote file has shrunk), Wget effectively requests bytes
+ after the end of file and the server response with 416
+ (or 200 with a <= Content-Length. */
+ logputs (LOG_VERBOSE, _("\
+\n The file is already fully retrieved; nothing to do.\n\n"));
+ /* In case the caller inspects. */
+ hs->len = contlen;
+ hs->res = 0;
+ /* Mark as successfully retrieved. */
+ *dt |= RETROKF;
+
+ /* Try to maintain the keep-alive connection. It is often cheaper to
+ * consume some bytes which have already been sent than to negotiate
+ * a new connection. However, if the body is too large, or we don't
+ * care about keep-alive, then simply terminate the connection */
+ if (keep_alive &&
+ skip_short_body (sock, contlen, chunked_transfer_encoding))
+ CLOSE_FINISH (sock);
+ else
+ CLOSE_INVALIDATE (sock);
+ retval = RETRUNNEEDED;
+ goto cleanup;
+ }
+ if ((contrange != 0 && contrange != hs->restval)
+ || (H_PARTIAL (statcode) && !contrange && hs->restval))
+ {
+ /* The Range request was somehow misunderstood by the server.
+ Bail out. */
+ CLOSE_INVALIDATE (sock);
+ retval = RANGEERR;
+ goto cleanup;
+ }
+ if (contlen == -1)
+ hs->contlen = -1;
+ /* If the response is gzipped, the uncompressed size is unknown. */
+ else if (hs->remote_encoding == ENC_GZIP)
+ hs->contlen = -1;
+ else
+ hs->contlen = contlen + contrange;
+
+ if (opt.verbose)
+ {
+ if (*dt & RETROKF)
+ {
+ /* No need to print this output if the body won't be
+ downloaded at all, or if the original server response is
+ printed. */
+ logputs (LOG_VERBOSE, _("Length: "));
+ if (contlen != -1)
+ {
+ logputs (LOG_VERBOSE, number_to_static_string (contlen + contrange));
+ if (contlen + contrange >= 1024)
+ logprintf (LOG_VERBOSE, " (%s)",
+ human_readable (contlen + contrange, 10, 1));
+ if (contrange)
+ {
+ if (contlen >= 1024)
+ logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
+ number_to_static_string (contlen),
+ human_readable (contlen, 10, 1));
+ else
+ logprintf (LOG_VERBOSE, _(", %s remaining"),
+ number_to_static_string (contlen));
+ }
+ }
+ else
+ logputs (LOG_VERBOSE,
+ opt.ignore_length ? _("ignored") : _("unspecified"));
+ if (type)
+ logprintf (LOG_VERBOSE, " [%s]\n", quotearg_style (escape_quoting_style, type));
+ else
+ logputs (LOG_VERBOSE, "\n");
+ }
+ }
+
+ /* Return if we have no intention of further downloading. */
+ if ((!(*dt & RETROKF) && !opt.content_on_error) || head_only || (opt.spider && !opt.recursive))
+ {
+ /* In case the caller cares to look... */
+ hs->len = 0;
+ hs->res = 0;
+ hs->restval = 0;
+
+ /* Normally we are not interested in the response body of a error responses.
+ But if we are writing a WARC file we are: we like to keep everything. */
+ if (warc_enabled)
+ {
+ int _err = read_response_body (hs, sock, NULL, contlen, 0,
+ chunked_transfer_encoding,
+ u->url, warc_timestamp_str,
+ warc_request_uuid, warc_ip, type,
+ statcode, head);
+
+ if (_err != RETRFINISHED || hs->res < 0)
+ {
+ CLOSE_INVALIDATE (sock);
+ retval = _err;
+ goto cleanup;
+ }
+
+ CLOSE_FINISH (sock);
+ }
+ else
+ {
+ /* Since WARC is disabled, we are not interested in the response body. */
+ if (head_only)
+ /* Pre-1.10 Wget used CLOSE_INVALIDATE here. Now we trust the
+ servers not to send body in response to a HEAD request, and
+ those that do will likely be caught by test_socket_open.
+ If not, they can be worked around using
+ `--no-http-keep-alive'. */
+ CLOSE_FINISH (sock);
+ else if (opt.spider && !opt.recursive)
+ /* we just want to see if the page exists - no downloading required */
+ CLOSE_INVALIDATE (sock);
+ else if (keep_alive
+ && skip_short_body (sock, contlen, chunked_transfer_encoding))
+ /* Successfully skipped the body; also keep using the socket. */
+ CLOSE_FINISH (sock);
+ else
+ CLOSE_INVALIDATE (sock);
+ }
+
+ if (statcode == HTTP_STATUS_GATEWAY_TIMEOUT)
+ retval = GATEWAYTIMEOUT;
+ else
+ retval = RETRFINISHED;
+
+ goto cleanup;
+ }
+
+ err = open_output_stream (hs, count, &fp);
+ if (err != RETROK)
+ {
+ CLOSE_INVALIDATE (sock);
+ retval = err;
+ goto cleanup;
+ }
+
+#ifdef ENABLE_XATTR
+ if (opt.enable_xattr)
+ {
+ if (original_url != u)
+ set_file_metadata (u, original_url, fp);
+ else
+ set_file_metadata (u, NULL, fp);
+ }
+#endif
+
+ err = read_response_body (hs, sock, fp, contlen, contrange,
+ chunked_transfer_encoding,
+ u->url, warc_timestamp_str,
+ warc_request_uuid, warc_ip, type,
+ statcode, head);
+
+ if (hs->res >= 0)
+ CLOSE_FINISH (sock);
+ else
+ CLOSE_INVALIDATE (sock);
+
+ if (!output_stream)
+ fclose (fp);
+
+ retval = err;
+
+ cleanup:
+ xfree (head);
+ xfree (type);
+ xfree (message);
+ resp_free (&resp);
+ request_free (&req);
+
+ return retval;
+}
+
+/* Check whether the supplied HTTP status code is among those
+ listed for the --retry-on-http-error option. */
+static bool
+check_retry_on_http_error (const int statcode)
+{
+ const char *tok = opt.retry_on_http_error;
+ while (tok && *tok)
+ {
+ if (atoi (tok) == statcode)
+ return true;
+ if ((tok = strchr (tok, ',')))
+ ++tok;
+ }
+ return false;
+}
+
+/* The genuine HTTP loop! This is the part where the retrieval is
+ retried, and retried, and retried, and... */
+uerr_t
+http_loop (const struct url *u, struct url *original_url, char **newloc,
+ char **local_file, const char *referer, int *dt, struct url *proxy,
+ struct iri *iri)
+{
+ int count;
+ bool got_head = false; /* used for time-stamping and filename detection */
+ bool time_came_from_head = false;
+ bool got_name = false;
+ char *tms;
+ const char *tmrate;
+ uerr_t err, ret = TRYLIMEXC;
+ time_t tmr = -1; /* remote time-stamp */
+ struct http_stat hstat; /* HTTP status */
+ struct stat st;
+ bool send_head_first = true;
+ bool force_full_retrieve = false;
+
+
+ /* If we are writing to a WARC file: always retrieve the whole file. */
+ if (opt.warc_filename != NULL)
+ force_full_retrieve = true;
+
+
+ /* Assert that no value for *LOCAL_FILE was passed. */
+ assert (local_file == NULL || *local_file == NULL);
+
+ /* Set LOCAL_FILE parameter. */
+ if (local_file && opt.output_document)
+ *local_file = HYPHENP (opt.output_document) ? NULL : xstrdup (opt.output_document);
+
+ /* Reset NEWLOC parameter. */
+ *newloc = NULL;
+
+ /* This used to be done in main, but it's a better idea to do it
+ here so that we don't go through the hoops if we're just using
+ FTP or whatever. */
+ if (opt.cookies)
+ load_cookies ();
+
+ /* Warn on (likely bogus) wildcard usage in HTTP. */
+ if (opt.ftp_glob && has_wildcards_p (u->path))
+ logputs (LOG_VERBOSE, _("Warning: wildcards not supported in HTTP.\n"));
+
+ /* Setup hstat struct. */
+ xzero (hstat);
+ hstat.referer = referer;
+
+ if (opt.output_document)
+ {
+ hstat.local_file = xstrdup (opt.output_document);
+ got_name = true;
+ }
+ else if (!opt.content_disposition)
+ {
+ hstat.local_file =
+ url_file_name (opt.trustservernames ? u : original_url, NULL);
+ got_name = true;
+ }
+
+ if (got_name && file_exists_p (hstat.local_file, NULL) && opt.noclobber && !opt.output_document)
+ {
+ /* If opt.noclobber is turned on and file already exists, do not
+ retrieve the file. But if the output_document was given, then this
+ test was already done and the file didn't exist. Hence the !opt.output_document */
+ get_file_flags (hstat.local_file, dt);
+ ret = RETROK;
+ goto exit;
+ }
+
+ /* Reset the counter. */
+ count = 0;
+
+ /* Reset the document type. */
+ *dt = 0;
+
+ /* Skip preliminary HEAD request if we're not in spider mode. */
+ if (!opt.spider)
+ send_head_first = false;
+
+ /* Send preliminary HEAD request if --content-disposition and -c are used
+ together. */
+ if (opt.content_disposition && opt.always_rest)
+ send_head_first = true;
+
+#ifdef HAVE_METALINK
+ if (opt.metalink_over_http)
+ {
+ *dt |= METALINK_METADATA;
+ send_head_first = true;
+ }
+#endif
+
+ if (opt.timestamping)
+ {
+ /* Use conditional get request if requested
+ * and if timestamp is known at this moment. */
+ if (opt.if_modified_since && !send_head_first && got_name && file_exists_p (hstat.local_file, NULL))
+ {
+ *dt |= IF_MODIFIED_SINCE;
+ {
+ uerr_t timestamp_err = set_file_timestamp (&hstat);
+ if (timestamp_err != RETROK)
+ return timestamp_err;
+ }
+ }
+ /* Send preliminary HEAD request if -N is given and we have existing
+ * destination file or content disposition is enabled. */
+ else if (opt.content_disposition || file_exists_p (hstat.local_file, NULL))
+ send_head_first = true;
+ }
+
+ /* THE loop */
+ do
+ {
+ /* Increment the pass counter. */
+ ++count;
+ sleep_between_retrievals (count);
+
+ /* Get the current time string. */
+ tms = datetime_str (time (NULL));
+
+ if (opt.spider && !got_head)
+ logprintf (LOG_VERBOSE,
+ _("Spider mode enabled. Check if remote file exists.\n"));
+
+ /* Print fetch message, if opt.verbose. */
+ if (opt.verbose)
+ {
+ char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
+
+ if (count > 1)
+ {
+ char tmp[256];
+ sprintf (tmp, _("(try:%2d)"), count);
+ logprintf (LOG_NOTQUIET, "--%s-- %s %s\n",
+ tms, tmp, hurl);
+ }
+ else
+ {
+ logprintf (LOG_NOTQUIET, "--%s-- %s\n",
+ tms, hurl);
+ }
+
+#ifdef WINDOWS
+ ws_changetitle (hurl);
+#endif
+ xfree (hurl);
+ }
+
+ /* Default document type is empty. However, if spider mode is
+ on or time-stamping is employed, HEAD_ONLY commands is
+ encoded within *dt. */
+ if (send_head_first && !got_head)
+ *dt |= HEAD_ONLY;
+ else
+ *dt &= ~HEAD_ONLY;
+
+ /* Decide whether or not to restart. */
+ if (force_full_retrieve)
+ hstat.restval = hstat.len;
+ else if (opt.start_pos >= 0)
+ hstat.restval = opt.start_pos;
+ else if (opt.always_rest
+ && got_name
+ && stat (hstat.local_file, &st) == 0
+ && S_ISREG (st.st_mode))
+ /* When -c is used, continue from on-disk size. (Can't use
+ hstat.len even if count>1 because we don't want a failed
+ first attempt to clobber existing data.) */
+ hstat.restval = st.st_size;
+ else if (count > 1)
+ {
+ /* otherwise, continue where the previous try left off */
+ if (hstat.len < hstat.restval)
+ hstat.restval -= hstat.len;
+ else
+ hstat.restval = hstat.len;
+ }
+ else
+ hstat.restval = 0;
+
+ /* Decide whether to send the no-cache directive. We send it in
+ two cases:
+ a) we're using a proxy, and we're past our first retrieval.
+ Some proxies are notorious for caching incomplete data, so
+ we require a fresh get.
+ b) caching is explicitly inhibited. */
+ if ((proxy && count > 1) /* a */
+ || !opt.allow_cache) /* b */
+ *dt |= SEND_NOCACHE;
+ else
+ *dt &= ~SEND_NOCACHE;
+
+ /* Try fetching the document, or at least its head. */
+ err = gethttp (u, original_url, &hstat, dt, proxy, iri, count);
+
+ /* Time? */
+ tms = datetime_str (time (NULL));
+
+ /* Get the new location (with or without the redirection). */
+ if (hstat.newloc)
+ *newloc = xstrdup (hstat.newloc);
+
+ switch (err)
+ {
+ case HERR: case HEOF: case CONSOCKERR:
+ case CONERROR: case READERR: case WRITEFAILED:
+ case RANGEERR: case FOPEN_EXCL_ERR: case GATEWAYTIMEOUT:
+ /* Non-fatal errors continue executing the loop, which will
+ bring them to "while" statement at the end, to judge
+ whether the number of tries was exceeded. */
+ printwhat (count, opt.ntry);
+ continue;
+ case FWRITEERR: case FOPENERR:
+ /* Another fatal error. */
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("Cannot write to %s (%s).\n"),
+ quote (hstat.local_file), strerror (errno));
+ ret = err;
+ goto exit;
+ case HOSTERR:
+ /* Fatal unless option set otherwise. */
+ if ( opt.retry_on_host_error )
+ {
+ printwhat (count, opt.ntry);
+ continue;
+ }
+ ret = err;
+ goto exit;
+ case CONIMPOSSIBLE: case PROXERR: case SSLINITFAILED:
+ case CONTNOTSUPPORTED: case VERIFCERTERR: case FILEBADFILE:
+ case UNKNOWNATTR:
+ /* Fatal errors just return from the function. */
+ ret = err;
+ goto exit;
+ case ATTRMISSING:
+ /* A missing attribute in a Header is a fatal Protocol error. */
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("Required attribute missing from Header received.\n"));
+ ret = err;
+ goto exit;
+ case AUTHFAILED:
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("Username/Password Authentication Failed.\n"));
+ ret = err;
+ goto exit;
+ case WARC_ERR:
+ /* A fatal WARC error. */
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("Cannot write to WARC file.\n"));
+ ret = err;
+ goto exit;
+ case WARC_TMP_FOPENERR: case WARC_TMP_FWRITEERR:
+ /* A fatal WARC error. */
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("Cannot write to temporary WARC file.\n"));
+ ret = err;
+ goto exit;
+ case CONSSLERR:
+ /* Another fatal error. */
+ logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n"));
+ ret = err;
+ goto exit;
+ case UNLINKERR:
+ /* Another fatal error. */
+ logputs (LOG_VERBOSE, "\n");
+ logprintf (LOG_NOTQUIET, _("Cannot unlink %s (%s).\n"),
+ quote (hstat.local_file), strerror (errno));
+ ret = err;
+ goto exit;
+ case NEWLOCATION:
+ case NEWLOCATION_KEEP_POST:
+ /* Return the new location to the caller. */
+ if (!*newloc)
+ {
+ logprintf (LOG_NOTQUIET,
+ _("ERROR: Redirection (%d) without location.\n"),
+ hstat.statcode);
+ ret = WRONGCODE;
+ }
+ else
+ {
+ ret = err;
+ }
+ goto exit;
+ case RETRUNNEEDED:
+ /* The file was already fully retrieved. */
+ ret = RETROK;
+ goto exit;
+ case RETRFINISHED:
+ /* Deal with you later. */
+ break;
+#ifdef HAVE_METALINK
+ case RETR_WITH_METALINK:
+ {
+ if (hstat.metalink == NULL)
+ {
+ logputs (LOG_NOTQUIET,
+ _("Could not find Metalink data in HTTP response. "
+ "Downloading file using HTTP GET.\n"));
+ *dt &= ~METALINK_METADATA;
+ *dt &= ~HEAD_ONLY;
+ got_head = true;
+ continue;
+ }
+
+ logputs (LOG_VERBOSE,
+ _("Metalink headers found. "
+ "Switching to Metalink mode.\n"));
+
+ ret = retrieve_from_metalink (hstat.metalink);
+ goto exit;
+ }
+ break;
+#endif
+ default:
+ /* All possibilities should have been exhausted. */
+ abort ();
+ }
+
+ if (!(*dt & RETROKF))
+ {
+ char *hurl = NULL;
+ if (!opt.verbose)
+ {
+ /* #### Ugly ugly ugly! */
+ hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
+ logprintf (LOG_NONVERBOSE, "%s:\n", hurl);
+ }
+
+ /* Fall back to GET if HEAD fails with a 500 or 501 error code. */
+ if (*dt & HEAD_ONLY
+ && (hstat.statcode == 500 || hstat.statcode == 501))
+ {
+ got_head = true;
+ xfree (hurl);
+ continue;
+ }
+ /* Maybe we should always keep track of broken links, not just in
+ * spider mode.
+ * Don't log error if it was UTF-8 encoded because we will try
+ * once unencoded. */
+ else if (opt.spider && !iri->utf8_encode)
+ {
+ /* #### Again: ugly ugly ugly! */
+ if (!hurl)
+ hurl = url_string (u, URL_AUTH_HIDE_PASSWD);
+ nonexisting_url (hurl);
+ logprintf (LOG_NOTQUIET, _("\
+Remote file does not exist -- broken link!!!\n"));
+ }
+ else if (check_retry_on_http_error (hstat.statcode))
+ {
+ printwhat (count, opt.ntry);
+ xfree (hurl);
+ continue;
+ }
+ else
+ {
+ logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"),
+ tms, hstat.statcode,
+ quotearg_style (escape_quoting_style, hstat.error));
+ }
+ logputs (LOG_VERBOSE, "\n");
+ ret = WRONGCODE;
+ xfree (hurl);
+ goto exit;
+ }
+
+ /* Did we get the time-stamp? */
+ if (!got_head || (opt.spider && !opt.recursive))
+ {
+ got_head = true; /* no more time-stamping */
+
+ if (opt.timestamping && !hstat.remote_time)
+ {
+ logputs (LOG_NOTQUIET, _("\
+Last-modified header missing -- time-stamps turned off.\n"));
+ }
+ else if (hstat.remote_time)
+ {
+ /* Convert the date-string into struct tm. */
+ tmr = http_atotm (hstat.remote_time);
+ if (tmr == (time_t) (-1))
+ logputs (LOG_VERBOSE, _("\
+Last-modified header invalid -- time-stamp ignored.\n"));
+ if (*dt & HEAD_ONLY)
+ time_came_from_head = true;
+ }
+
+ if (send_head_first)
+ {
+ /* The time-stamping section. */
+ if (opt.timestamping)
+ {
+ if (hstat.orig_file_name) /* Perform the following
+ checks only if the file
+ we're supposed to
+ download already exists. */
+ {
+ if (hstat.remote_time &&
+ tmr != (time_t) (-1))
+ {
+ /* Now time-stamping can be used validly.
+ Time-stamping means that if the sizes of
+ the local and remote file match, and local
+ file is newer than the remote file, it will
+ not be retrieved. Otherwise, the normal
+ download procedure is resumed. */
+ if (hstat.orig_file_tstamp >= tmr)
+ {
+ if (hstat.contlen == -1
+ || hstat.orig_file_size == hstat.contlen)
+ {
+ logprintf (LOG_VERBOSE, _("\
+Server file no newer than local file %s -- not retrieving.\n\n"),
+ quote (hstat.orig_file_name));
+ ret = RETROK;
+ goto exit;
+ }
+ else
+ {
+ logprintf (LOG_VERBOSE, _("\
+The sizes do not match (local %s) -- retrieving.\n"),
+ number_to_static_string (hstat.orig_file_size));
+ }
+ }
+ else
+ {
+ force_full_retrieve = true;
+ logputs (LOG_VERBOSE,
+ _("Remote file is newer, retrieving.\n"));
+ }
+
+ logputs (LOG_VERBOSE, "\n");
+ }
+ }
+
+ /* free_hstat (&hstat); */
+ hstat.timestamp_checked = true;
+ }
+
+ if (opt.spider)
+ {
+ bool finished = true;
+ if (opt.recursive)
+ {
+ if ((*dt & TEXTHTML) || (*dt & TEXTCSS))
+ {
+ logputs (LOG_VERBOSE, _("\
+Remote file exists and could contain links to other resources -- retrieving.\n\n"));
+ finished = false;
+ }
+ else
+ {
+ logprintf (LOG_VERBOSE, _("\
+Remote file exists but does not contain any link -- not retrieving.\n\n"));
+ ret = RETROK; /* RETRUNNEEDED is not for caller. */
+ }
+ }
+ else
+ {
+ if ((*dt & TEXTHTML) || (*dt & TEXTCSS))
+ {
+ logprintf (LOG_VERBOSE, _("\
+Remote file exists and could contain further links,\n\
+but recursion is disabled -- not retrieving.\n\n"));
+ }
+ else
+ {
+ logprintf (LOG_VERBOSE, _("\
+Remote file exists.\n\n"));
+ }
+ ret = RETROK; /* RETRUNNEEDED is not for caller. */
+ }
+
+ if (finished)
+ {
+ logprintf (LOG_NONVERBOSE,
+ _("%s URL: %s %2d %s\n"),
+ tms, u->url, hstat.statcode,
+ hstat.message ? quotearg_style (escape_quoting_style, hstat.message) : "");
+ goto exit;
+ }
+ }
+
+ got_name = true;
+ *dt &= ~HEAD_ONLY;
+ count = 0; /* the retrieve count for HEAD is reset */
+ continue;
+ } /* send_head_first */
+ } /* !got_head */
+
+ if (opt.useservertimestamps
+ && (tmr != (time_t) (-1))
+ && ((hstat.len == hstat.contlen) ||
+ ((hstat.res == 0) && (hstat.contlen == -1))))
+ {
+ const char *fl = NULL;
+ set_local_file (&fl, hstat.local_file);
+ if (fl)
+ {
+ time_t newtmr = -1;
+ /* Reparse time header, in case it's changed. */
+ if (time_came_from_head
+ && hstat.remote_time && hstat.remote_time[0])
+ {
+ newtmr = http_atotm (hstat.remote_time);
+ if (newtmr != (time_t)-1)
+ tmr = newtmr;
+ }
+ touch (fl, tmr);
+ }
+ }
+ /* End of time-stamping section. */
+
+ tmrate = retr_rate (hstat.rd_size, hstat.dltime);
+ total_download_time += hstat.dltime;
+
+ if (hstat.len == hstat.contlen)
+ {
+ if (*dt & RETROKF || opt.content_on_error)
+ {
+ bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
+
+ logprintf (LOG_VERBOSE,
+ write_to_stdout
+ ? _("%s (%s) - written to stdout %s[%s/%s]\n\n")
+ : _("%s (%s) - %s saved [%s/%s]\n\n"),
+ tms, tmrate,
+ write_to_stdout ? "" : quote (hstat.local_file),
+ number_to_static_string (hstat.len),
+ number_to_static_string (hstat.contlen));
+ logprintf (LOG_NONVERBOSE,
+ "%s URL:%s [%s/%s] -> \"%s\" [%d]\n",
+ tms, u->url,
+ number_to_static_string (hstat.len),
+ number_to_static_string (hstat.contlen),
+ hstat.local_file, count);
+ }
+ ++numurls;
+ total_downloaded_bytes += hstat.rd_size;
+
+ /* Remember that we downloaded the file for later ".orig" code. */
+ if (*dt & ADDED_HTML_EXTENSION)
+ downloaded_file (FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, hstat.local_file);
+ else
+ downloaded_file (FILE_DOWNLOADED_NORMALLY, hstat.local_file);
+
+ ret = RETROK;
+ goto exit;
+ }
+ else if (hstat.res == 0) /* No read error */
+ {
+ if (hstat.contlen == -1) /* We don't know how much we were supposed
+ to get, so assume we succeeded. */
+ {
+ if (*dt & RETROKF || opt.content_on_error)
+ {
+ bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document));
+
+ logprintf (LOG_VERBOSE,
+ write_to_stdout
+ ? _("%s (%s) - written to stdout %s[%s]\n\n")
+ : _("%s (%s) - %s saved [%s]\n\n"),
+ tms, tmrate,
+ write_to_stdout ? "" : quote (hstat.local_file),
+ number_to_static_string (hstat.len));
+ if (!(opt.verbose || opt.quiet))
+ {
+ char *url = url_string (u, URL_AUTH_HIDE_PASSWD);
+ logprintf (LOG_NONVERBOSE,
+ "%s URL:%s [%s] -> \"%s\" [%d]\n",
+ tms, url, number_to_static_string (hstat.len),
+ hstat.local_file, count);
+ xfree (url);
+ }
+ }
+ ++numurls;
+ total_downloaded_bytes += hstat.rd_size;
+
+ /* Remember that we downloaded the file for later ".orig" code. */
+ if (*dt & ADDED_HTML_EXTENSION)
+ downloaded_file (FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, hstat.local_file);
+ else
+ downloaded_file (FILE_DOWNLOADED_NORMALLY, hstat.local_file);
+
+ ret = RETROK;
+ goto exit;
+ }
+ else if (hstat.len < hstat.contlen) /* meaning we lost the
+ connection too soon */
+ {
+ logprintf (LOG_VERBOSE,
+ _("%s (%s) - Connection closed at byte %s. "),
+ tms, tmrate, number_to_static_string (hstat.len));
+ printwhat (count, opt.ntry);
+ continue;
+ }
+ else if (hstat.len != hstat.restval)
+ /* Getting here would mean reading more data than
+ requested with content-length, which we never do. */
+ abort ();
+ else
+ {
+ /* Getting here probably means that the content-length was
+ * _less_ than the original, local size. We should probably
+ * truncate or re-read, or something. FIXME */
+ ret = RETROK;
+ goto exit;
+ }
+ }
+ else /* from now on hstat.res can only be -1 */
+ {
+ if (hstat.contlen == -1)
+ {
+ logprintf (LOG_VERBOSE,
+ _("%s (%s) - Read error at byte %s (%s)."),
+ tms, tmrate, number_to_static_string (hstat.len),
+ hstat.rderrmsg);
+ printwhat (count, opt.ntry);
+ continue;
+ }
+ else /* hstat.res == -1 and contlen is given */
+ {
+ logprintf (LOG_VERBOSE,
+ _("%s (%s) - Read error at byte %s/%s (%s). "),
+ tms, tmrate,
+ number_to_static_string (hstat.len),
+ number_to_static_string (hstat.contlen),
+ hstat.rderrmsg);
+ printwhat (count, opt.ntry);
+ continue;
+ }
+ }
+ /* not reached */
+ }
+ while (!opt.ntry || (count < opt.ntry));
+
+exit:
+ if ((ret == RETROK || opt.content_on_error) && local_file)
+ {
+ xfree (*local_file);
+ /* Bugfix: Prevent SIGSEGV when hstat.local_file was left NULL
+ (i.e. due to opt.content_disposition). */
+ if (hstat.local_file)
+ {
+ *local_file = hstat.local_file;
+ hstat.local_file = NULL;
+ }
+ }
+ free_hstat (&hstat);
+
+ return ret;
+}
+
+/* Check whether the result of strptime() indicates success.
+ strptime() returns the pointer to how far it got to in the string.
+ The processing has been successful if the string is at `GMT' or
+ `+X', or at the end of the string.
+
+ In extended regexp parlance, the function returns 1 if P matches
+ "^ *(GMT|[+-][0-9]|$)", 0 otherwise. P being NULL (which strptime
+ can return) is considered a failure and 0 is returned. */
+static bool
+check_end (const char *p)
+{
+ if (!p)
+ return false;
+ while (c_isspace (*p))
+ ++p;
+ if (!*p
+ || (p[0] == 'G' && p[1] == 'M' && p[2] == 'T')
+ || ((p[0] == '+' || p[0] == '-') && c_isdigit (p[1])))
+ return true;
+ else
+ return false;
+}
+
+/* Convert the textual specification of time in TIME_STRING to the
+ number of seconds since the Epoch.
+
+ TIME_STRING can be in any of the three formats RFC2616 allows the
+ HTTP servers to emit -- RFC1123-date, RFC850-date or asctime-date,
+ as well as the time format used in the Set-Cookie header.
+ Timezones are ignored, and should be GMT.
+
+ Return the computed time_t representation, or -1 if the conversion
+ fails.
+
+ This function uses strptime with various string formats for parsing
+ TIME_STRING. This results in a parser that is not as lenient in
+ interpreting TIME_STRING as I would like it to be. Being based on
+ strptime, it always allows shortened months, one-digit days, etc.,
+ but due to the multitude of formats in which time can be
+ represented, an ideal HTTP time parser would be even more
+ forgiving. It should completely ignore things like week days and
+ concentrate only on the various forms of representing years,
+ months, days, hours, minutes, and seconds. For example, it would
+ be nice if it accepted ISO 8601 out of the box.
+
+ I've investigated free and PD code for this purpose, but none was
+ usable. getdate was big and unwieldy, and had potential copyright
+ issues, or so I was informed. Dr. Marcus Hennecke's atotm(),
+ distributed with phttpd, is excellent, but we cannot use it because
+ it is not assigned to the FSF. So I stuck it with strptime. */
+
+time_t
+http_atotm (const char *time_string)
+{
+ /* NOTE: Solaris strptime man page claims that %n and %t match white
+ space, but that's not universally available. Instead, we simply
+ use ` ' to mean "skip all WS", which works under all strptime
+ implementations I've tested. */
+
+ static const char *time_formats[] = {
+ "%a, %d %b %Y %T", /* rfc1123: Thu, 29 Jan 1998 22:12:57 */
+ "%A, %d-%b-%y %T", /* rfc850: Thursday, 29-Jan-98 22:12:57 */
+ "%a %b %d %T %Y", /* asctime: Thu Jan 29 22:12:57 1998 */
+ "%a, %d-%b-%Y %T" /* cookies: Thu, 29-Jan-1998 22:12:57
+ (used in Set-Cookie, defined in the
+ Netscape cookie specification.) */
+ };
+ const char *oldlocale;
+ char savedlocale[256];
+ size_t i;
+ time_t ret = (time_t) -1;
+
+ /* Solaris strptime fails to recognize English month names in
+ non-English locales, which we work around by temporarily setting
+ locale to C before invoking strptime. */
+ oldlocale = setlocale (LC_TIME, NULL);
+ if (oldlocale)
+ {
+ size_t l = strlen (oldlocale) + 1;
+ if (l >= sizeof savedlocale)
+ savedlocale[0] = '\0';
+ else
+ memcpy (savedlocale, oldlocale, l);
+ }
+ else savedlocale[0] = '\0';
+
+ setlocale (LC_TIME, "C");
+
+ for (i = 0; i < countof (time_formats); i++)
+ {
+ struct tm t;
+
+ /* Some versions of strptime use the existing contents of struct
+ tm to recalculate the date according to format. Zero it out
+ to prevent stack garbage from influencing strptime. */
+ xzero (t);
+
+ if (check_end (strptime (time_string, time_formats[i], &t)))
+ {
+ ret = timegm (&t);
+ break;
+ }
+ }
+
+ /* Restore the previous locale. */
+ if (savedlocale[0])
+ setlocale (LC_TIME, savedlocale);
+
+ return ret;
+}
+
+/* Authorization support: We support three authorization schemes:
+
+ * `Basic' scheme, consisting of base64-ing USER:PASSWORD string;
+
+ * `Digest' scheme, added by Junio Hamano <junio@twinsun.com>,
+ consisting of answering to the server's challenge with the proper
+ MD5 digests.
+
+ * `NTLM' ("NT Lan Manager") scheme, based on code written by Daniel
+ Stenberg for libcurl. Like digest, NTLM is based on a
+ challenge-response mechanism, but unlike digest, it is non-standard
+ (authenticates TCP connections rather than requests), undocumented
+ and Microsoft-specific. */
+
+/* Create the authentication header contents for the `Basic' scheme.
+ This is done by encoding the string "USER:PASS" to base64 and
+ prepending the string "Basic " in front of it. */
+
+static char *
+basic_authentication_encode (const char *user, const char *passwd)
+{
+ char buf_t1[256], buf_t2[256];
+ char *t1, *t2, *ret;
+ size_t len1 = strlen (user) + 1 + strlen (passwd);
+
+ if (len1 < sizeof (buf_t1))
+ t1 = buf_t1;
+ else
+ t1 = xmalloc(len1 + 1);
+
+ if (BASE64_LENGTH (len1) < sizeof (buf_t2))
+ t2 = buf_t2;
+ else
+ t2 = xmalloc (BASE64_LENGTH (len1) + 1);
+
+ sprintf (t1, "%s:%s", user, passwd);
+ wget_base64_encode (t1, len1, t2);
+
+ ret = concat_strings ("Basic ", t2, (char *) 0);
+
+ if (t2 != buf_t2)
+ xfree (t2);
+
+ if (t1 != buf_t1)
+ xfree (t1);
+
+ return ret;
+}
+
+#define SKIP_WS(x) do { \
+ while (c_isspace (*(x))) \
+ ++(x); \
+} while (0)
+
+#ifdef ENABLE_DIGEST
+/* Dump the hexadecimal representation of HASH to BUF. HASH should be
+ an array of 16 bytes containing the hash keys, and BUF should be a
+ buffer of 33 writable characters (32 for hex digits plus one for
+ zero termination). */
+static void
+dump_hash (char *buf, const unsigned char *hash)
+{
+ int i;
+
+ for (i = 0; i < MD5_DIGEST_SIZE; i++, hash++)
+ {
+ *buf++ = XNUM_TO_digit (*hash >> 4);
+ *buf++ = XNUM_TO_digit (*hash & 0xf);
+ }
+ *buf = '\0';
+}
+
+/* Take the line apart to find the challenge, and compose a digest
+ authorization header. See RFC2069 section 2.1.2. */
+static char *
+digest_authentication_encode (const char *au, const char *user,
+ const char *passwd, const char *method,
+ const char *path, uerr_t *auth_err)
+{
+ static char *realm, *opaque, *nonce, *qop, *algorithm;
+ static struct {
+ const char *name;
+ char **variable;
+ } options[] = {
+ { "realm", &realm },
+ { "opaque", &opaque },
+ { "nonce", &nonce },
+ { "qop", &qop },
+ { "algorithm", &algorithm }
+ };
+ char cnonce[16] = "";
+ char *res = NULL;
+ int res_len;
+ size_t res_size;
+ param_token name, value;
+
+
+ realm = opaque = nonce = algorithm = qop = NULL;
+
+ au += 6; /* skip over `Digest' */
+ while (extract_param (&au, &name, &value, ',', NULL))
+ {
+ size_t i;
+ size_t namelen = name.e - name.b;
+ for (i = 0; i < countof (options); i++)
+ if (namelen == strlen (options[i].name)
+ && 0 == strncmp (name.b, options[i].name,
+ namelen))
+ {
+ *options[i].variable = strdupdelim (value.b, value.e);
+ break;
+ }
+ }
+
+ if (qop && strcmp (qop, "auth"))
+ {
+ logprintf (LOG_NOTQUIET, _("Unsupported quality of protection '%s'.\n"), qop);
+ xfree (qop); /* force freeing mem and continue */
+ }
+ else if (algorithm && strcmp (algorithm,"MD5") && strcmp (algorithm,"MD5-sess"))
+ {
+ logprintf (LOG_NOTQUIET, _("Unsupported algorithm '%s'.\n"), algorithm);
+ xfree (algorithm); /* force freeing mem and continue */
+ }
+
+ if (!realm || !nonce || !user || !passwd || !path || !method)
+ {
+ *auth_err = ATTRMISSING;
+ goto cleanup;
+ }
+
+ /* Calculate the digest value. */
+ {
+ struct md5_ctx ctx;
+ unsigned char hash[MD5_DIGEST_SIZE];
+ char a1buf[MD5_DIGEST_SIZE * 2 + 1], a2buf[MD5_DIGEST_SIZE * 2 + 1];
+ char response_digest[MD5_DIGEST_SIZE * 2 + 1];
+
+ /* A1BUF = H(user ":" realm ":" password) */
+ md5_init_ctx (&ctx);
+ md5_process_bytes ((unsigned char *)user, strlen (user), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)realm, strlen (realm), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)passwd, strlen (passwd), &ctx);
+ md5_finish_ctx (&ctx, hash);
+
+ dump_hash (a1buf, hash);
+
+ if (algorithm && !strcmp (algorithm, "MD5-sess"))
+ {
+ /* A1BUF = H( H(user ":" realm ":" password) ":" nonce ":" cnonce ) */
+ snprintf (cnonce, sizeof (cnonce), "%08x",
+ (unsigned) random_number (INT_MAX));
+
+ md5_init_ctx (&ctx);
+ /* md5_process_bytes (hash, MD5_DIGEST_SIZE, &ctx); */
+ md5_process_bytes (a1buf, MD5_DIGEST_SIZE * 2, &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)cnonce, strlen (cnonce), &ctx);
+ md5_finish_ctx (&ctx, hash);
+
+ dump_hash (a1buf, hash);
+ }
+
+ /* A2BUF = H(method ":" path) */
+ md5_init_ctx (&ctx);
+ md5_process_bytes ((unsigned char *)method, strlen (method), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)path, strlen (path), &ctx);
+ md5_finish_ctx (&ctx, hash);
+ dump_hash (a2buf, hash);
+
+ if (qop && !strcmp (qop, "auth"))
+ {
+ /* RFC 2617 Digest Access Authentication */
+ /* generate random hex string */
+ if (!*cnonce)
+ snprintf (cnonce, sizeof (cnonce), "%08x",
+ (unsigned) random_number (INT_MAX));
+
+ /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" noncecount ":" clientnonce ":" qop ": " A2BUF) */
+ md5_init_ctx (&ctx);
+ md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)"00000001", 8, &ctx); /* TODO: keep track of server nonce values */
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)cnonce, strlen (cnonce), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)qop, strlen (qop), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx);
+ md5_finish_ctx (&ctx, hash);
+ }
+ else
+ {
+ /* RFC 2069 Digest Access Authentication */
+ /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" A2BUF) */
+ md5_init_ctx (&ctx);
+ md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx);
+ md5_process_bytes ((unsigned char *)":", 1, &ctx);
+ md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx);
+ md5_finish_ctx (&ctx, hash);
+ }
+
+ dump_hash (response_digest, hash);
+
+ res_size = strlen (user)
+ + strlen (realm)
+ + strlen (nonce)
+ + strlen (path)
+ + 2 * MD5_DIGEST_SIZE /*strlen (response_digest)*/
+ + (opaque ? strlen (opaque) : 0)
+ + (algorithm ? strlen (algorithm) : 0)
+ + (qop ? 128: 0)
+ + strlen (cnonce)
+ + 128;
+
+ res = xmalloc (res_size);
+
+ if (qop && !strcmp (qop, "auth"))
+ {
+ res_len = snprintf (res, res_size, "Digest "\
+ "username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\""\
+ ", qop=auth, nc=00000001, cnonce=\"%s\"",
+ user, realm, nonce, path, response_digest, cnonce);
+
+ }
+ else
+ {
+ res_len = snprintf (res, res_size, "Digest "\
+ "username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\"",
+ user, realm, nonce, path, response_digest);
+ }
+
+ if (opaque)
+ {
+ res_len += snprintf (res + res_len, res_size - res_len, ", opaque=\"%s\"", opaque);
+ }
+
+ if (algorithm)
+ {
+ snprintf (res + res_len, res_size - res_len, ", algorithm=\"%s\"", algorithm);
+ }
+ }
+
+cleanup:
+ xfree (realm);
+ xfree (opaque);
+ xfree (nonce);
+ xfree (qop);
+ xfree (algorithm);
+
+ return res;
+}
+#endif /* ENABLE_DIGEST */
+
+/* Computing the size of a string literal must take into account that
+ value returned by sizeof includes the terminating \0. */
+#define STRSIZE(literal) (sizeof (literal) - 1)
+
+/* Whether chars in [b, e) begin with the literal string provided as
+ first argument and are followed by whitespace or terminating \0.
+ The comparison is case-insensitive. */
+#define STARTS(literal, b, e) \
+ ((e > b) \
+ && ((size_t) ((e) - (b))) >= STRSIZE (literal) \
+ && 0 == c_strncasecmp (b, literal, STRSIZE (literal)) \
+ && ((size_t) ((e) - (b)) == STRSIZE (literal) \
+ || c_isspace (b[STRSIZE (literal)])))
+
+static bool
+known_authentication_scheme_p (const char *hdrbeg, const char *hdrend)
+{
+ return STARTS ("Basic", hdrbeg, hdrend)
+#ifdef ENABLE_DIGEST
+ || STARTS ("Digest", hdrbeg, hdrend)
+#endif
+#ifdef ENABLE_NTLM
+ || STARTS ("NTLM", hdrbeg, hdrend)
+#endif
+ ;
+}
+
+#undef STARTS
+
+/* Create the HTTP authorization request header. When the
+ `WWW-Authenticate' response header is seen, according to the
+ authorization scheme specified in that header (`Basic' and `Digest'
+ are supported by the current implementation), produce an
+ appropriate HTTP authorization request header. */
+static char *
+create_authorization_line (const char *au, const char *user,
+ const char *passwd, const char *method,
+ const char *path, bool *finished, uerr_t *auth_err)
+{
+ /* We are called only with known schemes, so we can dispatch on the
+ first letter. */
+ switch (c_toupper (*au))
+ {
+ case 'B': /* Basic */
+ *finished = true;
+ return basic_authentication_encode (user, passwd);
+#ifdef ENABLE_DIGEST
+ case 'D': /* Digest */
+ *finished = true;
+ return digest_authentication_encode (au, user, passwd, method, path, auth_err);
+#endif
+#ifdef ENABLE_NTLM
+ case 'N': /* NTLM */
+ if (!ntlm_input (&pconn.ntlm, au))
+ {
+ *finished = true;
+ return NULL;
+ }
+ return ntlm_output (&pconn.ntlm, user, passwd, finished);
+#endif
+ default:
+ /* We shouldn't get here -- this function should be only called
+ with values approved by known_authentication_scheme_p. */
+ abort ();
+ }
+}
+
+static void
+load_cookies (void)
+{
+ if (!wget_cookie_jar)
+ wget_cookie_jar = cookie_jar_new ();
+ if (opt.cookies_input && !cookies_loaded_p)
+ {
+ cookie_jar_load (wget_cookie_jar, opt.cookies_input);
+ cookies_loaded_p = true;
+ }
+}
+
+void
+save_cookies (void)
+{
+ if (wget_cookie_jar)
+ cookie_jar_save (wget_cookie_jar, opt.cookies_output);
+}
+
+#if defined DEBUG_MALLOC || defined TESTING
+void
+http_cleanup (void)
+{
+ if (pconn_active)
+ invalidate_persistent ();
+
+ if (wget_cookie_jar)
+ {
+ cookie_jar_delete (wget_cookie_jar);
+ wget_cookie_jar = NULL;
+ }
+
+ if (basic_authed_hosts)
+ {
+ hash_table_iterator iter;
+ for (hash_table_iterate (basic_authed_hosts, &iter); hash_table_iter_next (&iter); )
+ {
+ xfree (iter.key);
+ }
+ hash_table_destroy (basic_authed_hosts);
+ basic_authed_hosts = NULL;
+ }
+}
+#endif
+
+void
+ensure_extension (struct http_stat *hs, const char *ext, int *dt)
+{
+ char *last_period_in_local_filename = strrchr (hs->local_file, '.');
+ char shortext[8];
+ int len;
+ shortext[0] = '\0';
+ len = strlen (ext);
+ if (len == 5)
+ {
+ memcpy (shortext, ext, len - 1);
+ shortext[len - 1] = '\0';
+ }
+
+ if (last_period_in_local_filename == NULL
+ || !(0 == strcasecmp (last_period_in_local_filename, shortext)
+ || 0 == strcasecmp (last_period_in_local_filename, ext)))
+ {
+ int local_filename_len = strlen (hs->local_file);
+ /* Resize the local file, allowing for ".html" preceded by
+ optional ".NUMBER". */
+ hs->local_file = xrealloc (hs->local_file,
+ local_filename_len + 24 + len);
+ strcpy (hs->local_file + local_filename_len, ext);
+ /* If clobbering is not allowed and the file, as named,
+ exists, tack on ".NUMBER.html" instead. */
+ if (!ALLOW_CLOBBER && file_exists_p (hs->local_file, NULL))
+ {
+ int ext_num = 1;
+ do
+ sprintf (hs->local_file + local_filename_len,
+ ".%d%s", ext_num++, ext);
+ while (file_exists_p (hs->local_file, NULL));
+ }
+ *dt |= ADDED_HTML_EXTENSION;
+ }
+}
+
+#ifdef TESTING
+
+const char *
+test_parse_range_header (void)
+{
+ unsigned i;
+ static const struct {
+ const char * rangehdr;
+ const wgint firstbyte;
+ const wgint lastbyte;
+ const wgint length;
+ const bool shouldPass;
+ } test_array[] = {
+ { "bytes 0-1000/1000", 0, 1000, 1000, false },
+ { "bytes 0-999/1000", 0, 999, 1000, true },
+ { "bytes 100-99/1000", 100, 99, 1000, false },
+ { "bytes 100-100/1000", 100, 100, 1000, true },
+ { "bytes 0-1000/100000000", 0, 1000, 100000000, true },
+ { "bytes 1-999/1000", 1, 999, 1000, true },
+ { "bytes 42-1233/1234", 42, 1233, 1234, true },
+ { "bytes 42-1233/*", 42, 1233, -1, true },
+ { "bytes 0-2147483648/2147483649", 0, 2147483648U, 2147483649U, true },
+ { "bytes 2147483648-4294967296/4294967297", 2147483648U, 4294967296ULL, 4294967297ULL, true },
+ };
+
+ wgint firstbyteptr[sizeof(wgint)];
+ wgint lastbyteptr[sizeof(wgint)];
+ wgint lengthptr[sizeof(wgint)];
+ bool result;
+ for (i = 0; i < countof (test_array); i++)
+ {
+ result = parse_content_range (test_array[i].rangehdr, firstbyteptr, lastbyteptr, lengthptr);
+#if 0
+ printf ("%ld %ld\n", test_array[i].firstbyte, *firstbyteptr);
+ printf ("%ld %ld\n", test_array[i].lastbyte, *lastbyteptr);
+ printf ("%ld %ld\n", test_array[i].length, *lengthptr);
+ printf ("\n");
+#endif
+ mu_assert ("test_parse_range_header: False Negative", result == test_array[i].shouldPass);
+ mu_assert ("test_parse_range_header: Bad parse", test_array[i].firstbyte == *firstbyteptr &&
+ test_array[i].lastbyte == *lastbyteptr &&
+ test_array[i].length == *lengthptr);
+ }
+
+ return NULL;
+}
+
+const char *
+test_parse_content_disposition (void)
+{
+ unsigned i;
+ static const struct {
+ const char *hdrval;
+ const char *filename;
+ bool result;
+ } test_array[] = {
+ { "filename=\"file.ext\"", "file.ext", true },
+ { "attachment; filename=\"file.ext\"", "file.ext", true },
+ { "attachment; filename=\"file.ext\"; dummy", "file.ext", true },
+ { "attachment", NULL, false },
+ { "attachment; filename*=UTF-8'en-US'hello.txt", "hello.txt", true },
+ { "attachment; filename*0=\"hello\"; filename*1=\"world.txt\"",
+ "helloworld.txt", true },
+ { "attachment; filename=\"A.ext\"; filename*=\"B.ext\"", "B.ext", true },
+ { "attachment; filename*=\"A.ext\"; filename*0=\"B\"; filename*1=\"B.ext\"",
+ "A.ext", true },
+ { "filename**0=\"A\"; filename**1=\"A.ext\"; filename*0=\"B\";\
+filename*1=\"B\"", "AA.ext", true },
+ };
+
+ for (i = 0; i < countof (test_array); ++i)
+ {
+ char *filename;
+ bool res;
+
+ res = parse_content_disposition (test_array[i].hdrval, &filename);
+
+ mu_assert ("test_parse_content_disposition: wrong result",
+ res == test_array[i].result
+ && (res == false
+ || 0 == strcmp (test_array[i].filename, filename)));
+ xfree (filename);
+ }
+
+ return NULL;
+}
+
+#endif /* TESTING */
+
+/*
+ * vim: et sts=2 sw=2 cino+={s
+ */