diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/Makefile.am | 110 | ||||
| -rw-r--r-- | src/Makefile.in | 2804 | ||||
| -rw-r--r-- | src/build_info.c | 101 | ||||
| -rw-r--r-- | src/build_info.c.in | 18 | ||||
| -rw-r--r-- | src/config.h.in | 2685 | ||||
| -rw-r--r-- | src/connect.c | 1084 | ||||
| -rw-r--r-- | src/connect.h | 89 | ||||
| -rw-r--r-- | src/convert.c | 1230 | ||||
| -rw-r--r-- | src/convert.h | 115 | ||||
| -rw-r--r-- | src/cookies.c | 1539 | ||||
| -rw-r--r-- | src/cookies.h | 47 | ||||
| -rw-r--r-- | src/css-tokens.h | 65 | ||||
| -rw-r--r-- | src/css-url.c | 236 | ||||
| -rw-r--r-- | src/css-url.h | 37 | ||||
| -rw-r--r-- | src/css.c | 3932 | ||||
| -rw-r--r-- | src/css.l | 167 | ||||
| -rw-r--r-- | src/css_.c | 3933 | ||||
| -rw-r--r-- | src/exits.c | 93 | ||||
| -rw-r--r-- | src/exits.h | 48 | ||||
| -rw-r--r-- | src/ftp-basic.c | 1342 | ||||
| -rw-r--r-- | src/ftp-ls.c | 1175 | ||||
| -rw-r--r-- | src/ftp-opie.c | 2220 | ||||
| -rw-r--r-- | src/ftp.c | 2898 | ||||
| -rw-r--r-- | src/ftp.h | 184 | ||||
| -rw-r--r-- | src/gnutls.c | 1116 | ||||
| -rw-r--r-- | src/hash.c | 813 | ||||
| -rw-r--r-- | src/hash.h | 66 | ||||
| -rw-r--r-- | src/host.c | 1082 | ||||
| -rw-r--r-- | src/host.h | 107 | ||||
| -rw-r--r-- | src/hsts.c | 829 | ||||
| -rw-r--r-- | src/hsts.h | 53 | ||||
| -rw-r--r-- | src/html-parse.c | 1221 | ||||
| -rw-r--r-- | src/html-parse.h | 71 | ||||
| -rw-r--r-- | src/html-url.c | 977 | ||||
| -rw-r--r-- | src/html-url.h | 58 | ||||
| -rw-r--r-- | src/http-ntlm.c | 618 | ||||
| -rw-r--r-- | src/http-ntlm.h | 53 | ||||
| -rw-r--r-- | src/http.c | 5495 | ||||
| -rw-r--r-- | src/http.h | 51 | ||||
| -rw-r--r-- | src/init.c | 2135 | ||||
| -rw-r--r-- | src/init.h | 48 | ||||
| -rw-r--r-- | src/iri.c | 449 | ||||
| -rw-r--r-- | src/iri.h | 75 | ||||
| -rw-r--r-- | src/log.c | 995 | ||||
| -rw-r--r-- | src/log.h | 60 | ||||
| -rw-r--r-- | src/main.c | 2306 | ||||
| -rw-r--r-- | src/metalink.c | 1599 | ||||
| -rw-r--r-- | src/metalink.h | 75 | ||||
| -rw-r--r-- | src/mswindows.c | 652 | ||||
| -rw-r--r-- | src/mswindows.h | 78 | ||||
| -rw-r--r-- | src/netrc.c | 569 | ||||
| -rw-r--r-- | src/netrc.h | 39 | ||||
| -rw-r--r-- | src/openssl.c | 1260 | ||||
| -rw-r--r-- | src/options.h | 351 | ||||
| -rw-r--r-- | src/progress.c | 1462 | ||||
| -rw-r--r-- | src/progress.h | 45 | ||||
| -rw-r--r-- | src/ptimer.c | 413 | ||||
| -rw-r--r-- | src/ptimer.h | 46 | ||||
| -rw-r--r-- | src/recur.c | 919 | ||||
| -rw-r--r-- | src/recur.h | 49 | ||||
| -rw-r--r-- | src/res.c | 650 | ||||
| -rw-r--r-- | src/res.h | 50 | ||||
| -rw-r--r-- | src/retr.c | 1561 | ||||
| -rw-r--r-- | src/retr.h | 81 | ||||
| -rw-r--r-- | src/spider.c | 101 | ||||
| -rw-r--r-- | src/spider.h | 39 | ||||
| -rw-r--r-- | src/ssl.h | 40 | ||||
| -rw-r--r-- | src/sysdep.h | 59 | ||||
| -rw-r--r-- | src/url.c | 2540 | ||||
| -rw-r--r-- | src/url.h | 136 | ||||
| -rw-r--r-- | src/utils.c | 2993 | ||||
| -rw-r--r-- | src/utils.h | 179 | ||||
| -rw-r--r-- | src/version.h | 41 | ||||
| -rw-r--r-- | src/warc.c | 1662 | ||||
| -rw-r--r-- | src/warc.h | 27 | ||||
| -rw-r--r-- | src/wget.h | 333 | ||||
| -rw-r--r-- | src/xattr.c | 95 | ||||
| -rw-r--r-- | src/xattr.h | 46 |
78 files changed, 63020 insertions, 0 deletions
diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 0000000..322a3db --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,110 @@ +# Makefile for `wget' utility +# Copyright (C) 1995-2011, 2015, 2018-2020 Free Software Foundation, +# Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with Wget. If not, see <http://www.gnu.org/licenses/>. + +# Additional permission under GNU GPL version 3 section 7 + +# If you modify this program, or any covered work, by linking or +# combining it with the OpenSSL project's OpenSSL library (or a +# modified version of that library), containing parts covered by the +# terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +# grants you additional permission to convey the resulting work. +# Corresponding Source for a non-source form of such a combination +# shall include the source code for the parts of OpenSSL used as well +# as that of the covered work. + +# +# Version: @VERSION@ +# + +if IRI_IS_ENABLED +IRI_OBJ = iri.c +endif + +if METALINK_IS_ENABLED +METALINK_OBJ = metalink.c +endif + +if WITH_XATTR +XATTR_OBJ = xattr.c +endif + +# The following line is losing on some versions of make! +DEFS = @DEFS@ -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\" + +EXTRA_DIST = css.l css.c css_.c build_info.c.in build_info.c + +bin_PROGRAMS = wget +wget_SOURCES = connect.c convert.c cookies.c ftp.c \ + css_.c css-url.c \ + ftp-basic.c ftp-ls.c hash.c host.c hsts.c html-parse.c html-url.c \ + http.c init.c log.c main.c netrc.c progress.c ptimer.c \ + recur.c res.c retr.c spider.c url.c warc.c $(XATTR_OBJ) \ + utils.c exits.c build_info.c $(IRI_OBJ) $(METALINK_OBJ) \ + css-url.h css-tokens.h connect.h convert.h cookies.h \ + ftp.h hash.h host.h hsts.h html-parse.h html-url.h \ + http.h http-ntlm.h init.h log.h mswindows.h netrc.h \ + options.h progress.h ptimer.h recur.h res.h retr.h \ + spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h \ + exits.h version.h metalink.h xattr.h +nodist_wget_SOURCES = version.c +EXTRA_wget_SOURCES = iri.c +LDADD = $(CODE_COVERAGE_LIBS) $(LIBOBJS) ../lib/libgnu.a $(GETADDRINFO_LIB) $(HOSTENT_LIB)\ + $(INET_NTOP_LIB) $(LIBSOCKET) $(LIB_CLOCK_GETTIME) $(LIB_CRYPTO)\ + $(LIB_NANOSLEEP) $(LIB_POSIX_SPAWN) $(LIB_SELECT) $(LIBICONV) $(LIBINTL)\ + $(LIBTHREAD) $(LIBUNISTRING) $(SERVENT_LIB) +AM_CPPFLAGS = -I$(top_builddir)/lib -I$(top_srcdir)/lib $(CODE_COVERAGE_CPPFLAGS) +AM_CFLAGS = $(WERROR_CFLAGS) $(WARN_CFLAGS) $(CODE_COVERAGE_CFLAGS) + +../lib/libgnu.a: + cd ../lib && $(MAKE) $(AM_MAKEFLAGS) + +build_info.c: $(srcdir)/Makefile.am $(srcdir)/build_info.c.in + if test -n "$(VPATH)"; then cp "$(srcdir)/build_info.c.in" .; fi + $(PERL) "$(top_srcdir)/build-aux/build_info.pl" \ + "$(top_builddir)/src/build_info.c" + if test -n "$(VPATH)"; then rm -f build_info.c.in; fi + +ESCAPEQUOTE = sed -e 's/[\\"]/\\&/g' -e 's/\\"/"/' -e 's/\\";$$/";/' +version.c: $(wget_SOURCES) ../lib/libgnu.a + echo '/* version.c */' > $@ + echo '/* Autogenerated by Makefile - DO NOT EDIT */' >> $@ + echo '' >> $@ + echo '#include "version.h"' >> $@ + echo 'const char *version_string = "@VERSION@";' >> $@ + echo 'const char *compilation_string = "'$(COMPILE)'";' \ + | $(ESCAPEQUOTE) >> $@ + echo 'const char *link_string = "'$(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) $(LIBS) $(wget_LDADD)'";' \ + | $(ESCAPEQUOTE) >> $@ + +css.c: $(srcdir)/css.l + $(LEX) $(LFLAGS) -o$@ $^ + +css_.c: css.c + echo '#include "wget.h"' > $@ + cat css.c >> $@ + +distclean-local: + rm -f css.c css_.c + +check_LIBRARIES = libunittest.a +libunittest_a_SOURCES = $(wget_SOURCES) build_info.c +nodist_libunittest_a_SOURCES = version.c +libunittest_a_CPPFLAGS = -DTESTING "-I$(top_builddir)/lib" "-I$(top_srcdir)/lib" $(CODE_COVERAGE_CPPLAGS) +libunittest_a_LIBADD = $(LIBOBJS) + +CLEANFILES = *~ *.bak core core.[0-9]* build_info.c version.c diff --git a/src/Makefile.in b/src/Makefile.in new file mode 100644 index 0000000..3e7c745 --- /dev/null +++ b/src/Makefile.in @@ -0,0 +1,2804 @@ +# Makefile.in generated by automake 1.16.2 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2020 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + +# Makefile for `wget' utility +# Copyright (C) 1995-2011, 2015, 2018-2020 Free Software Foundation, +# Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with Wget. If not, see <http://www.gnu.org/licenses/>. + +# Additional permission under GNU GPL version 3 section 7 + +# If you modify this program, or any covered work, by linking or +# combining it with the OpenSSL project's OpenSSL library (or a +# modified version of that library), containing parts covered by the +# terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +# grants you additional permission to convey the resulting work. +# Corresponding Source for a non-source form of such a combination +# shall include the source code for the parts of OpenSSL used as well +# as that of the covered work. + +# +# Version: @VERSION@ +# + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +bin_PROGRAMS = wget$(EXEEXT) +subdir = src +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/00gnulib.m4 \ + $(top_srcdir)/m4/__inline.m4 \ + $(top_srcdir)/m4/absolute-header.m4 $(top_srcdir)/m4/access.m4 \ + $(top_srcdir)/m4/af_alg.m4 $(top_srcdir)/m4/alloca.m4 \ + $(top_srcdir)/m4/arpa_inet_h.m4 \ + $(top_srcdir)/m4/asm-underscore.m4 $(top_srcdir)/m4/base32.m4 \ + $(top_srcdir)/m4/btowc.m4 $(top_srcdir)/m4/builtin-expect.m4 \ + $(top_srcdir)/m4/byteswap.m4 $(top_srcdir)/m4/canonicalize.m4 \ + $(top_srcdir)/m4/chdir-long.m4 $(top_srcdir)/m4/clock_time.m4 \ + $(top_srcdir)/m4/close.m4 $(top_srcdir)/m4/closedir.m4 \ + $(top_srcdir)/m4/codeset.m4 $(top_srcdir)/m4/ctype.m4 \ + $(top_srcdir)/m4/d-ino.m4 $(top_srcdir)/m4/dirent_h.m4 \ + $(top_srcdir)/m4/dirfd.m4 \ + $(top_srcdir)/m4/double-slash-root.m4 $(top_srcdir)/m4/dup.m4 \ + $(top_srcdir)/m4/dup2.m4 $(top_srcdir)/m4/eaccess.m4 \ + $(top_srcdir)/m4/eealloc.m4 $(top_srcdir)/m4/environ.m4 \ + $(top_srcdir)/m4/errno_h.m4 $(top_srcdir)/m4/error.m4 \ + $(top_srcdir)/m4/exponentd.m4 $(top_srcdir)/m4/extensions.m4 \ + $(top_srcdir)/m4/extern-inline.m4 \ + $(top_srcdir)/m4/fatal-signal.m4 $(top_srcdir)/m4/fchdir.m4 \ + $(top_srcdir)/m4/fcntl-o.m4 $(top_srcdir)/m4/fcntl.m4 \ + $(top_srcdir)/m4/fcntl_h.m4 $(top_srcdir)/m4/fdopendir.m4 \ + $(top_srcdir)/m4/fflush.m4 $(top_srcdir)/m4/filenamecat.m4 \ + $(top_srcdir)/m4/findprog-in.m4 $(top_srcdir)/m4/flexmember.m4 \ + $(top_srcdir)/m4/float_h.m4 $(top_srcdir)/m4/flock.m4 \ + $(top_srcdir)/m4/fnmatch.m4 $(top_srcdir)/m4/fnmatch_h.m4 \ + $(top_srcdir)/m4/fopen.m4 $(top_srcdir)/m4/fpurge.m4 \ + $(top_srcdir)/m4/freading.m4 $(top_srcdir)/m4/free.m4 \ + $(top_srcdir)/m4/fseek.m4 $(top_srcdir)/m4/fseeko.m4 \ + $(top_srcdir)/m4/fstat.m4 $(top_srcdir)/m4/fstatat.m4 \ + $(top_srcdir)/m4/ftell.m4 $(top_srcdir)/m4/ftello.m4 \ + $(top_srcdir)/m4/futimens.m4 $(top_srcdir)/m4/getaddrinfo.m4 \ + $(top_srcdir)/m4/getcwd-abort-bug.m4 \ + $(top_srcdir)/m4/getcwd-path-max.m4 $(top_srcdir)/m4/getcwd.m4 \ + $(top_srcdir)/m4/getdelim.m4 $(top_srcdir)/m4/getdtablesize.m4 \ + $(top_srcdir)/m4/getgroups.m4 $(top_srcdir)/m4/getline.m4 \ + $(top_srcdir)/m4/getopt.m4 $(top_srcdir)/m4/getpagesize.m4 \ + $(top_srcdir)/m4/getpass.m4 $(top_srcdir)/m4/getprogname.m4 \ + $(top_srcdir)/m4/getrandom.m4 $(top_srcdir)/m4/gettext.m4 \ + $(top_srcdir)/m4/gettime.m4 $(top_srcdir)/m4/gettimeofday.m4 \ + $(top_srcdir)/m4/gl-openssl.m4 \ + $(top_srcdir)/m4/gnulib-common.m4 \ + $(top_srcdir)/m4/gnulib-comp.m4 \ + $(top_srcdir)/m4/group-member.m4 \ + $(top_srcdir)/m4/host-cpu-c-abi.m4 $(top_srcdir)/m4/hostent.m4 \ + $(top_srcdir)/m4/iconv.m4 $(top_srcdir)/m4/iconv_h.m4 \ + $(top_srcdir)/m4/include_next.m4 $(top_srcdir)/m4/inet_ntop.m4 \ + $(top_srcdir)/m4/inline.m4 $(top_srcdir)/m4/intlmacosx.m4 \ + $(top_srcdir)/m4/intmax_t.m4 $(top_srcdir)/m4/inttypes.m4 \ + $(top_srcdir)/m4/inttypes_h.m4 $(top_srcdir)/m4/ioctl.m4 \ + $(top_srcdir)/m4/isblank.m4 $(top_srcdir)/m4/iswblank.m4 \ + $(top_srcdir)/m4/iswdigit.m4 $(top_srcdir)/m4/iswxdigit.m4 \ + $(top_srcdir)/m4/langinfo_h.m4 $(top_srcdir)/m4/largefile.m4 \ + $(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \ + $(top_srcdir)/m4/lib-prefix.m4 \ + $(top_srcdir)/m4/libunistring-base.m4 \ + $(top_srcdir)/m4/libunistring-optional.m4 \ + $(top_srcdir)/m4/libunistring.m4 $(top_srcdir)/m4/limits-h.m4 \ + $(top_srcdir)/m4/link.m4 $(top_srcdir)/m4/localcharset.m4 \ + $(top_srcdir)/m4/locale-fr.m4 $(top_srcdir)/m4/locale-ja.m4 \ + $(top_srcdir)/m4/locale-zh.m4 $(top_srcdir)/m4/locale_h.m4 \ + $(top_srcdir)/m4/localeconv.m4 $(top_srcdir)/m4/lock.m4 \ + $(top_srcdir)/m4/lseek.m4 $(top_srcdir)/m4/lstat.m4 \ + $(top_srcdir)/m4/malloc.m4 $(top_srcdir)/m4/malloca.m4 \ + $(top_srcdir)/m4/mbchar.m4 $(top_srcdir)/m4/mbiter.m4 \ + $(top_srcdir)/m4/mbrtowc.m4 $(top_srcdir)/m4/mbsinit.m4 \ + $(top_srcdir)/m4/mbsrtowcs.m4 $(top_srcdir)/m4/mbstate_t.m4 \ + $(top_srcdir)/m4/mbtowc.m4 $(top_srcdir)/m4/md4.m4 \ + $(top_srcdir)/m4/md5.m4 $(top_srcdir)/m4/memchr.m4 \ + $(top_srcdir)/m4/mempcpy.m4 $(top_srcdir)/m4/memrchr.m4 \ + $(top_srcdir)/m4/minmax.m4 $(top_srcdir)/m4/mkdir.m4 \ + $(top_srcdir)/m4/mkostemp.m4 $(top_srcdir)/m4/mkstemp.m4 \ + $(top_srcdir)/m4/mktime.m4 $(top_srcdir)/m4/mmap-anon.m4 \ + $(top_srcdir)/m4/mode_t.m4 $(top_srcdir)/m4/msvc-inval.m4 \ + $(top_srcdir)/m4/msvc-nothrow.m4 $(top_srcdir)/m4/multiarch.m4 \ + $(top_srcdir)/m4/nanosleep.m4 $(top_srcdir)/m4/netdb_h.m4 \ + $(top_srcdir)/m4/netinet_in_h.m4 \ + $(top_srcdir)/m4/nl_langinfo.m4 $(top_srcdir)/m4/nls.m4 \ + $(top_srcdir)/m4/nocrash.m4 $(top_srcdir)/m4/off_t.m4 \ + $(top_srcdir)/m4/open-cloexec.m4 \ + $(top_srcdir)/m4/open-slash.m4 $(top_srcdir)/m4/open.m4 \ + $(top_srcdir)/m4/openat.m4 $(top_srcdir)/m4/opendir.m4 \ + $(top_srcdir)/m4/pathmax.m4 $(top_srcdir)/m4/pipe.m4 \ + $(top_srcdir)/m4/pipe2.m4 $(top_srcdir)/m4/po.m4 \ + $(top_srcdir)/m4/posix_spawn.m4 \ + $(top_srcdir)/m4/posix_spawn_faction_addchdir.m4 \ + $(top_srcdir)/m4/printf.m4 \ + $(top_srcdir)/m4/pthread_rwlock_rdlock.m4 \ + $(top_srcdir)/m4/quote.m4 $(top_srcdir)/m4/quotearg.m4 \ + $(top_srcdir)/m4/raise.m4 $(top_srcdir)/m4/rawmemchr.m4 \ + $(top_srcdir)/m4/readdir.m4 $(top_srcdir)/m4/readlink.m4 \ + $(top_srcdir)/m4/realloc.m4 $(top_srcdir)/m4/regex.m4 \ + $(top_srcdir)/m4/rewinddir.m4 $(top_srcdir)/m4/save-cwd.m4 \ + $(top_srcdir)/m4/sched_h.m4 $(top_srcdir)/m4/secure_getenv.m4 \ + $(top_srcdir)/m4/select.m4 $(top_srcdir)/m4/servent.m4 \ + $(top_srcdir)/m4/setlocale_null.m4 \ + $(top_srcdir)/m4/sh-filename.m4 $(top_srcdir)/m4/sha1.m4 \ + $(top_srcdir)/m4/sha256.m4 $(top_srcdir)/m4/sha512.m4 \ + $(top_srcdir)/m4/sig_atomic_t.m4 $(top_srcdir)/m4/sigaction.m4 \ + $(top_srcdir)/m4/signal_h.m4 \ + $(top_srcdir)/m4/signalblocking.m4 $(top_srcdir)/m4/sigpipe.m4 \ + $(top_srcdir)/m4/size_max.m4 $(top_srcdir)/m4/snprintf.m4 \ + $(top_srcdir)/m4/socketlib.m4 $(top_srcdir)/m4/sockets.m4 \ + $(top_srcdir)/m4/socklen.m4 $(top_srcdir)/m4/sockpfaf.m4 \ + $(top_srcdir)/m4/spawn-pipe.m4 $(top_srcdir)/m4/spawn_h.m4 \ + $(top_srcdir)/m4/ssize_t.m4 $(top_srcdir)/m4/stat-time.m4 \ + $(top_srcdir)/m4/stat.m4 $(top_srcdir)/m4/stdalign.m4 \ + $(top_srcdir)/m4/stdbool.m4 $(top_srcdir)/m4/stddef_h.m4 \ + $(top_srcdir)/m4/stdint.m4 $(top_srcdir)/m4/stdint_h.m4 \ + $(top_srcdir)/m4/stdio_h.m4 $(top_srcdir)/m4/stdlib_h.m4 \ + $(top_srcdir)/m4/stpcpy.m4 $(top_srcdir)/m4/strcase.m4 \ + $(top_srcdir)/m4/strchrnul.m4 $(top_srcdir)/m4/strdup.m4 \ + $(top_srcdir)/m4/strerror.m4 $(top_srcdir)/m4/strerror_r.m4 \ + $(top_srcdir)/m4/string_h.m4 $(top_srcdir)/m4/strings_h.m4 \ + $(top_srcdir)/m4/strndup.m4 $(top_srcdir)/m4/strnlen.m4 \ + $(top_srcdir)/m4/strpbrk.m4 $(top_srcdir)/m4/strptime.m4 \ + $(top_srcdir)/m4/strtok_r.m4 $(top_srcdir)/m4/strtol.m4 \ + $(top_srcdir)/m4/strtoll.m4 $(top_srcdir)/m4/symlink.m4 \ + $(top_srcdir)/m4/sys_file_h.m4 $(top_srcdir)/m4/sys_ioctl_h.m4 \ + $(top_srcdir)/m4/sys_random_h.m4 \ + $(top_srcdir)/m4/sys_select_h.m4 \ + $(top_srcdir)/m4/sys_socket_h.m4 \ + $(top_srcdir)/m4/sys_stat_h.m4 $(top_srcdir)/m4/sys_time_h.m4 \ + $(top_srcdir)/m4/sys_types_h.m4 $(top_srcdir)/m4/sys_uio_h.m4 \ + $(top_srcdir)/m4/sys_wait_h.m4 $(top_srcdir)/m4/tempname.m4 \ + $(top_srcdir)/m4/threadlib.m4 $(top_srcdir)/m4/time_h.m4 \ + $(top_srcdir)/m4/time_r.m4 $(top_srcdir)/m4/timegm.m4 \ + $(top_srcdir)/m4/timespec.m4 $(top_srcdir)/m4/tm_gmtoff.m4 \ + $(top_srcdir)/m4/tmpdir.m4 $(top_srcdir)/m4/unistd-safer.m4 \ + $(top_srcdir)/m4/unistd_h.m4 $(top_srcdir)/m4/unlink.m4 \ + $(top_srcdir)/m4/unlocked-io.m4 $(top_srcdir)/m4/utime.m4 \ + $(top_srcdir)/m4/utime_h.m4 $(top_srcdir)/m4/utimens.m4 \ + $(top_srcdir)/m4/utimes.m4 $(top_srcdir)/m4/vasnprintf.m4 \ + $(top_srcdir)/m4/vasprintf.m4 $(top_srcdir)/m4/visibility.m4 \ + $(top_srcdir)/m4/vsnprintf.m4 $(top_srcdir)/m4/wait-process.m4 \ + $(top_srcdir)/m4/waitpid.m4 $(top_srcdir)/m4/warn-on-use.m4 \ + $(top_srcdir)/m4/warnings.m4 $(top_srcdir)/m4/wchar_h.m4 \ + $(top_srcdir)/m4/wchar_t.m4 $(top_srcdir)/m4/wcrtomb.m4 \ + $(top_srcdir)/m4/wctype_h.m4 $(top_srcdir)/m4/wcwidth.m4 \ + $(top_srcdir)/m4/wget.m4 $(top_srcdir)/m4/wget_manywarnings.m4 \ + $(top_srcdir)/m4/wint_t.m4 $(top_srcdir)/m4/wmemchr.m4 \ + $(top_srcdir)/m4/wmempcpy.m4 $(top_srcdir)/m4/write.m4 \ + $(top_srcdir)/m4/xalloc.m4 $(top_srcdir)/m4/xsize.m4 \ + $(top_srcdir)/m4/xstrndup.m4 $(top_srcdir)/m4/zzgnulib.m4 \ + $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__installdirs = "$(DESTDIR)$(bindir)" +PROGRAMS = $(bin_PROGRAMS) +AM_V_AR = $(am__v_AR_@AM_V@) +am__v_AR_ = $(am__v_AR_@AM_DEFAULT_V@) +am__v_AR_0 = @echo " AR " $@; +am__v_AR_1 = +libunittest_a_AR = $(AR) $(ARFLAGS) +libunittest_a_DEPENDENCIES = $(LIBOBJS) +am__libunittest_a_SOURCES_DIST = connect.c convert.c cookies.c ftp.c \ + css_.c css-url.c ftp-basic.c ftp-ls.c hash.c host.c hsts.c \ + html-parse.c html-url.c http.c init.c log.c main.c netrc.c \ + progress.c ptimer.c recur.c res.c retr.c spider.c url.c warc.c \ + xattr.c utils.c exits.c build_info.c iri.c metalink.c \ + css-url.h css-tokens.h connect.h convert.h cookies.h ftp.h \ + hash.h host.h hsts.h html-parse.h html-url.h http.h \ + http-ntlm.h init.h log.h mswindows.h netrc.h options.h \ + progress.h ptimer.h recur.h res.h retr.h spider.h ssl.h \ + sysdep.h url.h warc.h utils.h wget.h iri.h exits.h version.h \ + metalink.h xattr.h +@WITH_XATTR_TRUE@am__objects_1 = libunittest_a-xattr.$(OBJEXT) +@IRI_IS_ENABLED_TRUE@am__objects_2 = libunittest_a-iri.$(OBJEXT) +@METALINK_IS_ENABLED_TRUE@am__objects_3 = \ +@METALINK_IS_ENABLED_TRUE@ libunittest_a-metalink.$(OBJEXT) +am__objects_4 = libunittest_a-connect.$(OBJEXT) \ + libunittest_a-convert.$(OBJEXT) \ + libunittest_a-cookies.$(OBJEXT) libunittest_a-ftp.$(OBJEXT) \ + libunittest_a-css_.$(OBJEXT) libunittest_a-css-url.$(OBJEXT) \ + libunittest_a-ftp-basic.$(OBJEXT) \ + libunittest_a-ftp-ls.$(OBJEXT) libunittest_a-hash.$(OBJEXT) \ + libunittest_a-host.$(OBJEXT) libunittest_a-hsts.$(OBJEXT) \ + libunittest_a-html-parse.$(OBJEXT) \ + libunittest_a-html-url.$(OBJEXT) libunittest_a-http.$(OBJEXT) \ + libunittest_a-init.$(OBJEXT) libunittest_a-log.$(OBJEXT) \ + libunittest_a-main.$(OBJEXT) libunittest_a-netrc.$(OBJEXT) \ + libunittest_a-progress.$(OBJEXT) \ + libunittest_a-ptimer.$(OBJEXT) libunittest_a-recur.$(OBJEXT) \ + libunittest_a-res.$(OBJEXT) libunittest_a-retr.$(OBJEXT) \ + libunittest_a-spider.$(OBJEXT) libunittest_a-url.$(OBJEXT) \ + libunittest_a-warc.$(OBJEXT) $(am__objects_1) \ + libunittest_a-utils.$(OBJEXT) libunittest_a-exits.$(OBJEXT) \ + libunittest_a-build_info.$(OBJEXT) $(am__objects_2) \ + $(am__objects_3) +am_libunittest_a_OBJECTS = $(am__objects_4) \ + libunittest_a-build_info.$(OBJEXT) +nodist_libunittest_a_OBJECTS = libunittest_a-version.$(OBJEXT) +libunittest_a_OBJECTS = $(am_libunittest_a_OBJECTS) \ + $(nodist_libunittest_a_OBJECTS) +am__wget_SOURCES_DIST = connect.c convert.c cookies.c ftp.c css_.c \ + css-url.c ftp-basic.c ftp-ls.c hash.c host.c hsts.c \ + html-parse.c html-url.c http.c init.c log.c main.c netrc.c \ + progress.c ptimer.c recur.c res.c retr.c spider.c url.c warc.c \ + xattr.c utils.c exits.c build_info.c iri.c metalink.c \ + css-url.h css-tokens.h connect.h convert.h cookies.h ftp.h \ + hash.h host.h hsts.h html-parse.h html-url.h http.h \ + http-ntlm.h init.h log.h mswindows.h netrc.h options.h \ + progress.h ptimer.h recur.h res.h retr.h spider.h ssl.h \ + sysdep.h url.h warc.h utils.h wget.h iri.h exits.h version.h \ + metalink.h xattr.h +@WITH_XATTR_TRUE@am__objects_5 = xattr.$(OBJEXT) +@IRI_IS_ENABLED_TRUE@am__objects_6 = iri.$(OBJEXT) +@METALINK_IS_ENABLED_TRUE@am__objects_7 = metalink.$(OBJEXT) +am_wget_OBJECTS = connect.$(OBJEXT) convert.$(OBJEXT) \ + cookies.$(OBJEXT) ftp.$(OBJEXT) css_.$(OBJEXT) \ + css-url.$(OBJEXT) ftp-basic.$(OBJEXT) ftp-ls.$(OBJEXT) \ + hash.$(OBJEXT) host.$(OBJEXT) hsts.$(OBJEXT) \ + html-parse.$(OBJEXT) html-url.$(OBJEXT) http.$(OBJEXT) \ + init.$(OBJEXT) log.$(OBJEXT) main.$(OBJEXT) netrc.$(OBJEXT) \ + progress.$(OBJEXT) ptimer.$(OBJEXT) recur.$(OBJEXT) \ + res.$(OBJEXT) retr.$(OBJEXT) spider.$(OBJEXT) url.$(OBJEXT) \ + warc.$(OBJEXT) $(am__objects_5) utils.$(OBJEXT) \ + exits.$(OBJEXT) build_info.$(OBJEXT) $(am__objects_6) \ + $(am__objects_7) +nodist_wget_OBJECTS = version.$(OBJEXT) +wget_OBJECTS = $(am_wget_OBJECTS) $(nodist_wget_OBJECTS) +wget_LDADD = $(LDADD) +am__DEPENDENCIES_1 = +wget_DEPENDENCIES = $(LIBOBJS) ../lib/libgnu.a $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ +depcomp = $(SHELL) $(top_srcdir)/build-aux/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = $(DEPDIR)/ftp-opie.Po $(DEPDIR)/gnutls.Po \ + $(DEPDIR)/http-ntlm.Po $(DEPDIR)/mswindows.Po \ + $(DEPDIR)/openssl.Po ./$(DEPDIR)/build_info.Po \ + ./$(DEPDIR)/connect.Po ./$(DEPDIR)/convert.Po \ + ./$(DEPDIR)/cookies.Po ./$(DEPDIR)/css-url.Po \ + ./$(DEPDIR)/css_.Po ./$(DEPDIR)/exits.Po \ + ./$(DEPDIR)/ftp-basic.Po ./$(DEPDIR)/ftp-ls.Po \ + ./$(DEPDIR)/ftp.Po ./$(DEPDIR)/hash.Po ./$(DEPDIR)/host.Po \ + ./$(DEPDIR)/hsts.Po ./$(DEPDIR)/html-parse.Po \ + ./$(DEPDIR)/html-url.Po ./$(DEPDIR)/http.Po \ + ./$(DEPDIR)/init.Po ./$(DEPDIR)/iri.Po \ + ./$(DEPDIR)/libunittest_a-build_info.Po \ + ./$(DEPDIR)/libunittest_a-connect.Po \ + ./$(DEPDIR)/libunittest_a-convert.Po \ + ./$(DEPDIR)/libunittest_a-cookies.Po \ + ./$(DEPDIR)/libunittest_a-css-url.Po \ + ./$(DEPDIR)/libunittest_a-css_.Po \ + ./$(DEPDIR)/libunittest_a-exits.Po \ + ./$(DEPDIR)/libunittest_a-ftp-basic.Po \ + ./$(DEPDIR)/libunittest_a-ftp-ls.Po \ + ./$(DEPDIR)/libunittest_a-ftp.Po \ + ./$(DEPDIR)/libunittest_a-hash.Po \ + ./$(DEPDIR)/libunittest_a-host.Po \ + ./$(DEPDIR)/libunittest_a-hsts.Po \ + ./$(DEPDIR)/libunittest_a-html-parse.Po \ + ./$(DEPDIR)/libunittest_a-html-url.Po \ + ./$(DEPDIR)/libunittest_a-http.Po \ + ./$(DEPDIR)/libunittest_a-init.Po \ + ./$(DEPDIR)/libunittest_a-iri.Po \ + ./$(DEPDIR)/libunittest_a-log.Po \ + ./$(DEPDIR)/libunittest_a-main.Po \ + ./$(DEPDIR)/libunittest_a-metalink.Po \ + ./$(DEPDIR)/libunittest_a-netrc.Po \ + ./$(DEPDIR)/libunittest_a-progress.Po \ + ./$(DEPDIR)/libunittest_a-ptimer.Po \ + ./$(DEPDIR)/libunittest_a-recur.Po \ + ./$(DEPDIR)/libunittest_a-res.Po \ + ./$(DEPDIR)/libunittest_a-retr.Po \ + ./$(DEPDIR)/libunittest_a-spider.Po \ + ./$(DEPDIR)/libunittest_a-url.Po \ + ./$(DEPDIR)/libunittest_a-utils.Po \ + ./$(DEPDIR)/libunittest_a-version.Po \ + ./$(DEPDIR)/libunittest_a-warc.Po \ + ./$(DEPDIR)/libunittest_a-xattr.Po ./$(DEPDIR)/log.Po \ + ./$(DEPDIR)/main.Po ./$(DEPDIR)/metalink.Po \ + ./$(DEPDIR)/netrc.Po ./$(DEPDIR)/progress.Po \ + ./$(DEPDIR)/ptimer.Po ./$(DEPDIR)/recur.Po ./$(DEPDIR)/res.Po \ + ./$(DEPDIR)/retr.Po ./$(DEPDIR)/spider.Po ./$(DEPDIR)/url.Po \ + ./$(DEPDIR)/utils.Po ./$(DEPDIR)/version.Po \ + ./$(DEPDIR)/warc.Po ./$(DEPDIR)/xattr.Po +am__mv = mv -f +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libunittest_a_SOURCES) $(nodist_libunittest_a_SOURCES) \ + $(wget_SOURCES) $(EXTRA_wget_SOURCES) $(nodist_wget_SOURCES) +DIST_SOURCES = $(am__libunittest_a_SOURCES_DIST) \ + $(am__wget_SOURCES_DIST) $(EXTRA_wget_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) \ + config.h.in +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/config.h.in \ + $(top_srcdir)/build-aux/depcomp ftp-opie.c gnutls.c \ + http-ntlm.c mswindows.c openssl.c +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ALLOCA = @ALLOCA@ +ALLOCA_H = @ALLOCA_H@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPLE_UNIVERSAL_BUILD = @APPLE_UNIVERSAL_BUILD@ +AR = @AR@ +ARFLAGS = @ARFLAGS@ +ASM_SYMBOL_PREFIX = @ASM_SYMBOL_PREFIX@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BITSIZEOF_PTRDIFF_T = @BITSIZEOF_PTRDIFF_T@ +BITSIZEOF_SIG_ATOMIC_T = @BITSIZEOF_SIG_ATOMIC_T@ +BITSIZEOF_SIZE_T = @BITSIZEOF_SIZE_T@ +BITSIZEOF_WCHAR_T = @BITSIZEOF_WCHAR_T@ +BITSIZEOF_WINT_T = @BITSIZEOF_WINT_T@ +BYTESWAP_H = @BYTESWAP_H@ +CARES_CFLAGS = @CARES_CFLAGS@ +CARES_LIBS = @CARES_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CFLAG_VISIBILITY = @CFLAG_VISIBILITY@ +COMMENT_IF_NO_POD2MAN = @COMMENT_IF_NO_POD2MAN@ +CONFIG_INCLUDE = @CONFIG_INCLUDE@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CYGPATH_W = @CYGPATH_W@ + +# The following line is losing on some versions of make! +DEFS = @DEFS@ -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\" +DEPDIR = @DEPDIR@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EMULTIHOP_HIDDEN = @EMULTIHOP_HIDDEN@ +EMULTIHOP_VALUE = @EMULTIHOP_VALUE@ +ENOLINK_HIDDEN = @ENOLINK_HIDDEN@ +ENOLINK_VALUE = @ENOLINK_VALUE@ +EOVERFLOW_HIDDEN = @EOVERFLOW_HIDDEN@ +EOVERFLOW_VALUE = @EOVERFLOW_VALUE@ +ERRNO_H = @ERRNO_H@ +EXEEXT = @EXEEXT@ +FLOAT_H = @FLOAT_H@ +FNMATCH_H = @FNMATCH_H@ +FUZZ_LIBS = @FUZZ_LIBS@ +GETADDRINFO_LIB = @GETADDRINFO_LIB@ +GETOPT_CDEFS_H = @GETOPT_CDEFS_H@ +GETOPT_H = @GETOPT_H@ +GETTEXT_MACRO_VERSION = @GETTEXT_MACRO_VERSION@ +GMSGFMT = @GMSGFMT@ +GMSGFMT_015 = @GMSGFMT_015@ +GNULIB_ACCEPT = @GNULIB_ACCEPT@ +GNULIB_ACCEPT4 = @GNULIB_ACCEPT4@ +GNULIB_ACCESS = @GNULIB_ACCESS@ +GNULIB_ALIGNED_ALLOC = @GNULIB_ALIGNED_ALLOC@ +GNULIB_ALPHASORT = @GNULIB_ALPHASORT@ +GNULIB_ATOLL = @GNULIB_ATOLL@ +GNULIB_BIND = @GNULIB_BIND@ +GNULIB_BTOWC = @GNULIB_BTOWC@ +GNULIB_CALLOC_POSIX = @GNULIB_CALLOC_POSIX@ +GNULIB_CANONICALIZE_FILE_NAME = @GNULIB_CANONICALIZE_FILE_NAME@ +GNULIB_CHDIR = @GNULIB_CHDIR@ +GNULIB_CHOWN = @GNULIB_CHOWN@ +GNULIB_CLOSE = @GNULIB_CLOSE@ +GNULIB_CLOSEDIR = @GNULIB_CLOSEDIR@ +GNULIB_CONNECT = @GNULIB_CONNECT@ +GNULIB_COPY_FILE_RANGE = @GNULIB_COPY_FILE_RANGE@ +GNULIB_CREAT = @GNULIB_CREAT@ +GNULIB_CTIME = @GNULIB_CTIME@ +GNULIB_DIRFD = @GNULIB_DIRFD@ +GNULIB_DPRINTF = @GNULIB_DPRINTF@ +GNULIB_DUP = @GNULIB_DUP@ +GNULIB_DUP2 = @GNULIB_DUP2@ +GNULIB_DUP3 = @GNULIB_DUP3@ +GNULIB_DUPLOCALE = @GNULIB_DUPLOCALE@ +GNULIB_ENVIRON = @GNULIB_ENVIRON@ +GNULIB_EUIDACCESS = @GNULIB_EUIDACCESS@ +GNULIB_EXECL = @GNULIB_EXECL@ +GNULIB_EXECLE = @GNULIB_EXECLE@ +GNULIB_EXECLP = @GNULIB_EXECLP@ +GNULIB_EXECV = @GNULIB_EXECV@ +GNULIB_EXECVE = @GNULIB_EXECVE@ +GNULIB_EXECVP = @GNULIB_EXECVP@ +GNULIB_EXECVPE = @GNULIB_EXECVPE@ +GNULIB_EXPLICIT_BZERO = @GNULIB_EXPLICIT_BZERO@ +GNULIB_FACCESSAT = @GNULIB_FACCESSAT@ +GNULIB_FCHDIR = @GNULIB_FCHDIR@ +GNULIB_FCHMODAT = @GNULIB_FCHMODAT@ +GNULIB_FCHOWNAT = @GNULIB_FCHOWNAT@ +GNULIB_FCLOSE = @GNULIB_FCLOSE@ +GNULIB_FCNTL = @GNULIB_FCNTL@ +GNULIB_FDATASYNC = @GNULIB_FDATASYNC@ +GNULIB_FDOPEN = @GNULIB_FDOPEN@ +GNULIB_FDOPENDIR = @GNULIB_FDOPENDIR@ +GNULIB_FFLUSH = @GNULIB_FFLUSH@ +GNULIB_FFS = @GNULIB_FFS@ +GNULIB_FFSL = @GNULIB_FFSL@ +GNULIB_FFSLL = @GNULIB_FFSLL@ +GNULIB_FGETC = @GNULIB_FGETC@ +GNULIB_FGETS = @GNULIB_FGETS@ +GNULIB_FLOCK = @GNULIB_FLOCK@ +GNULIB_FNMATCH = @GNULIB_FNMATCH@ +GNULIB_FOPEN = @GNULIB_FOPEN@ +GNULIB_FPRINTF = @GNULIB_FPRINTF@ +GNULIB_FPRINTF_POSIX = @GNULIB_FPRINTF_POSIX@ +GNULIB_FPURGE = @GNULIB_FPURGE@ +GNULIB_FPUTC = @GNULIB_FPUTC@ +GNULIB_FPUTS = @GNULIB_FPUTS@ +GNULIB_FREAD = @GNULIB_FREAD@ +GNULIB_FREE_POSIX = @GNULIB_FREE_POSIX@ +GNULIB_FREOPEN = @GNULIB_FREOPEN@ +GNULIB_FSCANF = @GNULIB_FSCANF@ +GNULIB_FSEEK = @GNULIB_FSEEK@ +GNULIB_FSEEKO = @GNULIB_FSEEKO@ +GNULIB_FSTAT = @GNULIB_FSTAT@ +GNULIB_FSTATAT = @GNULIB_FSTATAT@ +GNULIB_FSYNC = @GNULIB_FSYNC@ +GNULIB_FTELL = @GNULIB_FTELL@ +GNULIB_FTELLO = @GNULIB_FTELLO@ +GNULIB_FTRUNCATE = @GNULIB_FTRUNCATE@ +GNULIB_FUTIMENS = @GNULIB_FUTIMENS@ +GNULIB_FWRITE = @GNULIB_FWRITE@ +GNULIB_GETADDRINFO = @GNULIB_GETADDRINFO@ +GNULIB_GETC = @GNULIB_GETC@ +GNULIB_GETCHAR = @GNULIB_GETCHAR@ +GNULIB_GETCWD = @GNULIB_GETCWD@ +GNULIB_GETDELIM = @GNULIB_GETDELIM@ +GNULIB_GETDOMAINNAME = @GNULIB_GETDOMAINNAME@ +GNULIB_GETDTABLESIZE = @GNULIB_GETDTABLESIZE@ +GNULIB_GETENTROPY = @GNULIB_GETENTROPY@ +GNULIB_GETGROUPS = @GNULIB_GETGROUPS@ +GNULIB_GETHOSTNAME = @GNULIB_GETHOSTNAME@ +GNULIB_GETLINE = @GNULIB_GETLINE@ +GNULIB_GETLOADAVG = @GNULIB_GETLOADAVG@ +GNULIB_GETLOGIN = @GNULIB_GETLOGIN@ +GNULIB_GETLOGIN_R = @GNULIB_GETLOGIN_R@ +GNULIB_GETOPT_POSIX = @GNULIB_GETOPT_POSIX@ +GNULIB_GETPAGESIZE = @GNULIB_GETPAGESIZE@ +GNULIB_GETPASS = @GNULIB_GETPASS@ +GNULIB_GETPEERNAME = @GNULIB_GETPEERNAME@ +GNULIB_GETRANDOM = @GNULIB_GETRANDOM@ +GNULIB_GETSOCKNAME = @GNULIB_GETSOCKNAME@ +GNULIB_GETSOCKOPT = @GNULIB_GETSOCKOPT@ +GNULIB_GETSUBOPT = @GNULIB_GETSUBOPT@ +GNULIB_GETTIMEOFDAY = @GNULIB_GETTIMEOFDAY@ +GNULIB_GETUMASK = @GNULIB_GETUMASK@ +GNULIB_GETUSERSHELL = @GNULIB_GETUSERSHELL@ +GNULIB_GL_UNISTD_H_GETOPT = @GNULIB_GL_UNISTD_H_GETOPT@ +GNULIB_GRANTPT = @GNULIB_GRANTPT@ +GNULIB_GROUP_MEMBER = @GNULIB_GROUP_MEMBER@ +GNULIB_ICONV = @GNULIB_ICONV@ +GNULIB_IMAXABS = @GNULIB_IMAXABS@ +GNULIB_IMAXDIV = @GNULIB_IMAXDIV@ +GNULIB_INET_NTOP = @GNULIB_INET_NTOP@ +GNULIB_INET_PTON = @GNULIB_INET_PTON@ +GNULIB_IOCTL = @GNULIB_IOCTL@ +GNULIB_ISATTY = @GNULIB_ISATTY@ +GNULIB_ISBLANK = @GNULIB_ISBLANK@ +GNULIB_ISWBLANK = @GNULIB_ISWBLANK@ +GNULIB_ISWCTYPE = @GNULIB_ISWCTYPE@ +GNULIB_ISWDIGIT = @GNULIB_ISWDIGIT@ +GNULIB_ISWXDIGIT = @GNULIB_ISWXDIGIT@ +GNULIB_LCHMOD = @GNULIB_LCHMOD@ +GNULIB_LCHOWN = @GNULIB_LCHOWN@ +GNULIB_LINK = @GNULIB_LINK@ +GNULIB_LINKAT = @GNULIB_LINKAT@ +GNULIB_LISTEN = @GNULIB_LISTEN@ +GNULIB_LOCALECONV = @GNULIB_LOCALECONV@ +GNULIB_LOCALENAME = @GNULIB_LOCALENAME@ +GNULIB_LOCALTIME = @GNULIB_LOCALTIME@ +GNULIB_LSEEK = @GNULIB_LSEEK@ +GNULIB_LSTAT = @GNULIB_LSTAT@ +GNULIB_MALLOC_POSIX = @GNULIB_MALLOC_POSIX@ +GNULIB_MBRLEN = @GNULIB_MBRLEN@ +GNULIB_MBRTOWC = @GNULIB_MBRTOWC@ +GNULIB_MBSCASECMP = @GNULIB_MBSCASECMP@ +GNULIB_MBSCASESTR = @GNULIB_MBSCASESTR@ +GNULIB_MBSCHR = @GNULIB_MBSCHR@ +GNULIB_MBSCSPN = @GNULIB_MBSCSPN@ +GNULIB_MBSINIT = @GNULIB_MBSINIT@ +GNULIB_MBSLEN = @GNULIB_MBSLEN@ +GNULIB_MBSNCASECMP = @GNULIB_MBSNCASECMP@ +GNULIB_MBSNLEN = @GNULIB_MBSNLEN@ +GNULIB_MBSNRTOWCS = @GNULIB_MBSNRTOWCS@ +GNULIB_MBSPBRK = @GNULIB_MBSPBRK@ +GNULIB_MBSPCASECMP = @GNULIB_MBSPCASECMP@ +GNULIB_MBSRCHR = @GNULIB_MBSRCHR@ +GNULIB_MBSRTOWCS = @GNULIB_MBSRTOWCS@ +GNULIB_MBSSEP = @GNULIB_MBSSEP@ +GNULIB_MBSSPN = @GNULIB_MBSSPN@ +GNULIB_MBSSTR = @GNULIB_MBSSTR@ +GNULIB_MBSTOK_R = @GNULIB_MBSTOK_R@ +GNULIB_MBTOWC = @GNULIB_MBTOWC@ +GNULIB_MDA_ACCESS = @GNULIB_MDA_ACCESS@ +GNULIB_MDA_CHDIR = @GNULIB_MDA_CHDIR@ +GNULIB_MDA_CHMOD = @GNULIB_MDA_CHMOD@ +GNULIB_MDA_CLOSE = @GNULIB_MDA_CLOSE@ +GNULIB_MDA_CREAT = @GNULIB_MDA_CREAT@ +GNULIB_MDA_DUP = @GNULIB_MDA_DUP@ +GNULIB_MDA_DUP2 = @GNULIB_MDA_DUP2@ +GNULIB_MDA_ECVT = @GNULIB_MDA_ECVT@ +GNULIB_MDA_EXECL = @GNULIB_MDA_EXECL@ +GNULIB_MDA_EXECLE = @GNULIB_MDA_EXECLE@ +GNULIB_MDA_EXECLP = @GNULIB_MDA_EXECLP@ +GNULIB_MDA_EXECV = @GNULIB_MDA_EXECV@ +GNULIB_MDA_EXECVE = @GNULIB_MDA_EXECVE@ +GNULIB_MDA_EXECVP = @GNULIB_MDA_EXECVP@ +GNULIB_MDA_EXECVPE = @GNULIB_MDA_EXECVPE@ +GNULIB_MDA_FCLOSEALL = @GNULIB_MDA_FCLOSEALL@ +GNULIB_MDA_FCVT = @GNULIB_MDA_FCVT@ +GNULIB_MDA_FDOPEN = @GNULIB_MDA_FDOPEN@ +GNULIB_MDA_FILENO = @GNULIB_MDA_FILENO@ +GNULIB_MDA_GCVT = @GNULIB_MDA_GCVT@ +GNULIB_MDA_GETCWD = @GNULIB_MDA_GETCWD@ +GNULIB_MDA_GETPID = @GNULIB_MDA_GETPID@ +GNULIB_MDA_GETW = @GNULIB_MDA_GETW@ +GNULIB_MDA_ISATTY = @GNULIB_MDA_ISATTY@ +GNULIB_MDA_LSEEK = @GNULIB_MDA_LSEEK@ +GNULIB_MDA_MEMCCPY = @GNULIB_MDA_MEMCCPY@ +GNULIB_MDA_MKDIR = @GNULIB_MDA_MKDIR@ +GNULIB_MDA_MKTEMP = @GNULIB_MDA_MKTEMP@ +GNULIB_MDA_OPEN = @GNULIB_MDA_OPEN@ +GNULIB_MDA_PUTENV = @GNULIB_MDA_PUTENV@ +GNULIB_MDA_PUTW = @GNULIB_MDA_PUTW@ +GNULIB_MDA_READ = @GNULIB_MDA_READ@ +GNULIB_MDA_RMDIR = @GNULIB_MDA_RMDIR@ +GNULIB_MDA_STRDUP = @GNULIB_MDA_STRDUP@ +GNULIB_MDA_SWAB = @GNULIB_MDA_SWAB@ +GNULIB_MDA_TEMPNAM = @GNULIB_MDA_TEMPNAM@ +GNULIB_MDA_TZSET = @GNULIB_MDA_TZSET@ +GNULIB_MDA_UMASK = @GNULIB_MDA_UMASK@ +GNULIB_MDA_UNLINK = @GNULIB_MDA_UNLINK@ +GNULIB_MDA_UTIME = @GNULIB_MDA_UTIME@ +GNULIB_MDA_WCSDUP = @GNULIB_MDA_WCSDUP@ +GNULIB_MDA_WRITE = @GNULIB_MDA_WRITE@ +GNULIB_MEMCHR = @GNULIB_MEMCHR@ +GNULIB_MEMMEM = @GNULIB_MEMMEM@ +GNULIB_MEMPCPY = @GNULIB_MEMPCPY@ +GNULIB_MEMRCHR = @GNULIB_MEMRCHR@ +GNULIB_MKDIR = @GNULIB_MKDIR@ +GNULIB_MKDIRAT = @GNULIB_MKDIRAT@ +GNULIB_MKDTEMP = @GNULIB_MKDTEMP@ +GNULIB_MKFIFO = @GNULIB_MKFIFO@ +GNULIB_MKFIFOAT = @GNULIB_MKFIFOAT@ +GNULIB_MKNOD = @GNULIB_MKNOD@ +GNULIB_MKNODAT = @GNULIB_MKNODAT@ +GNULIB_MKOSTEMP = @GNULIB_MKOSTEMP@ +GNULIB_MKOSTEMPS = @GNULIB_MKOSTEMPS@ +GNULIB_MKSTEMP = @GNULIB_MKSTEMP@ +GNULIB_MKSTEMPS = @GNULIB_MKSTEMPS@ +GNULIB_MKTIME = @GNULIB_MKTIME@ +GNULIB_NANOSLEEP = @GNULIB_NANOSLEEP@ +GNULIB_NL_LANGINFO = @GNULIB_NL_LANGINFO@ +GNULIB_NONBLOCKING = @GNULIB_NONBLOCKING@ +GNULIB_OBSTACK_PRINTF = @GNULIB_OBSTACK_PRINTF@ +GNULIB_OBSTACK_PRINTF_POSIX = @GNULIB_OBSTACK_PRINTF_POSIX@ +GNULIB_OPEN = @GNULIB_OPEN@ +GNULIB_OPENAT = @GNULIB_OPENAT@ +GNULIB_OPENDIR = @GNULIB_OPENDIR@ +GNULIB_OVERRIDES_STRUCT_STAT = @GNULIB_OVERRIDES_STRUCT_STAT@ +GNULIB_OVERRIDES_WINT_T = @GNULIB_OVERRIDES_WINT_T@ +GNULIB_PCLOSE = @GNULIB_PCLOSE@ +GNULIB_PERROR = @GNULIB_PERROR@ +GNULIB_PIPE = @GNULIB_PIPE@ +GNULIB_PIPE2 = @GNULIB_PIPE2@ +GNULIB_POPEN = @GNULIB_POPEN@ +GNULIB_POSIX_MEMALIGN = @GNULIB_POSIX_MEMALIGN@ +GNULIB_POSIX_OPENPT = @GNULIB_POSIX_OPENPT@ +GNULIB_POSIX_SPAWN = @GNULIB_POSIX_SPAWN@ +GNULIB_POSIX_SPAWNATTR_DESTROY = @GNULIB_POSIX_SPAWNATTR_DESTROY@ +GNULIB_POSIX_SPAWNATTR_GETFLAGS = @GNULIB_POSIX_SPAWNATTR_GETFLAGS@ +GNULIB_POSIX_SPAWNATTR_GETPGROUP = @GNULIB_POSIX_SPAWNATTR_GETPGROUP@ +GNULIB_POSIX_SPAWNATTR_GETSCHEDPARAM = @GNULIB_POSIX_SPAWNATTR_GETSCHEDPARAM@ +GNULIB_POSIX_SPAWNATTR_GETSCHEDPOLICY = @GNULIB_POSIX_SPAWNATTR_GETSCHEDPOLICY@ +GNULIB_POSIX_SPAWNATTR_GETSIGDEFAULT = @GNULIB_POSIX_SPAWNATTR_GETSIGDEFAULT@ +GNULIB_POSIX_SPAWNATTR_GETSIGMASK = @GNULIB_POSIX_SPAWNATTR_GETSIGMASK@ +GNULIB_POSIX_SPAWNATTR_INIT = @GNULIB_POSIX_SPAWNATTR_INIT@ +GNULIB_POSIX_SPAWNATTR_SETFLAGS = @GNULIB_POSIX_SPAWNATTR_SETFLAGS@ +GNULIB_POSIX_SPAWNATTR_SETPGROUP = @GNULIB_POSIX_SPAWNATTR_SETPGROUP@ +GNULIB_POSIX_SPAWNATTR_SETSCHEDPARAM = @GNULIB_POSIX_SPAWNATTR_SETSCHEDPARAM@ +GNULIB_POSIX_SPAWNATTR_SETSCHEDPOLICY = @GNULIB_POSIX_SPAWNATTR_SETSCHEDPOLICY@ +GNULIB_POSIX_SPAWNATTR_SETSIGDEFAULT = @GNULIB_POSIX_SPAWNATTR_SETSIGDEFAULT@ +GNULIB_POSIX_SPAWNATTR_SETSIGMASK = @GNULIB_POSIX_SPAWNATTR_SETSIGMASK@ +GNULIB_POSIX_SPAWNP = @GNULIB_POSIX_SPAWNP@ +GNULIB_POSIX_SPAWN_FILE_ACTIONS_ADDCHDIR = @GNULIB_POSIX_SPAWN_FILE_ACTIONS_ADDCHDIR@ +GNULIB_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE = @GNULIB_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE@ +GNULIB_POSIX_SPAWN_FILE_ACTIONS_ADDDUP2 = @GNULIB_POSIX_SPAWN_FILE_ACTIONS_ADDDUP2@ +GNULIB_POSIX_SPAWN_FILE_ACTIONS_ADDFCHDIR = @GNULIB_POSIX_SPAWN_FILE_ACTIONS_ADDFCHDIR@ +GNULIB_POSIX_SPAWN_FILE_ACTIONS_ADDOPEN = @GNULIB_POSIX_SPAWN_FILE_ACTIONS_ADDOPEN@ +GNULIB_POSIX_SPAWN_FILE_ACTIONS_DESTROY = @GNULIB_POSIX_SPAWN_FILE_ACTIONS_DESTROY@ +GNULIB_POSIX_SPAWN_FILE_ACTIONS_INIT = @GNULIB_POSIX_SPAWN_FILE_ACTIONS_INIT@ +GNULIB_PREAD = @GNULIB_PREAD@ +GNULIB_PRINTF = @GNULIB_PRINTF@ +GNULIB_PRINTF_POSIX = @GNULIB_PRINTF_POSIX@ +GNULIB_PSELECT = @GNULIB_PSELECT@ +GNULIB_PTHREAD_SIGMASK = @GNULIB_PTHREAD_SIGMASK@ +GNULIB_PTSNAME = @GNULIB_PTSNAME@ +GNULIB_PTSNAME_R = @GNULIB_PTSNAME_R@ +GNULIB_PUTC = @GNULIB_PUTC@ +GNULIB_PUTCHAR = @GNULIB_PUTCHAR@ +GNULIB_PUTENV = @GNULIB_PUTENV@ +GNULIB_PUTS = @GNULIB_PUTS@ +GNULIB_PWRITE = @GNULIB_PWRITE@ +GNULIB_QSORT_R = @GNULIB_QSORT_R@ +GNULIB_RAISE = @GNULIB_RAISE@ +GNULIB_RANDOM = @GNULIB_RANDOM@ +GNULIB_RANDOM_R = @GNULIB_RANDOM_R@ +GNULIB_RAWMEMCHR = @GNULIB_RAWMEMCHR@ +GNULIB_READ = @GNULIB_READ@ +GNULIB_READDIR = @GNULIB_READDIR@ +GNULIB_READLINK = @GNULIB_READLINK@ +GNULIB_READLINKAT = @GNULIB_READLINKAT@ +GNULIB_REALLOCARRAY = @GNULIB_REALLOCARRAY@ +GNULIB_REALLOC_POSIX = @GNULIB_REALLOC_POSIX@ +GNULIB_REALPATH = @GNULIB_REALPATH@ +GNULIB_RECV = @GNULIB_RECV@ +GNULIB_RECVFROM = @GNULIB_RECVFROM@ +GNULIB_REMOVE = @GNULIB_REMOVE@ +GNULIB_RENAME = @GNULIB_RENAME@ +GNULIB_RENAMEAT = @GNULIB_RENAMEAT@ +GNULIB_REWINDDIR = @GNULIB_REWINDDIR@ +GNULIB_RMDIR = @GNULIB_RMDIR@ +GNULIB_RPMATCH = @GNULIB_RPMATCH@ +GNULIB_SCANDIR = @GNULIB_SCANDIR@ +GNULIB_SCANF = @GNULIB_SCANF@ +GNULIB_SCHED_YIELD = @GNULIB_SCHED_YIELD@ +GNULIB_SECURE_GETENV = @GNULIB_SECURE_GETENV@ +GNULIB_SELECT = @GNULIB_SELECT@ +GNULIB_SEND = @GNULIB_SEND@ +GNULIB_SENDTO = @GNULIB_SENDTO@ +GNULIB_SETENV = @GNULIB_SETENV@ +GNULIB_SETHOSTNAME = @GNULIB_SETHOSTNAME@ +GNULIB_SETLOCALE = @GNULIB_SETLOCALE@ +GNULIB_SETLOCALE_NULL = @GNULIB_SETLOCALE_NULL@ +GNULIB_SETSOCKOPT = @GNULIB_SETSOCKOPT@ +GNULIB_SHUTDOWN = @GNULIB_SHUTDOWN@ +GNULIB_SIGABBREV_NP = @GNULIB_SIGABBREV_NP@ +GNULIB_SIGACTION = @GNULIB_SIGACTION@ +GNULIB_SIGDESCR_NP = @GNULIB_SIGDESCR_NP@ +GNULIB_SIGNAL_H_SIGPIPE = @GNULIB_SIGNAL_H_SIGPIPE@ +GNULIB_SIGPROCMASK = @GNULIB_SIGPROCMASK@ +GNULIB_SLEEP = @GNULIB_SLEEP@ +GNULIB_SNPRINTF = @GNULIB_SNPRINTF@ +GNULIB_SOCKET = @GNULIB_SOCKET@ +GNULIB_SPRINTF_POSIX = @GNULIB_SPRINTF_POSIX@ +GNULIB_STAT = @GNULIB_STAT@ +GNULIB_STDIO_H_NONBLOCKING = @GNULIB_STDIO_H_NONBLOCKING@ +GNULIB_STDIO_H_SIGPIPE = @GNULIB_STDIO_H_SIGPIPE@ +GNULIB_STPCPY = @GNULIB_STPCPY@ +GNULIB_STPNCPY = @GNULIB_STPNCPY@ +GNULIB_STRCASESTR = @GNULIB_STRCASESTR@ +GNULIB_STRCHRNUL = @GNULIB_STRCHRNUL@ +GNULIB_STRDUP = @GNULIB_STRDUP@ +GNULIB_STRERROR = @GNULIB_STRERROR@ +GNULIB_STRERRORNAME_NP = @GNULIB_STRERRORNAME_NP@ +GNULIB_STRERROR_R = @GNULIB_STRERROR_R@ +GNULIB_STRFTIME = @GNULIB_STRFTIME@ +GNULIB_STRNCAT = @GNULIB_STRNCAT@ +GNULIB_STRNDUP = @GNULIB_STRNDUP@ +GNULIB_STRNLEN = @GNULIB_STRNLEN@ +GNULIB_STRPBRK = @GNULIB_STRPBRK@ +GNULIB_STRPTIME = @GNULIB_STRPTIME@ +GNULIB_STRSEP = @GNULIB_STRSEP@ +GNULIB_STRSIGNAL = @GNULIB_STRSIGNAL@ +GNULIB_STRSTR = @GNULIB_STRSTR@ +GNULIB_STRTOD = @GNULIB_STRTOD@ +GNULIB_STRTOIMAX = @GNULIB_STRTOIMAX@ +GNULIB_STRTOK_R = @GNULIB_STRTOK_R@ +GNULIB_STRTOLD = @GNULIB_STRTOLD@ +GNULIB_STRTOLL = @GNULIB_STRTOLL@ +GNULIB_STRTOULL = @GNULIB_STRTOULL@ +GNULIB_STRTOUMAX = @GNULIB_STRTOUMAX@ +GNULIB_STRVERSCMP = @GNULIB_STRVERSCMP@ +GNULIB_SYMLINK = @GNULIB_SYMLINK@ +GNULIB_SYMLINKAT = @GNULIB_SYMLINKAT@ +GNULIB_SYSTEM_POSIX = @GNULIB_SYSTEM_POSIX@ +GNULIB_TIMEGM = @GNULIB_TIMEGM@ +GNULIB_TIME_R = @GNULIB_TIME_R@ +GNULIB_TIME_RZ = @GNULIB_TIME_RZ@ +GNULIB_TMPFILE = @GNULIB_TMPFILE@ +GNULIB_TOWCTRANS = @GNULIB_TOWCTRANS@ +GNULIB_TRUNCATE = @GNULIB_TRUNCATE@ +GNULIB_TTYNAME_R = @GNULIB_TTYNAME_R@ +GNULIB_TZSET = @GNULIB_TZSET@ +GNULIB_UNISTD_H_NONBLOCKING = @GNULIB_UNISTD_H_NONBLOCKING@ +GNULIB_UNISTD_H_SIGPIPE = @GNULIB_UNISTD_H_SIGPIPE@ +GNULIB_UNLINK = @GNULIB_UNLINK@ +GNULIB_UNLINKAT = @GNULIB_UNLINKAT@ +GNULIB_UNLOCKPT = @GNULIB_UNLOCKPT@ +GNULIB_UNSETENV = @GNULIB_UNSETENV@ +GNULIB_USLEEP = @GNULIB_USLEEP@ +GNULIB_UTIME = @GNULIB_UTIME@ +GNULIB_UTIMENSAT = @GNULIB_UTIMENSAT@ +GNULIB_VASPRINTF = @GNULIB_VASPRINTF@ +GNULIB_VDPRINTF = @GNULIB_VDPRINTF@ +GNULIB_VFPRINTF = @GNULIB_VFPRINTF@ +GNULIB_VFPRINTF_POSIX = @GNULIB_VFPRINTF_POSIX@ +GNULIB_VFSCANF = @GNULIB_VFSCANF@ +GNULIB_VPRINTF = @GNULIB_VPRINTF@ +GNULIB_VPRINTF_POSIX = @GNULIB_VPRINTF_POSIX@ +GNULIB_VSCANF = @GNULIB_VSCANF@ +GNULIB_VSNPRINTF = @GNULIB_VSNPRINTF@ +GNULIB_VSPRINTF_POSIX = @GNULIB_VSPRINTF_POSIX@ +GNULIB_WAITPID = @GNULIB_WAITPID@ +GNULIB_WARN_CFLAGS = @GNULIB_WARN_CFLAGS@ +GNULIB_WCPCPY = @GNULIB_WCPCPY@ +GNULIB_WCPNCPY = @GNULIB_WCPNCPY@ +GNULIB_WCRTOMB = @GNULIB_WCRTOMB@ +GNULIB_WCSCASECMP = @GNULIB_WCSCASECMP@ +GNULIB_WCSCAT = @GNULIB_WCSCAT@ +GNULIB_WCSCHR = @GNULIB_WCSCHR@ +GNULIB_WCSCMP = @GNULIB_WCSCMP@ +GNULIB_WCSCOLL = @GNULIB_WCSCOLL@ +GNULIB_WCSCPY = @GNULIB_WCSCPY@ +GNULIB_WCSCSPN = @GNULIB_WCSCSPN@ +GNULIB_WCSDUP = @GNULIB_WCSDUP@ +GNULIB_WCSFTIME = @GNULIB_WCSFTIME@ +GNULIB_WCSLEN = @GNULIB_WCSLEN@ +GNULIB_WCSNCASECMP = @GNULIB_WCSNCASECMP@ +GNULIB_WCSNCAT = @GNULIB_WCSNCAT@ +GNULIB_WCSNCMP = @GNULIB_WCSNCMP@ +GNULIB_WCSNCPY = @GNULIB_WCSNCPY@ +GNULIB_WCSNLEN = @GNULIB_WCSNLEN@ +GNULIB_WCSNRTOMBS = @GNULIB_WCSNRTOMBS@ +GNULIB_WCSPBRK = @GNULIB_WCSPBRK@ +GNULIB_WCSRCHR = @GNULIB_WCSRCHR@ +GNULIB_WCSRTOMBS = @GNULIB_WCSRTOMBS@ +GNULIB_WCSSPN = @GNULIB_WCSSPN@ +GNULIB_WCSSTR = @GNULIB_WCSSTR@ +GNULIB_WCSTOK = @GNULIB_WCSTOK@ +GNULIB_WCSWIDTH = @GNULIB_WCSWIDTH@ +GNULIB_WCSXFRM = @GNULIB_WCSXFRM@ +GNULIB_WCTOB = @GNULIB_WCTOB@ +GNULIB_WCTOMB = @GNULIB_WCTOMB@ +GNULIB_WCTRANS = @GNULIB_WCTRANS@ +GNULIB_WCTYPE = @GNULIB_WCTYPE@ +GNULIB_WCWIDTH = @GNULIB_WCWIDTH@ +GNULIB_WMEMCHR = @GNULIB_WMEMCHR@ +GNULIB_WMEMCMP = @GNULIB_WMEMCMP@ +GNULIB_WMEMCPY = @GNULIB_WMEMCPY@ +GNULIB_WMEMMOVE = @GNULIB_WMEMMOVE@ +GNULIB_WMEMPCPY = @GNULIB_WMEMPCPY@ +GNULIB_WMEMSET = @GNULIB_WMEMSET@ +GNULIB_WRITE = @GNULIB_WRITE@ +GNULIB__EXIT = @GNULIB__EXIT@ +GNUTLS_CFLAGS = @GNUTLS_CFLAGS@ +GNUTLS_LIBS = @GNUTLS_LIBS@ +GPGME_CFLAGS = @GPGME_CFLAGS@ +GPGME_CONFIG = @GPGME_CONFIG@ +GPGME_LIBS = @GPGME_LIBS@ +GREP = @GREP@ +HAVE_ACCEPT4 = @HAVE_ACCEPT4@ +HAVE_ALIGNED_ALLOC = @HAVE_ALIGNED_ALLOC@ +HAVE_ALLOCA_H = @HAVE_ALLOCA_H@ +HAVE_ALPHASORT = @HAVE_ALPHASORT@ +HAVE_ARPA_INET_H = @HAVE_ARPA_INET_H@ +HAVE_ATOLL = @HAVE_ATOLL@ +HAVE_BTOWC = @HAVE_BTOWC@ +HAVE_C99_STDINT_H = @HAVE_C99_STDINT_H@ +HAVE_CANONICALIZE_FILE_NAME = @HAVE_CANONICALIZE_FILE_NAME@ +HAVE_CHOWN = @HAVE_CHOWN@ +HAVE_CLOSEDIR = @HAVE_CLOSEDIR@ +HAVE_COPY_FILE_RANGE = @HAVE_COPY_FILE_RANGE@ +HAVE_CRTDEFS_H = @HAVE_CRTDEFS_H@ +HAVE_DECL_DIRFD = @HAVE_DECL_DIRFD@ +HAVE_DECL_ECVT = @HAVE_DECL_ECVT@ +HAVE_DECL_ENVIRON = @HAVE_DECL_ENVIRON@ +HAVE_DECL_EXECVPE = @HAVE_DECL_EXECVPE@ +HAVE_DECL_FCHDIR = @HAVE_DECL_FCHDIR@ +HAVE_DECL_FCLOSEALL = @HAVE_DECL_FCLOSEALL@ +HAVE_DECL_FCVT = @HAVE_DECL_FCVT@ +HAVE_DECL_FDATASYNC = @HAVE_DECL_FDATASYNC@ +HAVE_DECL_FDOPENDIR = @HAVE_DECL_FDOPENDIR@ +HAVE_DECL_FPURGE = @HAVE_DECL_FPURGE@ +HAVE_DECL_FREEADDRINFO = @HAVE_DECL_FREEADDRINFO@ +HAVE_DECL_FSEEKO = @HAVE_DECL_FSEEKO@ +HAVE_DECL_FTELLO = @HAVE_DECL_FTELLO@ +HAVE_DECL_GAI_STRERROR = @HAVE_DECL_GAI_STRERROR@ +HAVE_DECL_GCVT = @HAVE_DECL_GCVT@ +HAVE_DECL_GETADDRINFO = @HAVE_DECL_GETADDRINFO@ +HAVE_DECL_GETDELIM = @HAVE_DECL_GETDELIM@ +HAVE_DECL_GETDOMAINNAME = @HAVE_DECL_GETDOMAINNAME@ +HAVE_DECL_GETLINE = @HAVE_DECL_GETLINE@ +HAVE_DECL_GETLOADAVG = @HAVE_DECL_GETLOADAVG@ +HAVE_DECL_GETLOGIN = @HAVE_DECL_GETLOGIN@ +HAVE_DECL_GETLOGIN_R = @HAVE_DECL_GETLOGIN_R@ +HAVE_DECL_GETNAMEINFO = @HAVE_DECL_GETNAMEINFO@ +HAVE_DECL_GETPAGESIZE = @HAVE_DECL_GETPAGESIZE@ +HAVE_DECL_GETUSERSHELL = @HAVE_DECL_GETUSERSHELL@ +HAVE_DECL_IMAXABS = @HAVE_DECL_IMAXABS@ +HAVE_DECL_IMAXDIV = @HAVE_DECL_IMAXDIV@ +HAVE_DECL_INET_NTOP = @HAVE_DECL_INET_NTOP@ +HAVE_DECL_INET_PTON = @HAVE_DECL_INET_PTON@ +HAVE_DECL_INITSTATE = @HAVE_DECL_INITSTATE@ +HAVE_DECL_LOCALTIME_R = @HAVE_DECL_LOCALTIME_R@ +HAVE_DECL_MEMMEM = @HAVE_DECL_MEMMEM@ +HAVE_DECL_MEMRCHR = @HAVE_DECL_MEMRCHR@ +HAVE_DECL_OBSTACK_PRINTF = @HAVE_DECL_OBSTACK_PRINTF@ +HAVE_DECL_SETENV = @HAVE_DECL_SETENV@ +HAVE_DECL_SETHOSTNAME = @HAVE_DECL_SETHOSTNAME@ +HAVE_DECL_SETSTATE = @HAVE_DECL_SETSTATE@ +HAVE_DECL_SNPRINTF = @HAVE_DECL_SNPRINTF@ +HAVE_DECL_STRDUP = @HAVE_DECL_STRDUP@ +HAVE_DECL_STRERROR_R = @HAVE_DECL_STRERROR_R@ +HAVE_DECL_STRNCASECMP = @HAVE_DECL_STRNCASECMP@ +HAVE_DECL_STRNDUP = @HAVE_DECL_STRNDUP@ +HAVE_DECL_STRNLEN = @HAVE_DECL_STRNLEN@ +HAVE_DECL_STRSIGNAL = @HAVE_DECL_STRSIGNAL@ +HAVE_DECL_STRTOIMAX = @HAVE_DECL_STRTOIMAX@ +HAVE_DECL_STRTOK_R = @HAVE_DECL_STRTOK_R@ +HAVE_DECL_STRTOUMAX = @HAVE_DECL_STRTOUMAX@ +HAVE_DECL_TRUNCATE = @HAVE_DECL_TRUNCATE@ +HAVE_DECL_TTYNAME_R = @HAVE_DECL_TTYNAME_R@ +HAVE_DECL_UNSETENV = @HAVE_DECL_UNSETENV@ +HAVE_DECL_VSNPRINTF = @HAVE_DECL_VSNPRINTF@ +HAVE_DECL_WCSDUP = @HAVE_DECL_WCSDUP@ +HAVE_DECL_WCTOB = @HAVE_DECL_WCTOB@ +HAVE_DECL_WCWIDTH = @HAVE_DECL_WCWIDTH@ +HAVE_DIRENT_H = @HAVE_DIRENT_H@ +HAVE_DPRINTF = @HAVE_DPRINTF@ +HAVE_DUP3 = @HAVE_DUP3@ +HAVE_DUPLOCALE = @HAVE_DUPLOCALE@ +HAVE_EUIDACCESS = @HAVE_EUIDACCESS@ +HAVE_EXECVPE = @HAVE_EXECVPE@ +HAVE_EXPLICIT_BZERO = @HAVE_EXPLICIT_BZERO@ +HAVE_FACCESSAT = @HAVE_FACCESSAT@ +HAVE_FCHDIR = @HAVE_FCHDIR@ +HAVE_FCHMODAT = @HAVE_FCHMODAT@ +HAVE_FCHOWNAT = @HAVE_FCHOWNAT@ +HAVE_FCNTL = @HAVE_FCNTL@ +HAVE_FDATASYNC = @HAVE_FDATASYNC@ +HAVE_FDOPENDIR = @HAVE_FDOPENDIR@ +HAVE_FEATURES_H = @HAVE_FEATURES_H@ +HAVE_FFS = @HAVE_FFS@ +HAVE_FFSL = @HAVE_FFSL@ +HAVE_FFSLL = @HAVE_FFSLL@ +HAVE_FLOCK = @HAVE_FLOCK@ +HAVE_FNMATCH = @HAVE_FNMATCH@ +HAVE_FNMATCH_H = @HAVE_FNMATCH_H@ +HAVE_FREELOCALE = @HAVE_FREELOCALE@ +HAVE_FSEEKO = @HAVE_FSEEKO@ +HAVE_FSTATAT = @HAVE_FSTATAT@ +HAVE_FSYNC = @HAVE_FSYNC@ +HAVE_FTELLO = @HAVE_FTELLO@ +HAVE_FTRUNCATE = @HAVE_FTRUNCATE@ +HAVE_FUTIMENS = @HAVE_FUTIMENS@ +HAVE_GETDTABLESIZE = @HAVE_GETDTABLESIZE@ +HAVE_GETENTROPY = @HAVE_GETENTROPY@ +HAVE_GETGROUPS = @HAVE_GETGROUPS@ +HAVE_GETHOSTNAME = @HAVE_GETHOSTNAME@ +HAVE_GETLOGIN = @HAVE_GETLOGIN@ +HAVE_GETOPT_H = @HAVE_GETOPT_H@ +HAVE_GETPAGESIZE = @HAVE_GETPAGESIZE@ +HAVE_GETPASS = @HAVE_GETPASS@ +HAVE_GETRANDOM = @HAVE_GETRANDOM@ +HAVE_GETSUBOPT = @HAVE_GETSUBOPT@ +HAVE_GETTIMEOFDAY = @HAVE_GETTIMEOFDAY@ +HAVE_GETUMASK = @HAVE_GETUMASK@ +HAVE_GRANTPT = @HAVE_GRANTPT@ +HAVE_GROUP_MEMBER = @HAVE_GROUP_MEMBER@ +HAVE_IMAXDIV_T = @HAVE_IMAXDIV_T@ +HAVE_INITSTATE = @HAVE_INITSTATE@ +HAVE_INTTYPES_H = @HAVE_INTTYPES_H@ +HAVE_ISBLANK = @HAVE_ISBLANK@ +HAVE_ISWBLANK = @HAVE_ISWBLANK@ +HAVE_ISWCNTRL = @HAVE_ISWCNTRL@ +HAVE_LANGINFO_ALTMON = @HAVE_LANGINFO_ALTMON@ +HAVE_LANGINFO_CODESET = @HAVE_LANGINFO_CODESET@ +HAVE_LANGINFO_ERA = @HAVE_LANGINFO_ERA@ +HAVE_LANGINFO_H = @HAVE_LANGINFO_H@ +HAVE_LANGINFO_T_FMT_AMPM = @HAVE_LANGINFO_T_FMT_AMPM@ +HAVE_LANGINFO_YESEXPR = @HAVE_LANGINFO_YESEXPR@ +HAVE_LCHMOD = @HAVE_LCHMOD@ +HAVE_LCHOWN = @HAVE_LCHOWN@ +HAVE_LIBGNUTLS = @HAVE_LIBGNUTLS@ +HAVE_LIBSSL = @HAVE_LIBSSL@ +HAVE_LIBUNISTRING = @HAVE_LIBUNISTRING@ +HAVE_LINK = @HAVE_LINK@ +HAVE_LINKAT = @HAVE_LINKAT@ +HAVE_LSTAT = @HAVE_LSTAT@ +HAVE_MAX_ALIGN_T = @HAVE_MAX_ALIGN_T@ +HAVE_MBRLEN = @HAVE_MBRLEN@ +HAVE_MBRTOWC = @HAVE_MBRTOWC@ +HAVE_MBSINIT = @HAVE_MBSINIT@ +HAVE_MBSLEN = @HAVE_MBSLEN@ +HAVE_MBSNRTOWCS = @HAVE_MBSNRTOWCS@ +HAVE_MBSRTOWCS = @HAVE_MBSRTOWCS@ +HAVE_MBTOWC = @HAVE_MBTOWC@ +HAVE_MEMPCPY = @HAVE_MEMPCPY@ +HAVE_MKDIRAT = @HAVE_MKDIRAT@ +HAVE_MKDTEMP = @HAVE_MKDTEMP@ +HAVE_MKFIFO = @HAVE_MKFIFO@ +HAVE_MKFIFOAT = @HAVE_MKFIFOAT@ +HAVE_MKNOD = @HAVE_MKNOD@ +HAVE_MKNODAT = @HAVE_MKNODAT@ +HAVE_MKOSTEMP = @HAVE_MKOSTEMP@ +HAVE_MKOSTEMPS = @HAVE_MKOSTEMPS@ +HAVE_MKSTEMP = @HAVE_MKSTEMP@ +HAVE_MKSTEMPS = @HAVE_MKSTEMPS@ +HAVE_MSVC_INVALID_PARAMETER_HANDLER = @HAVE_MSVC_INVALID_PARAMETER_HANDLER@ +HAVE_NANOSLEEP = @HAVE_NANOSLEEP@ +HAVE_NETDB_H = @HAVE_NETDB_H@ +HAVE_NETINET_IN_H = @HAVE_NETINET_IN_H@ +HAVE_NEWLOCALE = @HAVE_NEWLOCALE@ +HAVE_NL_LANGINFO = @HAVE_NL_LANGINFO@ +HAVE_OPENAT = @HAVE_OPENAT@ +HAVE_OPENDIR = @HAVE_OPENDIR@ +HAVE_OS_H = @HAVE_OS_H@ +HAVE_PCLOSE = @HAVE_PCLOSE@ +HAVE_PIPE = @HAVE_PIPE@ +HAVE_PIPE2 = @HAVE_PIPE2@ +HAVE_POPEN = @HAVE_POPEN@ +HAVE_POSIX_MEMALIGN = @HAVE_POSIX_MEMALIGN@ +HAVE_POSIX_OPENPT = @HAVE_POSIX_OPENPT@ +HAVE_POSIX_SIGNALBLOCKING = @HAVE_POSIX_SIGNALBLOCKING@ +HAVE_POSIX_SPAWN = @HAVE_POSIX_SPAWN@ +HAVE_POSIX_SPAWNATTR_T = @HAVE_POSIX_SPAWNATTR_T@ +HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCHDIR = @HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCHDIR@ +HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDFCHDIR = @HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDFCHDIR@ +HAVE_POSIX_SPAWN_FILE_ACTIONS_T = @HAVE_POSIX_SPAWN_FILE_ACTIONS_T@ +HAVE_PREAD = @HAVE_PREAD@ +HAVE_PSELECT = @HAVE_PSELECT@ +HAVE_PTHREAD_SIGMASK = @HAVE_PTHREAD_SIGMASK@ +HAVE_PTSNAME = @HAVE_PTSNAME@ +HAVE_PTSNAME_R = @HAVE_PTSNAME_R@ +HAVE_PWRITE = @HAVE_PWRITE@ +HAVE_QSORT_R = @HAVE_QSORT_R@ +HAVE_RAISE = @HAVE_RAISE@ +HAVE_RANDOM = @HAVE_RANDOM@ +HAVE_RANDOM_H = @HAVE_RANDOM_H@ +HAVE_RANDOM_R = @HAVE_RANDOM_R@ +HAVE_RAWMEMCHR = @HAVE_RAWMEMCHR@ +HAVE_READDIR = @HAVE_READDIR@ +HAVE_READLINK = @HAVE_READLINK@ +HAVE_READLINKAT = @HAVE_READLINKAT@ +HAVE_REALLOCARRAY = @HAVE_REALLOCARRAY@ +HAVE_REALPATH = @HAVE_REALPATH@ +HAVE_RENAMEAT = @HAVE_RENAMEAT@ +HAVE_REWINDDIR = @HAVE_REWINDDIR@ +HAVE_RPMATCH = @HAVE_RPMATCH@ +HAVE_SA_FAMILY_T = @HAVE_SA_FAMILY_T@ +HAVE_SCANDIR = @HAVE_SCANDIR@ +HAVE_SCHED_H = @HAVE_SCHED_H@ +HAVE_SCHED_YIELD = @HAVE_SCHED_YIELD@ +HAVE_SECURE_GETENV = @HAVE_SECURE_GETENV@ +HAVE_SETENV = @HAVE_SETENV@ +HAVE_SETHOSTNAME = @HAVE_SETHOSTNAME@ +HAVE_SETSTATE = @HAVE_SETSTATE@ +HAVE_SIGABBREV_NP = @HAVE_SIGABBREV_NP@ +HAVE_SIGACTION = @HAVE_SIGACTION@ +HAVE_SIGDESCR_NP = @HAVE_SIGDESCR_NP@ +HAVE_SIGHANDLER_T = @HAVE_SIGHANDLER_T@ +HAVE_SIGINFO_T = @HAVE_SIGINFO_T@ +HAVE_SIGNED_SIG_ATOMIC_T = @HAVE_SIGNED_SIG_ATOMIC_T@ +HAVE_SIGNED_WCHAR_T = @HAVE_SIGNED_WCHAR_T@ +HAVE_SIGNED_WINT_T = @HAVE_SIGNED_WINT_T@ +HAVE_SIGSET_T = @HAVE_SIGSET_T@ +HAVE_SLEEP = @HAVE_SLEEP@ +HAVE_SPAWN_H = @HAVE_SPAWN_H@ +HAVE_STDINT_H = @HAVE_STDINT_H@ +HAVE_STPCPY = @HAVE_STPCPY@ +HAVE_STPNCPY = @HAVE_STPNCPY@ +HAVE_STRCASECMP = @HAVE_STRCASECMP@ +HAVE_STRCASESTR = @HAVE_STRCASESTR@ +HAVE_STRCHRNUL = @HAVE_STRCHRNUL@ +HAVE_STRERRORNAME_NP = @HAVE_STRERRORNAME_NP@ +HAVE_STRINGS_H = @HAVE_STRINGS_H@ +HAVE_STRPBRK = @HAVE_STRPBRK@ +HAVE_STRPTIME = @HAVE_STRPTIME@ +HAVE_STRSEP = @HAVE_STRSEP@ +HAVE_STRTOD = @HAVE_STRTOD@ +HAVE_STRTOLD = @HAVE_STRTOLD@ +HAVE_STRTOLL = @HAVE_STRTOLL@ +HAVE_STRTOULL = @HAVE_STRTOULL@ +HAVE_STRUCT_ADDRINFO = @HAVE_STRUCT_ADDRINFO@ +HAVE_STRUCT_RANDOM_DATA = @HAVE_STRUCT_RANDOM_DATA@ +HAVE_STRUCT_SCHED_PARAM = @HAVE_STRUCT_SCHED_PARAM@ +HAVE_STRUCT_SIGACTION_SA_SIGACTION = @HAVE_STRUCT_SIGACTION_SA_SIGACTION@ +HAVE_STRUCT_SOCKADDR_STORAGE = @HAVE_STRUCT_SOCKADDR_STORAGE@ +HAVE_STRUCT_SOCKADDR_STORAGE_SS_FAMILY = @HAVE_STRUCT_SOCKADDR_STORAGE_SS_FAMILY@ +HAVE_STRUCT_TIMEVAL = @HAVE_STRUCT_TIMEVAL@ +HAVE_STRVERSCMP = @HAVE_STRVERSCMP@ +HAVE_SYMLINK = @HAVE_SYMLINK@ +HAVE_SYMLINKAT = @HAVE_SYMLINKAT@ +HAVE_SYS_BITYPES_H = @HAVE_SYS_BITYPES_H@ +HAVE_SYS_CDEFS_H = @HAVE_SYS_CDEFS_H@ +HAVE_SYS_FILE_H = @HAVE_SYS_FILE_H@ +HAVE_SYS_INTTYPES_H = @HAVE_SYS_INTTYPES_H@ +HAVE_SYS_IOCTL_H = @HAVE_SYS_IOCTL_H@ +HAVE_SYS_LOADAVG_H = @HAVE_SYS_LOADAVG_H@ +HAVE_SYS_PARAM_H = @HAVE_SYS_PARAM_H@ +HAVE_SYS_RANDOM_H = @HAVE_SYS_RANDOM_H@ +HAVE_SYS_SELECT_H = @HAVE_SYS_SELECT_H@ +HAVE_SYS_SOCKET_H = @HAVE_SYS_SOCKET_H@ +HAVE_SYS_TIME_H = @HAVE_SYS_TIME_H@ +HAVE_SYS_TYPES_H = @HAVE_SYS_TYPES_H@ +HAVE_SYS_UIO_H = @HAVE_SYS_UIO_H@ +HAVE_TIMEGM = @HAVE_TIMEGM@ +HAVE_TIMEZONE_T = @HAVE_TIMEZONE_T@ +HAVE_TYPE_VOLATILE_SIG_ATOMIC_T = @HAVE_TYPE_VOLATILE_SIG_ATOMIC_T@ +HAVE_UNISTD_H = @HAVE_UNISTD_H@ +HAVE_UNLINKAT = @HAVE_UNLINKAT@ +HAVE_UNLOCKPT = @HAVE_UNLOCKPT@ +HAVE_USLEEP = @HAVE_USLEEP@ +HAVE_UTIME = @HAVE_UTIME@ +HAVE_UTIMENSAT = @HAVE_UTIMENSAT@ +HAVE_UTIME_H = @HAVE_UTIME_H@ +HAVE_VALGRIND = @HAVE_VALGRIND@ +HAVE_VASPRINTF = @HAVE_VASPRINTF@ +HAVE_VDPRINTF = @HAVE_VDPRINTF@ +HAVE_VISIBILITY = @HAVE_VISIBILITY@ +HAVE_WCHAR_H = @HAVE_WCHAR_H@ +HAVE_WCHAR_T = @HAVE_WCHAR_T@ +HAVE_WCPCPY = @HAVE_WCPCPY@ +HAVE_WCPNCPY = @HAVE_WCPNCPY@ +HAVE_WCRTOMB = @HAVE_WCRTOMB@ +HAVE_WCSCASECMP = @HAVE_WCSCASECMP@ +HAVE_WCSCAT = @HAVE_WCSCAT@ +HAVE_WCSCHR = @HAVE_WCSCHR@ +HAVE_WCSCMP = @HAVE_WCSCMP@ +HAVE_WCSCOLL = @HAVE_WCSCOLL@ +HAVE_WCSCPY = @HAVE_WCSCPY@ +HAVE_WCSCSPN = @HAVE_WCSCSPN@ +HAVE_WCSDUP = @HAVE_WCSDUP@ +HAVE_WCSFTIME = @HAVE_WCSFTIME@ +HAVE_WCSLEN = @HAVE_WCSLEN@ +HAVE_WCSNCASECMP = @HAVE_WCSNCASECMP@ +HAVE_WCSNCAT = @HAVE_WCSNCAT@ +HAVE_WCSNCMP = @HAVE_WCSNCMP@ +HAVE_WCSNCPY = @HAVE_WCSNCPY@ +HAVE_WCSNLEN = @HAVE_WCSNLEN@ +HAVE_WCSNRTOMBS = @HAVE_WCSNRTOMBS@ +HAVE_WCSPBRK = @HAVE_WCSPBRK@ +HAVE_WCSRCHR = @HAVE_WCSRCHR@ +HAVE_WCSRTOMBS = @HAVE_WCSRTOMBS@ +HAVE_WCSSPN = @HAVE_WCSSPN@ +HAVE_WCSSTR = @HAVE_WCSSTR@ +HAVE_WCSTOK = @HAVE_WCSTOK@ +HAVE_WCSWIDTH = @HAVE_WCSWIDTH@ +HAVE_WCSXFRM = @HAVE_WCSXFRM@ +HAVE_WCTRANS_T = @HAVE_WCTRANS_T@ +HAVE_WCTYPE_H = @HAVE_WCTYPE_H@ +HAVE_WCTYPE_T = @HAVE_WCTYPE_T@ +HAVE_WINSOCK2_H = @HAVE_WINSOCK2_H@ +HAVE_WINT_T = @HAVE_WINT_T@ +HAVE_WMEMCHR = @HAVE_WMEMCHR@ +HAVE_WMEMCMP = @HAVE_WMEMCMP@ +HAVE_WMEMCPY = @HAVE_WMEMCPY@ +HAVE_WMEMMOVE = @HAVE_WMEMMOVE@ +HAVE_WMEMPCPY = @HAVE_WMEMPCPY@ +HAVE_WMEMSET = @HAVE_WMEMSET@ +HAVE_WS2TCPIP_H = @HAVE_WS2TCPIP_H@ +HAVE_XLOCALE_H = @HAVE_XLOCALE_H@ +HAVE__BOOL = @HAVE__BOOL@ +HAVE__EXIT = @HAVE__EXIT@ +HOSTENT_LIB = @HOSTENT_LIB@ +ICONV_CONST = @ICONV_CONST@ +ICONV_H = @ICONV_H@ +INCLUDE_NEXT = @INCLUDE_NEXT@ +INCLUDE_NEXT_AS_FIRST_DIRECTIVE = @INCLUDE_NEXT_AS_FIRST_DIRECTIVE@ +INET_NTOP_LIB = @INET_NTOP_LIB@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +INT32_MAX_LT_INTMAX_MAX = @INT32_MAX_LT_INTMAX_MAX@ +INT64_MAX_EQ_LONG_MAX = @INT64_MAX_EQ_LONG_MAX@ +INTLLIBS = @INTLLIBS@ +INTL_MACOSX_LIBS = @INTL_MACOSX_LIBS@ +LDFLAGS = @LDFLAGS@ +LEX = @LEX@ +LEXLIB = @LEXLIB@ +LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@ +LIBGNUTLS = @LIBGNUTLS@ +LIBGNUTLS_PREFIX = @LIBGNUTLS_PREFIX@ +LIBGNU_LIBDEPS = @LIBGNU_LIBDEPS@ +LIBGNU_LTLIBDEPS = @LIBGNU_LTLIBDEPS@ +LIBICONV = @LIBICONV@ +LIBINTL = @LIBINTL@ +LIBMULTITHREAD = @LIBMULTITHREAD@ +LIBOBJS = @LIBOBJS@ +LIBPMULTITHREAD = @LIBPMULTITHREAD@ +LIBPSL_CFLAGS = @LIBPSL_CFLAGS@ +LIBPSL_LIBS = @LIBPSL_LIBS@ +LIBPTHREAD = @LIBPTHREAD@ +LIBS = @LIBS@ +LIBSOCKET = @LIBSOCKET@ +LIBSSL = @LIBSSL@ +LIBSSL_PREFIX = @LIBSSL_PREFIX@ +LIBSTDTHREAD = @LIBSTDTHREAD@ +LIBTHREAD = @LIBTHREAD@ +LIBUNISTRING = @LIBUNISTRING@ +LIBUNISTRING_PREFIX = @LIBUNISTRING_PREFIX@ +LIBUNISTRING_UNICASE_H = @LIBUNISTRING_UNICASE_H@ +LIBUNISTRING_UNICTYPE_H = @LIBUNISTRING_UNICTYPE_H@ +LIBUNISTRING_UNINORM_H = @LIBUNISTRING_UNINORM_H@ +LIBUNISTRING_UNISTR_H = @LIBUNISTRING_UNISTR_H@ +LIBUNISTRING_UNITYPES_H = @LIBUNISTRING_UNITYPES_H@ +LIBUNISTRING_UNIWIDTH_H = @LIBUNISTRING_UNIWIDTH_H@ +LIB_CLOCK_GETTIME = @LIB_CLOCK_GETTIME@ +LIB_CRYPTO = @LIB_CRYPTO@ +LIB_FUZZING_ENGINE = @LIB_FUZZING_ENGINE@ +LIB_GETRANDOM = @LIB_GETRANDOM@ +LIB_HARD_LOCALE = @LIB_HARD_LOCALE@ +LIB_MBRTOWC = @LIB_MBRTOWC@ +LIB_NANOSLEEP = @LIB_NANOSLEEP@ +LIB_NL_LANGINFO = @LIB_NL_LANGINFO@ +LIB_POSIX_SPAWN = @LIB_POSIX_SPAWN@ +LIB_SCHED_YIELD = @LIB_SCHED_YIELD@ +LIB_SELECT = @LIB_SELECT@ +LIB_SETLOCALE_NULL = @LIB_SETLOCALE_NULL@ +LIMITS_H = @LIMITS_H@ +LOCALCHARSET_TESTS_ENVIRONMENT = @LOCALCHARSET_TESTS_ENVIRONMENT@ +LOCALE_FR = @LOCALE_FR@ +LOCALE_FR_UTF8 = @LOCALE_FR_UTF8@ +LOCALE_JA = @LOCALE_JA@ +LOCALE_ZH_CN = @LOCALE_ZH_CN@ +LTLIBGNUTLS = @LTLIBGNUTLS@ +LTLIBICONV = @LTLIBICONV@ +LTLIBINTL = @LTLIBINTL@ +LTLIBMULTITHREAD = @LTLIBMULTITHREAD@ +LTLIBOBJS = @LTLIBOBJS@ +LTLIBSSL = @LTLIBSSL@ +LTLIBTHREAD = @LTLIBTHREAD@ +LTLIBUNISTRING = @LTLIBUNISTRING@ +MAKEINFO = @MAKEINFO@ +METALINK_CFLAGS = @METALINK_CFLAGS@ +METALINK_LIBS = @METALINK_LIBS@ +MKDIR_P = @MKDIR_P@ +MSGFMT = @MSGFMT@ +MSGFMT_015 = @MSGFMT_015@ +MSGMERGE = @MSGMERGE@ +NETINET_IN_H = @NETINET_IN_H@ +NETTLE_LIBS = @NETTLE_LIBS@ +NEXT_ARPA_INET_H = @NEXT_ARPA_INET_H@ +NEXT_AS_FIRST_DIRECTIVE_ARPA_INET_H = @NEXT_AS_FIRST_DIRECTIVE_ARPA_INET_H@ +NEXT_AS_FIRST_DIRECTIVE_CTYPE_H = @NEXT_AS_FIRST_DIRECTIVE_CTYPE_H@ +NEXT_AS_FIRST_DIRECTIVE_DIRENT_H = @NEXT_AS_FIRST_DIRECTIVE_DIRENT_H@ +NEXT_AS_FIRST_DIRECTIVE_ERRNO_H = @NEXT_AS_FIRST_DIRECTIVE_ERRNO_H@ +NEXT_AS_FIRST_DIRECTIVE_FCNTL_H = @NEXT_AS_FIRST_DIRECTIVE_FCNTL_H@ +NEXT_AS_FIRST_DIRECTIVE_FLOAT_H = @NEXT_AS_FIRST_DIRECTIVE_FLOAT_H@ +NEXT_AS_FIRST_DIRECTIVE_FNMATCH_H = @NEXT_AS_FIRST_DIRECTIVE_FNMATCH_H@ +NEXT_AS_FIRST_DIRECTIVE_GETOPT_H = @NEXT_AS_FIRST_DIRECTIVE_GETOPT_H@ +NEXT_AS_FIRST_DIRECTIVE_ICONV_H = @NEXT_AS_FIRST_DIRECTIVE_ICONV_H@ +NEXT_AS_FIRST_DIRECTIVE_INTTYPES_H = @NEXT_AS_FIRST_DIRECTIVE_INTTYPES_H@ +NEXT_AS_FIRST_DIRECTIVE_LANGINFO_H = @NEXT_AS_FIRST_DIRECTIVE_LANGINFO_H@ +NEXT_AS_FIRST_DIRECTIVE_LIMITS_H = @NEXT_AS_FIRST_DIRECTIVE_LIMITS_H@ +NEXT_AS_FIRST_DIRECTIVE_LOCALE_H = @NEXT_AS_FIRST_DIRECTIVE_LOCALE_H@ +NEXT_AS_FIRST_DIRECTIVE_NETDB_H = @NEXT_AS_FIRST_DIRECTIVE_NETDB_H@ +NEXT_AS_FIRST_DIRECTIVE_NETINET_IN_H = @NEXT_AS_FIRST_DIRECTIVE_NETINET_IN_H@ +NEXT_AS_FIRST_DIRECTIVE_SCHED_H = @NEXT_AS_FIRST_DIRECTIVE_SCHED_H@ +NEXT_AS_FIRST_DIRECTIVE_SIGNAL_H = @NEXT_AS_FIRST_DIRECTIVE_SIGNAL_H@ +NEXT_AS_FIRST_DIRECTIVE_SPAWN_H = @NEXT_AS_FIRST_DIRECTIVE_SPAWN_H@ +NEXT_AS_FIRST_DIRECTIVE_STDDEF_H = @NEXT_AS_FIRST_DIRECTIVE_STDDEF_H@ +NEXT_AS_FIRST_DIRECTIVE_STDINT_H = @NEXT_AS_FIRST_DIRECTIVE_STDINT_H@ +NEXT_AS_FIRST_DIRECTIVE_STDIO_H = @NEXT_AS_FIRST_DIRECTIVE_STDIO_H@ +NEXT_AS_FIRST_DIRECTIVE_STDLIB_H = @NEXT_AS_FIRST_DIRECTIVE_STDLIB_H@ +NEXT_AS_FIRST_DIRECTIVE_STRINGS_H = @NEXT_AS_FIRST_DIRECTIVE_STRINGS_H@ +NEXT_AS_FIRST_DIRECTIVE_STRING_H = @NEXT_AS_FIRST_DIRECTIVE_STRING_H@ +NEXT_AS_FIRST_DIRECTIVE_SYS_FILE_H = @NEXT_AS_FIRST_DIRECTIVE_SYS_FILE_H@ +NEXT_AS_FIRST_DIRECTIVE_SYS_IOCTL_H = @NEXT_AS_FIRST_DIRECTIVE_SYS_IOCTL_H@ +NEXT_AS_FIRST_DIRECTIVE_SYS_RANDOM_H = @NEXT_AS_FIRST_DIRECTIVE_SYS_RANDOM_H@ +NEXT_AS_FIRST_DIRECTIVE_SYS_SELECT_H = @NEXT_AS_FIRST_DIRECTIVE_SYS_SELECT_H@ +NEXT_AS_FIRST_DIRECTIVE_SYS_SOCKET_H = @NEXT_AS_FIRST_DIRECTIVE_SYS_SOCKET_H@ +NEXT_AS_FIRST_DIRECTIVE_SYS_STAT_H = @NEXT_AS_FIRST_DIRECTIVE_SYS_STAT_H@ +NEXT_AS_FIRST_DIRECTIVE_SYS_TIME_H = @NEXT_AS_FIRST_DIRECTIVE_SYS_TIME_H@ +NEXT_AS_FIRST_DIRECTIVE_SYS_TYPES_H = @NEXT_AS_FIRST_DIRECTIVE_SYS_TYPES_H@ +NEXT_AS_FIRST_DIRECTIVE_SYS_UIO_H = @NEXT_AS_FIRST_DIRECTIVE_SYS_UIO_H@ +NEXT_AS_FIRST_DIRECTIVE_SYS_WAIT_H = @NEXT_AS_FIRST_DIRECTIVE_SYS_WAIT_H@ +NEXT_AS_FIRST_DIRECTIVE_TIME_H = @NEXT_AS_FIRST_DIRECTIVE_TIME_H@ +NEXT_AS_FIRST_DIRECTIVE_UNISTD_H = @NEXT_AS_FIRST_DIRECTIVE_UNISTD_H@ +NEXT_AS_FIRST_DIRECTIVE_UTIME_H = @NEXT_AS_FIRST_DIRECTIVE_UTIME_H@ +NEXT_AS_FIRST_DIRECTIVE_WCHAR_H = @NEXT_AS_FIRST_DIRECTIVE_WCHAR_H@ +NEXT_AS_FIRST_DIRECTIVE_WCTYPE_H = @NEXT_AS_FIRST_DIRECTIVE_WCTYPE_H@ +NEXT_CTYPE_H = @NEXT_CTYPE_H@ +NEXT_DIRENT_H = @NEXT_DIRENT_H@ +NEXT_ERRNO_H = @NEXT_ERRNO_H@ +NEXT_FCNTL_H = @NEXT_FCNTL_H@ +NEXT_FLOAT_H = @NEXT_FLOAT_H@ +NEXT_FNMATCH_H = @NEXT_FNMATCH_H@ +NEXT_GETOPT_H = @NEXT_GETOPT_H@ +NEXT_ICONV_H = @NEXT_ICONV_H@ +NEXT_INTTYPES_H = @NEXT_INTTYPES_H@ +NEXT_LANGINFO_H = @NEXT_LANGINFO_H@ +NEXT_LIMITS_H = @NEXT_LIMITS_H@ +NEXT_LOCALE_H = @NEXT_LOCALE_H@ +NEXT_NETDB_H = @NEXT_NETDB_H@ +NEXT_NETINET_IN_H = @NEXT_NETINET_IN_H@ +NEXT_SCHED_H = @NEXT_SCHED_H@ +NEXT_SIGNAL_H = @NEXT_SIGNAL_H@ +NEXT_SPAWN_H = @NEXT_SPAWN_H@ +NEXT_STDDEF_H = @NEXT_STDDEF_H@ +NEXT_STDINT_H = @NEXT_STDINT_H@ +NEXT_STDIO_H = @NEXT_STDIO_H@ +NEXT_STDLIB_H = @NEXT_STDLIB_H@ +NEXT_STRINGS_H = @NEXT_STRINGS_H@ +NEXT_STRING_H = @NEXT_STRING_H@ +NEXT_SYS_FILE_H = @NEXT_SYS_FILE_H@ +NEXT_SYS_IOCTL_H = @NEXT_SYS_IOCTL_H@ +NEXT_SYS_RANDOM_H = @NEXT_SYS_RANDOM_H@ +NEXT_SYS_SELECT_H = @NEXT_SYS_SELECT_H@ +NEXT_SYS_SOCKET_H = @NEXT_SYS_SOCKET_H@ +NEXT_SYS_STAT_H = @NEXT_SYS_STAT_H@ +NEXT_SYS_TIME_H = @NEXT_SYS_TIME_H@ +NEXT_SYS_TYPES_H = @NEXT_SYS_TYPES_H@ +NEXT_SYS_UIO_H = @NEXT_SYS_UIO_H@ +NEXT_SYS_WAIT_H = @NEXT_SYS_WAIT_H@ +NEXT_TIME_H = @NEXT_TIME_H@ +NEXT_UNISTD_H = @NEXT_UNISTD_H@ +NEXT_UTIME_H = @NEXT_UTIME_H@ +NEXT_WCHAR_H = @NEXT_WCHAR_H@ +NEXT_WCTYPE_H = @NEXT_WCTYPE_H@ +OBJEXT = @OBJEXT@ +OPENSSL_CFLAGS = @OPENSSL_CFLAGS@ +OPENSSL_LIBS = @OPENSSL_LIBS@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PCRE2_CFLAGS = @PCRE2_CFLAGS@ +PCRE2_LIBS = @PCRE2_LIBS@ +PCRE_CFLAGS = @PCRE_CFLAGS@ +PCRE_LIBS = @PCRE_LIBS@ +PERL = @PERL@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +POD2MAN = @POD2MAN@ +POSUB = @POSUB@ +PRAGMA_COLUMNS = @PRAGMA_COLUMNS@ +PRAGMA_SYSTEM_HEADER = @PRAGMA_SYSTEM_HEADER@ +PRIPTR_PREFIX = @PRIPTR_PREFIX@ +PTHREAD_H_DEFINES_STRUCT_TIMESPEC = @PTHREAD_H_DEFINES_STRUCT_TIMESPEC@ +PTRDIFF_T_SUFFIX = @PTRDIFF_T_SUFFIX@ +PYTHON = @PYTHON@ +PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@ +PYTHON_PLATFORM = @PYTHON_PLATFORM@ +PYTHON_PREFIX = @PYTHON_PREFIX@ +PYTHON_VERSION = @PYTHON_VERSION@ +RANLIB = @RANLIB@ +REPLACE_ACCESS = @REPLACE_ACCESS@ +REPLACE_ALIGNED_ALLOC = @REPLACE_ALIGNED_ALLOC@ +REPLACE_BTOWC = @REPLACE_BTOWC@ +REPLACE_CALLOC = @REPLACE_CALLOC@ +REPLACE_CANONICALIZE_FILE_NAME = @REPLACE_CANONICALIZE_FILE_NAME@ +REPLACE_CHOWN = @REPLACE_CHOWN@ +REPLACE_CLOSE = @REPLACE_CLOSE@ +REPLACE_CLOSEDIR = @REPLACE_CLOSEDIR@ +REPLACE_CREAT = @REPLACE_CREAT@ +REPLACE_CTIME = @REPLACE_CTIME@ +REPLACE_DIRFD = @REPLACE_DIRFD@ +REPLACE_DPRINTF = @REPLACE_DPRINTF@ +REPLACE_DUP = @REPLACE_DUP@ +REPLACE_DUP2 = @REPLACE_DUP2@ +REPLACE_DUPLOCALE = @REPLACE_DUPLOCALE@ +REPLACE_EXECL = @REPLACE_EXECL@ +REPLACE_EXECLE = @REPLACE_EXECLE@ +REPLACE_EXECLP = @REPLACE_EXECLP@ +REPLACE_EXECV = @REPLACE_EXECV@ +REPLACE_EXECVE = @REPLACE_EXECVE@ +REPLACE_EXECVP = @REPLACE_EXECVP@ +REPLACE_EXECVPE = @REPLACE_EXECVPE@ +REPLACE_FACCESSAT = @REPLACE_FACCESSAT@ +REPLACE_FCHMODAT = @REPLACE_FCHMODAT@ +REPLACE_FCHOWNAT = @REPLACE_FCHOWNAT@ +REPLACE_FCLOSE = @REPLACE_FCLOSE@ +REPLACE_FCNTL = @REPLACE_FCNTL@ +REPLACE_FDOPEN = @REPLACE_FDOPEN@ +REPLACE_FDOPENDIR = @REPLACE_FDOPENDIR@ +REPLACE_FFLUSH = @REPLACE_FFLUSH@ +REPLACE_FNMATCH = @REPLACE_FNMATCH@ +REPLACE_FOPEN = @REPLACE_FOPEN@ +REPLACE_FPRINTF = @REPLACE_FPRINTF@ +REPLACE_FPURGE = @REPLACE_FPURGE@ +REPLACE_FREE = @REPLACE_FREE@ +REPLACE_FREELOCALE = @REPLACE_FREELOCALE@ +REPLACE_FREOPEN = @REPLACE_FREOPEN@ +REPLACE_FSEEK = @REPLACE_FSEEK@ +REPLACE_FSEEKO = @REPLACE_FSEEKO@ +REPLACE_FSTAT = @REPLACE_FSTAT@ +REPLACE_FSTATAT = @REPLACE_FSTATAT@ +REPLACE_FTELL = @REPLACE_FTELL@ +REPLACE_FTELLO = @REPLACE_FTELLO@ +REPLACE_FTRUNCATE = @REPLACE_FTRUNCATE@ +REPLACE_FUTIMENS = @REPLACE_FUTIMENS@ +REPLACE_GAI_STRERROR = @REPLACE_GAI_STRERROR@ +REPLACE_GETADDRINFO = @REPLACE_GETADDRINFO@ +REPLACE_GETCWD = @REPLACE_GETCWD@ +REPLACE_GETDELIM = @REPLACE_GETDELIM@ +REPLACE_GETDOMAINNAME = @REPLACE_GETDOMAINNAME@ +REPLACE_GETDTABLESIZE = @REPLACE_GETDTABLESIZE@ +REPLACE_GETGROUPS = @REPLACE_GETGROUPS@ +REPLACE_GETLINE = @REPLACE_GETLINE@ +REPLACE_GETLOGIN_R = @REPLACE_GETLOGIN_R@ +REPLACE_GETPAGESIZE = @REPLACE_GETPAGESIZE@ +REPLACE_GETPASS = @REPLACE_GETPASS@ +REPLACE_GETRANDOM = @REPLACE_GETRANDOM@ +REPLACE_GETTIMEOFDAY = @REPLACE_GETTIMEOFDAY@ +REPLACE_GMTIME = @REPLACE_GMTIME@ +REPLACE_ICONV = @REPLACE_ICONV@ +REPLACE_ICONV_OPEN = @REPLACE_ICONV_OPEN@ +REPLACE_ICONV_UTF = @REPLACE_ICONV_UTF@ +REPLACE_INET_NTOP = @REPLACE_INET_NTOP@ +REPLACE_INET_PTON = @REPLACE_INET_PTON@ +REPLACE_INITSTATE = @REPLACE_INITSTATE@ +REPLACE_IOCTL = @REPLACE_IOCTL@ +REPLACE_ISATTY = @REPLACE_ISATTY@ +REPLACE_ISWBLANK = @REPLACE_ISWBLANK@ +REPLACE_ISWCNTRL = @REPLACE_ISWCNTRL@ +REPLACE_ISWDIGIT = @REPLACE_ISWDIGIT@ +REPLACE_ISWXDIGIT = @REPLACE_ISWXDIGIT@ +REPLACE_ITOLD = @REPLACE_ITOLD@ +REPLACE_LCHOWN = @REPLACE_LCHOWN@ +REPLACE_LINK = @REPLACE_LINK@ +REPLACE_LINKAT = @REPLACE_LINKAT@ +REPLACE_LOCALECONV = @REPLACE_LOCALECONV@ +REPLACE_LOCALTIME = @REPLACE_LOCALTIME@ +REPLACE_LOCALTIME_R = @REPLACE_LOCALTIME_R@ +REPLACE_LSEEK = @REPLACE_LSEEK@ +REPLACE_LSTAT = @REPLACE_LSTAT@ +REPLACE_MALLOC = @REPLACE_MALLOC@ +REPLACE_MBRLEN = @REPLACE_MBRLEN@ +REPLACE_MBRTOWC = @REPLACE_MBRTOWC@ +REPLACE_MBSINIT = @REPLACE_MBSINIT@ +REPLACE_MBSNRTOWCS = @REPLACE_MBSNRTOWCS@ +REPLACE_MBSRTOWCS = @REPLACE_MBSRTOWCS@ +REPLACE_MBSTATE_T = @REPLACE_MBSTATE_T@ +REPLACE_MBTOWC = @REPLACE_MBTOWC@ +REPLACE_MEMCHR = @REPLACE_MEMCHR@ +REPLACE_MEMMEM = @REPLACE_MEMMEM@ +REPLACE_MKDIR = @REPLACE_MKDIR@ +REPLACE_MKFIFO = @REPLACE_MKFIFO@ +REPLACE_MKNOD = @REPLACE_MKNOD@ +REPLACE_MKSTEMP = @REPLACE_MKSTEMP@ +REPLACE_MKTIME = @REPLACE_MKTIME@ +REPLACE_NANOSLEEP = @REPLACE_NANOSLEEP@ +REPLACE_NEWLOCALE = @REPLACE_NEWLOCALE@ +REPLACE_NL_LANGINFO = @REPLACE_NL_LANGINFO@ +REPLACE_NULL = @REPLACE_NULL@ +REPLACE_OBSTACK_PRINTF = @REPLACE_OBSTACK_PRINTF@ +REPLACE_OPEN = @REPLACE_OPEN@ +REPLACE_OPENAT = @REPLACE_OPENAT@ +REPLACE_OPENDIR = @REPLACE_OPENDIR@ +REPLACE_PERROR = @REPLACE_PERROR@ +REPLACE_POPEN = @REPLACE_POPEN@ +REPLACE_POSIX_MEMALIGN = @REPLACE_POSIX_MEMALIGN@ +REPLACE_POSIX_SPAWN = @REPLACE_POSIX_SPAWN@ +REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCHDIR = @REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCHDIR@ +REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE = @REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE@ +REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDDUP2 = @REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDDUP2@ +REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDFCHDIR = @REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDFCHDIR@ +REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDOPEN = @REPLACE_POSIX_SPAWN_FILE_ACTIONS_ADDOPEN@ +REPLACE_PREAD = @REPLACE_PREAD@ +REPLACE_PRINTF = @REPLACE_PRINTF@ +REPLACE_PSELECT = @REPLACE_PSELECT@ +REPLACE_PTHREAD_SIGMASK = @REPLACE_PTHREAD_SIGMASK@ +REPLACE_PTSNAME = @REPLACE_PTSNAME@ +REPLACE_PTSNAME_R = @REPLACE_PTSNAME_R@ +REPLACE_PUTENV = @REPLACE_PUTENV@ +REPLACE_PWRITE = @REPLACE_PWRITE@ +REPLACE_QSORT_R = @REPLACE_QSORT_R@ +REPLACE_RAISE = @REPLACE_RAISE@ +REPLACE_RANDOM = @REPLACE_RANDOM@ +REPLACE_RANDOM_R = @REPLACE_RANDOM_R@ +REPLACE_READ = @REPLACE_READ@ +REPLACE_READLINK = @REPLACE_READLINK@ +REPLACE_READLINKAT = @REPLACE_READLINKAT@ +REPLACE_REALLOC = @REPLACE_REALLOC@ +REPLACE_REALPATH = @REPLACE_REALPATH@ +REPLACE_REMOVE = @REPLACE_REMOVE@ +REPLACE_RENAME = @REPLACE_RENAME@ +REPLACE_RENAMEAT = @REPLACE_RENAMEAT@ +REPLACE_RMDIR = @REPLACE_RMDIR@ +REPLACE_SCHED_YIELD = @REPLACE_SCHED_YIELD@ +REPLACE_SELECT = @REPLACE_SELECT@ +REPLACE_SETENV = @REPLACE_SETENV@ +REPLACE_SETLOCALE = @REPLACE_SETLOCALE@ +REPLACE_SETSTATE = @REPLACE_SETSTATE@ +REPLACE_SLEEP = @REPLACE_SLEEP@ +REPLACE_SNPRINTF = @REPLACE_SNPRINTF@ +REPLACE_SPRINTF = @REPLACE_SPRINTF@ +REPLACE_STAT = @REPLACE_STAT@ +REPLACE_STDIO_READ_FUNCS = @REPLACE_STDIO_READ_FUNCS@ +REPLACE_STDIO_WRITE_FUNCS = @REPLACE_STDIO_WRITE_FUNCS@ +REPLACE_STPNCPY = @REPLACE_STPNCPY@ +REPLACE_STRCASESTR = @REPLACE_STRCASESTR@ +REPLACE_STRCHRNUL = @REPLACE_STRCHRNUL@ +REPLACE_STRDUP = @REPLACE_STRDUP@ +REPLACE_STRERROR = @REPLACE_STRERROR@ +REPLACE_STRERRORNAME_NP = @REPLACE_STRERRORNAME_NP@ +REPLACE_STRERROR_R = @REPLACE_STRERROR_R@ +REPLACE_STRFTIME = @REPLACE_STRFTIME@ +REPLACE_STRNCAT = @REPLACE_STRNCAT@ +REPLACE_STRNDUP = @REPLACE_STRNDUP@ +REPLACE_STRNLEN = @REPLACE_STRNLEN@ +REPLACE_STRSIGNAL = @REPLACE_STRSIGNAL@ +REPLACE_STRSTR = @REPLACE_STRSTR@ +REPLACE_STRTOD = @REPLACE_STRTOD@ +REPLACE_STRTOIMAX = @REPLACE_STRTOIMAX@ +REPLACE_STRTOK_R = @REPLACE_STRTOK_R@ +REPLACE_STRTOLD = @REPLACE_STRTOLD@ +REPLACE_STRTOUMAX = @REPLACE_STRTOUMAX@ +REPLACE_STRUCT_LCONV = @REPLACE_STRUCT_LCONV@ +REPLACE_STRUCT_TIMEVAL = @REPLACE_STRUCT_TIMEVAL@ +REPLACE_SYMLINK = @REPLACE_SYMLINK@ +REPLACE_SYMLINKAT = @REPLACE_SYMLINKAT@ +REPLACE_TIMEGM = @REPLACE_TIMEGM@ +REPLACE_TMPFILE = @REPLACE_TMPFILE@ +REPLACE_TOWLOWER = @REPLACE_TOWLOWER@ +REPLACE_TRUNCATE = @REPLACE_TRUNCATE@ +REPLACE_TTYNAME_R = @REPLACE_TTYNAME_R@ +REPLACE_TZSET = @REPLACE_TZSET@ +REPLACE_UNLINK = @REPLACE_UNLINK@ +REPLACE_UNLINKAT = @REPLACE_UNLINKAT@ +REPLACE_UNSETENV = @REPLACE_UNSETENV@ +REPLACE_USLEEP = @REPLACE_USLEEP@ +REPLACE_UTIME = @REPLACE_UTIME@ +REPLACE_UTIMENSAT = @REPLACE_UTIMENSAT@ +REPLACE_VASPRINTF = @REPLACE_VASPRINTF@ +REPLACE_VDPRINTF = @REPLACE_VDPRINTF@ +REPLACE_VFPRINTF = @REPLACE_VFPRINTF@ +REPLACE_VPRINTF = @REPLACE_VPRINTF@ +REPLACE_VSNPRINTF = @REPLACE_VSNPRINTF@ +REPLACE_VSPRINTF = @REPLACE_VSPRINTF@ +REPLACE_WCRTOMB = @REPLACE_WCRTOMB@ +REPLACE_WCSFTIME = @REPLACE_WCSFTIME@ +REPLACE_WCSNRTOMBS = @REPLACE_WCSNRTOMBS@ +REPLACE_WCSRTOMBS = @REPLACE_WCSRTOMBS@ +REPLACE_WCSTOK = @REPLACE_WCSTOK@ +REPLACE_WCSWIDTH = @REPLACE_WCSWIDTH@ +REPLACE_WCTOB = @REPLACE_WCTOB@ +REPLACE_WCTOMB = @REPLACE_WCTOMB@ +REPLACE_WCWIDTH = @REPLACE_WCWIDTH@ +REPLACE_WRITE = @REPLACE_WRITE@ +SED = @SED@ +SERVENT_LIB = @SERVENT_LIB@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SIG_ATOMIC_T_SUFFIX = @SIG_ATOMIC_T_SUFFIX@ +SIZE_T_SUFFIX = @SIZE_T_SUFFIX@ +STDALIGN_H = @STDALIGN_H@ +STDBOOL_H = @STDBOOL_H@ +STDDEF_H = @STDDEF_H@ +STDINT_H = @STDINT_H@ +STRIP = @STRIP@ +SYS_IOCTL_H_HAVE_WINSOCK2_H = @SYS_IOCTL_H_HAVE_WINSOCK2_H@ +SYS_IOCTL_H_HAVE_WINSOCK2_H_AND_USE_SOCKETS = @SYS_IOCTL_H_HAVE_WINSOCK2_H_AND_USE_SOCKETS@ +SYS_TIME_H_DEFINES_STRUCT_TIMESPEC = @SYS_TIME_H_DEFINES_STRUCT_TIMESPEC@ +TIME_H_DEFINES_STRUCT_TIMESPEC = @TIME_H_DEFINES_STRUCT_TIMESPEC@ +UINT32_MAX_LT_UINTMAX_MAX = @UINT32_MAX_LT_UINTMAX_MAX@ +UINT64_MAX_EQ_ULONG_MAX = @UINT64_MAX_EQ_ULONG_MAX@ +UNDEFINE_STRTOK_R = @UNDEFINE_STRTOK_R@ +UNISTD_H_DEFINES_STRUCT_TIMESPEC = @UNISTD_H_DEFINES_STRUCT_TIMESPEC@ +UNISTD_H_HAVE_SYS_RANDOM_H = @UNISTD_H_HAVE_SYS_RANDOM_H@ +UNISTD_H_HAVE_WINSOCK2_H = @UNISTD_H_HAVE_WINSOCK2_H@ +UNISTD_H_HAVE_WINSOCK2_H_AND_USE_SOCKETS = @UNISTD_H_HAVE_WINSOCK2_H_AND_USE_SOCKETS@ +USE_NLS = @USE_NLS@ +UUID_CFLAGS = @UUID_CFLAGS@ +UUID_LIBS = @UUID_LIBS@ +VALGRIND_TESTS = @VALGRIND_TESTS@ +VERSION = @VERSION@ +WARN_CFLAGS = @WARN_CFLAGS@ +WCHAR_T_SUFFIX = @WCHAR_T_SUFFIX@ +WINDOWS_64_BIT_OFF_T = @WINDOWS_64_BIT_OFF_T@ +WINDOWS_64_BIT_ST_SIZE = @WINDOWS_64_BIT_ST_SIZE@ +WINDOWS_STAT_INODES = @WINDOWS_STAT_INODES@ +WINDOWS_STAT_TIMESPEC = @WINDOWS_STAT_TIMESPEC@ +WINT_T_SUFFIX = @WINT_T_SUFFIX@ +XGETTEXT = @XGETTEXT@ +XGETTEXT_015 = @XGETTEXT_015@ +XGETTEXT_EXTRA_OPTIONS = @XGETTEXT_EXTRA_OPTIONS@ +ZLIB_CFLAGS = @ZLIB_CFLAGS@ +ZLIB_LIBS = @ZLIB_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_CC = @ac_ct_CC@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +gl_LIBOBJS = @gl_LIBOBJS@ +gl_LTLIBOBJS = @gl_LTLIBOBJS@ +gltests_LIBOBJS = @gltests_LIBOBJS@ +gltests_LTLIBOBJS = @gltests_LTLIBOBJS@ +gltests_WITNESS = @gltests_WITNESS@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +pkgpyexecdir = @pkgpyexecdir@ +pkgpythondir = @pkgpythondir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +pyexecdir = @pyexecdir@ +pythondir = @pythondir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +sysconfdir = @sysconfdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +@IRI_IS_ENABLED_TRUE@IRI_OBJ = iri.c +@METALINK_IS_ENABLED_TRUE@METALINK_OBJ = metalink.c +@WITH_XATTR_TRUE@XATTR_OBJ = xattr.c +EXTRA_DIST = css.l css.c css_.c build_info.c.in build_info.c +wget_SOURCES = connect.c convert.c cookies.c ftp.c \ + css_.c css-url.c \ + ftp-basic.c ftp-ls.c hash.c host.c hsts.c html-parse.c html-url.c \ + http.c init.c log.c main.c netrc.c progress.c ptimer.c \ + recur.c res.c retr.c spider.c url.c warc.c $(XATTR_OBJ) \ + utils.c exits.c build_info.c $(IRI_OBJ) $(METALINK_OBJ) \ + css-url.h css-tokens.h connect.h convert.h cookies.h \ + ftp.h hash.h host.h hsts.h html-parse.h html-url.h \ + http.h http-ntlm.h init.h log.h mswindows.h netrc.h \ + options.h progress.h ptimer.h recur.h res.h retr.h \ + spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h \ + exits.h version.h metalink.h xattr.h + +nodist_wget_SOURCES = version.c +EXTRA_wget_SOURCES = iri.c +LDADD = $(CODE_COVERAGE_LIBS) $(LIBOBJS) ../lib/libgnu.a $(GETADDRINFO_LIB) $(HOSTENT_LIB)\ + $(INET_NTOP_LIB) $(LIBSOCKET) $(LIB_CLOCK_GETTIME) $(LIB_CRYPTO)\ + $(LIB_NANOSLEEP) $(LIB_POSIX_SPAWN) $(LIB_SELECT) $(LIBICONV) $(LIBINTL)\ + $(LIBTHREAD) $(LIBUNISTRING) $(SERVENT_LIB) + +AM_CPPFLAGS = -I$(top_builddir)/lib -I$(top_srcdir)/lib $(CODE_COVERAGE_CPPFLAGS) +AM_CFLAGS = $(WERROR_CFLAGS) $(WARN_CFLAGS) $(CODE_COVERAGE_CFLAGS) +ESCAPEQUOTE = sed -e 's/[\\"]/\\&/g' -e 's/\\"/"/' -e 's/\\";$$/";/' +check_LIBRARIES = libunittest.a +libunittest_a_SOURCES = $(wget_SOURCES) build_info.c +nodist_libunittest_a_SOURCES = version.c +libunittest_a_CPPFLAGS = -DTESTING "-I$(top_builddir)/lib" "-I$(top_srcdir)/lib" $(CODE_COVERAGE_CPPLAGS) +libunittest_a_LIBADD = $(LIBOBJS) +CLEANFILES = *~ *.bak core core.[0-9]* build_info.c version.c +all: config.h + $(MAKE) $(AM_MAKEFLAGS) all-am + +.SUFFIXES: +.SUFFIXES: .c .o .obj +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu src/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --gnu src/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +config.h: stamp-h1 + @test -f $@ || rm -f stamp-h1 + @test -f $@ || $(MAKE) $(AM_MAKEFLAGS) stamp-h1 + +stamp-h1: $(srcdir)/config.h.in $(top_builddir)/config.status + @rm -f stamp-h1 + cd $(top_builddir) && $(SHELL) ./config.status src/config.h +$(srcdir)/config.h.in: $(am__configure_deps) + ($(am__cd) $(top_srcdir) && $(AUTOHEADER)) + rm -f stamp-h1 + touch $@ + +distclean-hdr: + -rm -f config.h stamp-h1 +install-binPROGRAMS: $(bin_PROGRAMS) + @$(NORMAL_INSTALL) + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(bindir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(bindir)" || exit 1; \ + fi; \ + for p in $$list; do echo "$$p $$p"; done | \ + sed 's/$(EXEEXT)$$//' | \ + while read p p1; do if test -f $$p \ + ; then echo "$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n;h' \ + -e 's|.*|.|' \ + -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ + sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) files[d] = files[d] " " $$1; \ + else { print "f", $$3 "/" $$4, $$1; } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_PROGRAM_ENV) $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(bindir)$$dir'"; \ + $(INSTALL_PROGRAM_ENV) $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-binPROGRAMS: + @$(NORMAL_UNINSTALL) + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ + -e 's/$$/$(EXEEXT)/' \ + `; \ + test -n "$$list" || exit 0; \ + echo " ( cd '$(DESTDIR)$(bindir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(bindir)" && rm -f $$files + +clean-binPROGRAMS: + -test -z "$(bin_PROGRAMS)" || rm -f $(bin_PROGRAMS) + +clean-checkLIBRARIES: + -test -z "$(check_LIBRARIES)" || rm -f $(check_LIBRARIES) + +libunittest.a: $(libunittest_a_OBJECTS) $(libunittest_a_DEPENDENCIES) $(EXTRA_libunittest_a_DEPENDENCIES) + $(AM_V_at)-rm -f libunittest.a + $(AM_V_AR)$(libunittest_a_AR) libunittest.a $(libunittest_a_OBJECTS) $(libunittest_a_LIBADD) + $(AM_V_at)$(RANLIB) libunittest.a + +wget$(EXEEXT): $(wget_OBJECTS) $(wget_DEPENDENCIES) $(EXTRA_wget_DEPENDENCIES) + @rm -f wget$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(wget_OBJECTS) $(wget_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/ftp-opie.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/gnutls.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/http-ntlm.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/mswindows.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/openssl.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/build_info.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/connect.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/convert.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cookies.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/css-url.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/css_.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/exits.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ftp-basic.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ftp-ls.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ftp.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hash.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/host.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hsts.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/html-parse.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/html-url.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/init.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/iri.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-build_info.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-connect.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-convert.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-cookies.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-css-url.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-css_.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-exits.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-ftp-basic.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-ftp-ls.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-ftp.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-hash.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-host.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-hsts.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-html-parse.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-html-url.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-http.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-init.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-iri.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-log.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-main.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-metalink.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-netrc.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-progress.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-ptimer.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-recur.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-res.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-retr.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-spider.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-url.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-utils.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-version.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-warc.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libunittest_a-xattr.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/log.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/main.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/metalink.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/netrc.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/progress.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ptimer.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/recur.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/res.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/retr.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spider.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/url.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/utils.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/version.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/warc.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xattr.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.o$$||'`;\ +@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ +@am__fastdepCC_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.obj$$||'`;\ +@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ `$(CYGPATH_W) '$<'` &&\ +@am__fastdepCC_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +libunittest_a-connect.o: connect.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-connect.o -MD -MP -MF $(DEPDIR)/libunittest_a-connect.Tpo -c -o libunittest_a-connect.o `test -f 'connect.c' || echo '$(srcdir)/'`connect.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-connect.Tpo $(DEPDIR)/libunittest_a-connect.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='connect.c' object='libunittest_a-connect.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-connect.o `test -f 'connect.c' || echo '$(srcdir)/'`connect.c + +libunittest_a-connect.obj: connect.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-connect.obj -MD -MP -MF $(DEPDIR)/libunittest_a-connect.Tpo -c -o libunittest_a-connect.obj `if test -f 'connect.c'; then $(CYGPATH_W) 'connect.c'; else $(CYGPATH_W) '$(srcdir)/connect.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-connect.Tpo $(DEPDIR)/libunittest_a-connect.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='connect.c' object='libunittest_a-connect.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-connect.obj `if test -f 'connect.c'; then $(CYGPATH_W) 'connect.c'; else $(CYGPATH_W) '$(srcdir)/connect.c'; fi` + +libunittest_a-convert.o: convert.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-convert.o -MD -MP -MF $(DEPDIR)/libunittest_a-convert.Tpo -c -o libunittest_a-convert.o `test -f 'convert.c' || echo '$(srcdir)/'`convert.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-convert.Tpo $(DEPDIR)/libunittest_a-convert.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='convert.c' object='libunittest_a-convert.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-convert.o `test -f 'convert.c' || echo '$(srcdir)/'`convert.c + +libunittest_a-convert.obj: convert.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-convert.obj -MD -MP -MF $(DEPDIR)/libunittest_a-convert.Tpo -c -o libunittest_a-convert.obj `if test -f 'convert.c'; then $(CYGPATH_W) 'convert.c'; else $(CYGPATH_W) '$(srcdir)/convert.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-convert.Tpo $(DEPDIR)/libunittest_a-convert.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='convert.c' object='libunittest_a-convert.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-convert.obj `if test -f 'convert.c'; then $(CYGPATH_W) 'convert.c'; else $(CYGPATH_W) '$(srcdir)/convert.c'; fi` + +libunittest_a-cookies.o: cookies.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-cookies.o -MD -MP -MF $(DEPDIR)/libunittest_a-cookies.Tpo -c -o libunittest_a-cookies.o `test -f 'cookies.c' || echo '$(srcdir)/'`cookies.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-cookies.Tpo $(DEPDIR)/libunittest_a-cookies.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='cookies.c' object='libunittest_a-cookies.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-cookies.o `test -f 'cookies.c' || echo '$(srcdir)/'`cookies.c + +libunittest_a-cookies.obj: cookies.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-cookies.obj -MD -MP -MF $(DEPDIR)/libunittest_a-cookies.Tpo -c -o libunittest_a-cookies.obj `if test -f 'cookies.c'; then $(CYGPATH_W) 'cookies.c'; else $(CYGPATH_W) '$(srcdir)/cookies.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-cookies.Tpo $(DEPDIR)/libunittest_a-cookies.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='cookies.c' object='libunittest_a-cookies.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-cookies.obj `if test -f 'cookies.c'; then $(CYGPATH_W) 'cookies.c'; else $(CYGPATH_W) '$(srcdir)/cookies.c'; fi` + +libunittest_a-ftp.o: ftp.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-ftp.o -MD -MP -MF $(DEPDIR)/libunittest_a-ftp.Tpo -c -o libunittest_a-ftp.o `test -f 'ftp.c' || echo '$(srcdir)/'`ftp.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-ftp.Tpo $(DEPDIR)/libunittest_a-ftp.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='ftp.c' object='libunittest_a-ftp.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-ftp.o `test -f 'ftp.c' || echo '$(srcdir)/'`ftp.c + +libunittest_a-ftp.obj: ftp.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-ftp.obj -MD -MP -MF $(DEPDIR)/libunittest_a-ftp.Tpo -c -o libunittest_a-ftp.obj `if test -f 'ftp.c'; then $(CYGPATH_W) 'ftp.c'; else $(CYGPATH_W) '$(srcdir)/ftp.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-ftp.Tpo $(DEPDIR)/libunittest_a-ftp.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='ftp.c' object='libunittest_a-ftp.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-ftp.obj `if test -f 'ftp.c'; then $(CYGPATH_W) 'ftp.c'; else $(CYGPATH_W) '$(srcdir)/ftp.c'; fi` + +libunittest_a-css_.o: css_.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-css_.o -MD -MP -MF $(DEPDIR)/libunittest_a-css_.Tpo -c -o libunittest_a-css_.o `test -f 'css_.c' || echo '$(srcdir)/'`css_.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-css_.Tpo $(DEPDIR)/libunittest_a-css_.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='css_.c' object='libunittest_a-css_.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-css_.o `test -f 'css_.c' || echo '$(srcdir)/'`css_.c + +libunittest_a-css_.obj: css_.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-css_.obj -MD -MP -MF $(DEPDIR)/libunittest_a-css_.Tpo -c -o libunittest_a-css_.obj `if test -f 'css_.c'; then $(CYGPATH_W) 'css_.c'; else $(CYGPATH_W) '$(srcdir)/css_.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-css_.Tpo $(DEPDIR)/libunittest_a-css_.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='css_.c' object='libunittest_a-css_.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-css_.obj `if test -f 'css_.c'; then $(CYGPATH_W) 'css_.c'; else $(CYGPATH_W) '$(srcdir)/css_.c'; fi` + +libunittest_a-css-url.o: css-url.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-css-url.o -MD -MP -MF $(DEPDIR)/libunittest_a-css-url.Tpo -c -o libunittest_a-css-url.o `test -f 'css-url.c' || echo '$(srcdir)/'`css-url.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-css-url.Tpo $(DEPDIR)/libunittest_a-css-url.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='css-url.c' object='libunittest_a-css-url.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-css-url.o `test -f 'css-url.c' || echo '$(srcdir)/'`css-url.c + +libunittest_a-css-url.obj: css-url.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-css-url.obj -MD -MP -MF $(DEPDIR)/libunittest_a-css-url.Tpo -c -o libunittest_a-css-url.obj `if test -f 'css-url.c'; then $(CYGPATH_W) 'css-url.c'; else $(CYGPATH_W) '$(srcdir)/css-url.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-css-url.Tpo $(DEPDIR)/libunittest_a-css-url.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='css-url.c' object='libunittest_a-css-url.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-css-url.obj `if test -f 'css-url.c'; then $(CYGPATH_W) 'css-url.c'; else $(CYGPATH_W) '$(srcdir)/css-url.c'; fi` + +libunittest_a-ftp-basic.o: ftp-basic.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-ftp-basic.o -MD -MP -MF $(DEPDIR)/libunittest_a-ftp-basic.Tpo -c -o libunittest_a-ftp-basic.o `test -f 'ftp-basic.c' || echo '$(srcdir)/'`ftp-basic.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-ftp-basic.Tpo $(DEPDIR)/libunittest_a-ftp-basic.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='ftp-basic.c' object='libunittest_a-ftp-basic.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-ftp-basic.o `test -f 'ftp-basic.c' || echo '$(srcdir)/'`ftp-basic.c + +libunittest_a-ftp-basic.obj: ftp-basic.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-ftp-basic.obj -MD -MP -MF $(DEPDIR)/libunittest_a-ftp-basic.Tpo -c -o libunittest_a-ftp-basic.obj `if test -f 'ftp-basic.c'; then $(CYGPATH_W) 'ftp-basic.c'; else $(CYGPATH_W) '$(srcdir)/ftp-basic.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-ftp-basic.Tpo $(DEPDIR)/libunittest_a-ftp-basic.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='ftp-basic.c' object='libunittest_a-ftp-basic.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-ftp-basic.obj `if test -f 'ftp-basic.c'; then $(CYGPATH_W) 'ftp-basic.c'; else $(CYGPATH_W) '$(srcdir)/ftp-basic.c'; fi` + +libunittest_a-ftp-ls.o: ftp-ls.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-ftp-ls.o -MD -MP -MF $(DEPDIR)/libunittest_a-ftp-ls.Tpo -c -o libunittest_a-ftp-ls.o `test -f 'ftp-ls.c' || echo '$(srcdir)/'`ftp-ls.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-ftp-ls.Tpo $(DEPDIR)/libunittest_a-ftp-ls.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='ftp-ls.c' object='libunittest_a-ftp-ls.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-ftp-ls.o `test -f 'ftp-ls.c' || echo '$(srcdir)/'`ftp-ls.c + +libunittest_a-ftp-ls.obj: ftp-ls.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-ftp-ls.obj -MD -MP -MF $(DEPDIR)/libunittest_a-ftp-ls.Tpo -c -o libunittest_a-ftp-ls.obj `if test -f 'ftp-ls.c'; then $(CYGPATH_W) 'ftp-ls.c'; else $(CYGPATH_W) '$(srcdir)/ftp-ls.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-ftp-ls.Tpo $(DEPDIR)/libunittest_a-ftp-ls.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='ftp-ls.c' object='libunittest_a-ftp-ls.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-ftp-ls.obj `if test -f 'ftp-ls.c'; then $(CYGPATH_W) 'ftp-ls.c'; else $(CYGPATH_W) '$(srcdir)/ftp-ls.c'; fi` + +libunittest_a-hash.o: hash.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-hash.o -MD -MP -MF $(DEPDIR)/libunittest_a-hash.Tpo -c -o libunittest_a-hash.o `test -f 'hash.c' || echo '$(srcdir)/'`hash.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-hash.Tpo $(DEPDIR)/libunittest_a-hash.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='hash.c' object='libunittest_a-hash.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-hash.o `test -f 'hash.c' || echo '$(srcdir)/'`hash.c + +libunittest_a-hash.obj: hash.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-hash.obj -MD -MP -MF $(DEPDIR)/libunittest_a-hash.Tpo -c -o libunittest_a-hash.obj `if test -f 'hash.c'; then $(CYGPATH_W) 'hash.c'; else $(CYGPATH_W) '$(srcdir)/hash.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-hash.Tpo $(DEPDIR)/libunittest_a-hash.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='hash.c' object='libunittest_a-hash.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-hash.obj `if test -f 'hash.c'; then $(CYGPATH_W) 'hash.c'; else $(CYGPATH_W) '$(srcdir)/hash.c'; fi` + +libunittest_a-host.o: host.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-host.o -MD -MP -MF $(DEPDIR)/libunittest_a-host.Tpo -c -o libunittest_a-host.o `test -f 'host.c' || echo '$(srcdir)/'`host.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-host.Tpo $(DEPDIR)/libunittest_a-host.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='host.c' object='libunittest_a-host.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-host.o `test -f 'host.c' || echo '$(srcdir)/'`host.c + +libunittest_a-host.obj: host.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-host.obj -MD -MP -MF $(DEPDIR)/libunittest_a-host.Tpo -c -o libunittest_a-host.obj `if test -f 'host.c'; then $(CYGPATH_W) 'host.c'; else $(CYGPATH_W) '$(srcdir)/host.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-host.Tpo $(DEPDIR)/libunittest_a-host.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='host.c' object='libunittest_a-host.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-host.obj `if test -f 'host.c'; then $(CYGPATH_W) 'host.c'; else $(CYGPATH_W) '$(srcdir)/host.c'; fi` + +libunittest_a-hsts.o: hsts.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-hsts.o -MD -MP -MF $(DEPDIR)/libunittest_a-hsts.Tpo -c -o libunittest_a-hsts.o `test -f 'hsts.c' || echo '$(srcdir)/'`hsts.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-hsts.Tpo $(DEPDIR)/libunittest_a-hsts.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='hsts.c' object='libunittest_a-hsts.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-hsts.o `test -f 'hsts.c' || echo '$(srcdir)/'`hsts.c + +libunittest_a-hsts.obj: hsts.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-hsts.obj -MD -MP -MF $(DEPDIR)/libunittest_a-hsts.Tpo -c -o libunittest_a-hsts.obj `if test -f 'hsts.c'; then $(CYGPATH_W) 'hsts.c'; else $(CYGPATH_W) '$(srcdir)/hsts.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-hsts.Tpo $(DEPDIR)/libunittest_a-hsts.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='hsts.c' object='libunittest_a-hsts.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-hsts.obj `if test -f 'hsts.c'; then $(CYGPATH_W) 'hsts.c'; else $(CYGPATH_W) '$(srcdir)/hsts.c'; fi` + +libunittest_a-html-parse.o: html-parse.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-html-parse.o -MD -MP -MF $(DEPDIR)/libunittest_a-html-parse.Tpo -c -o libunittest_a-html-parse.o `test -f 'html-parse.c' || echo '$(srcdir)/'`html-parse.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-html-parse.Tpo $(DEPDIR)/libunittest_a-html-parse.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='html-parse.c' object='libunittest_a-html-parse.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-html-parse.o `test -f 'html-parse.c' || echo '$(srcdir)/'`html-parse.c + +libunittest_a-html-parse.obj: html-parse.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-html-parse.obj -MD -MP -MF $(DEPDIR)/libunittest_a-html-parse.Tpo -c -o libunittest_a-html-parse.obj `if test -f 'html-parse.c'; then $(CYGPATH_W) 'html-parse.c'; else $(CYGPATH_W) '$(srcdir)/html-parse.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-html-parse.Tpo $(DEPDIR)/libunittest_a-html-parse.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='html-parse.c' object='libunittest_a-html-parse.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-html-parse.obj `if test -f 'html-parse.c'; then $(CYGPATH_W) 'html-parse.c'; else $(CYGPATH_W) '$(srcdir)/html-parse.c'; fi` + +libunittest_a-html-url.o: html-url.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-html-url.o -MD -MP -MF $(DEPDIR)/libunittest_a-html-url.Tpo -c -o libunittest_a-html-url.o `test -f 'html-url.c' || echo '$(srcdir)/'`html-url.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-html-url.Tpo $(DEPDIR)/libunittest_a-html-url.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='html-url.c' object='libunittest_a-html-url.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-html-url.o `test -f 'html-url.c' || echo '$(srcdir)/'`html-url.c + +libunittest_a-html-url.obj: html-url.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-html-url.obj -MD -MP -MF $(DEPDIR)/libunittest_a-html-url.Tpo -c -o libunittest_a-html-url.obj `if test -f 'html-url.c'; then $(CYGPATH_W) 'html-url.c'; else $(CYGPATH_W) '$(srcdir)/html-url.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-html-url.Tpo $(DEPDIR)/libunittest_a-html-url.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='html-url.c' object='libunittest_a-html-url.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-html-url.obj `if test -f 'html-url.c'; then $(CYGPATH_W) 'html-url.c'; else $(CYGPATH_W) '$(srcdir)/html-url.c'; fi` + +libunittest_a-http.o: http.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-http.o -MD -MP -MF $(DEPDIR)/libunittest_a-http.Tpo -c -o libunittest_a-http.o `test -f 'http.c' || echo '$(srcdir)/'`http.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-http.Tpo $(DEPDIR)/libunittest_a-http.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='http.c' object='libunittest_a-http.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-http.o `test -f 'http.c' || echo '$(srcdir)/'`http.c + +libunittest_a-http.obj: http.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-http.obj -MD -MP -MF $(DEPDIR)/libunittest_a-http.Tpo -c -o libunittest_a-http.obj `if test -f 'http.c'; then $(CYGPATH_W) 'http.c'; else $(CYGPATH_W) '$(srcdir)/http.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-http.Tpo $(DEPDIR)/libunittest_a-http.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='http.c' object='libunittest_a-http.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-http.obj `if test -f 'http.c'; then $(CYGPATH_W) 'http.c'; else $(CYGPATH_W) '$(srcdir)/http.c'; fi` + +libunittest_a-init.o: init.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-init.o -MD -MP -MF $(DEPDIR)/libunittest_a-init.Tpo -c -o libunittest_a-init.o `test -f 'init.c' || echo '$(srcdir)/'`init.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-init.Tpo $(DEPDIR)/libunittest_a-init.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='init.c' object='libunittest_a-init.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-init.o `test -f 'init.c' || echo '$(srcdir)/'`init.c + +libunittest_a-init.obj: init.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-init.obj -MD -MP -MF $(DEPDIR)/libunittest_a-init.Tpo -c -o libunittest_a-init.obj `if test -f 'init.c'; then $(CYGPATH_W) 'init.c'; else $(CYGPATH_W) '$(srcdir)/init.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-init.Tpo $(DEPDIR)/libunittest_a-init.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='init.c' object='libunittest_a-init.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-init.obj `if test -f 'init.c'; then $(CYGPATH_W) 'init.c'; else $(CYGPATH_W) '$(srcdir)/init.c'; fi` + +libunittest_a-log.o: log.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-log.o -MD -MP -MF $(DEPDIR)/libunittest_a-log.Tpo -c -o libunittest_a-log.o `test -f 'log.c' || echo '$(srcdir)/'`log.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-log.Tpo $(DEPDIR)/libunittest_a-log.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='log.c' object='libunittest_a-log.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-log.o `test -f 'log.c' || echo '$(srcdir)/'`log.c + +libunittest_a-log.obj: log.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-log.obj -MD -MP -MF $(DEPDIR)/libunittest_a-log.Tpo -c -o libunittest_a-log.obj `if test -f 'log.c'; then $(CYGPATH_W) 'log.c'; else $(CYGPATH_W) '$(srcdir)/log.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-log.Tpo $(DEPDIR)/libunittest_a-log.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='log.c' object='libunittest_a-log.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-log.obj `if test -f 'log.c'; then $(CYGPATH_W) 'log.c'; else $(CYGPATH_W) '$(srcdir)/log.c'; fi` + +libunittest_a-main.o: main.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-main.o -MD -MP -MF $(DEPDIR)/libunittest_a-main.Tpo -c -o libunittest_a-main.o `test -f 'main.c' || echo '$(srcdir)/'`main.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-main.Tpo $(DEPDIR)/libunittest_a-main.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='main.c' object='libunittest_a-main.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-main.o `test -f 'main.c' || echo '$(srcdir)/'`main.c + +libunittest_a-main.obj: main.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-main.obj -MD -MP -MF $(DEPDIR)/libunittest_a-main.Tpo -c -o libunittest_a-main.obj `if test -f 'main.c'; then $(CYGPATH_W) 'main.c'; else $(CYGPATH_W) '$(srcdir)/main.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-main.Tpo $(DEPDIR)/libunittest_a-main.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='main.c' object='libunittest_a-main.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-main.obj `if test -f 'main.c'; then $(CYGPATH_W) 'main.c'; else $(CYGPATH_W) '$(srcdir)/main.c'; fi` + +libunittest_a-netrc.o: netrc.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-netrc.o -MD -MP -MF $(DEPDIR)/libunittest_a-netrc.Tpo -c -o libunittest_a-netrc.o `test -f 'netrc.c' || echo '$(srcdir)/'`netrc.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-netrc.Tpo $(DEPDIR)/libunittest_a-netrc.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='netrc.c' object='libunittest_a-netrc.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-netrc.o `test -f 'netrc.c' || echo '$(srcdir)/'`netrc.c + +libunittest_a-netrc.obj: netrc.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-netrc.obj -MD -MP -MF $(DEPDIR)/libunittest_a-netrc.Tpo -c -o libunittest_a-netrc.obj `if test -f 'netrc.c'; then $(CYGPATH_W) 'netrc.c'; else $(CYGPATH_W) '$(srcdir)/netrc.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-netrc.Tpo $(DEPDIR)/libunittest_a-netrc.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='netrc.c' object='libunittest_a-netrc.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-netrc.obj `if test -f 'netrc.c'; then $(CYGPATH_W) 'netrc.c'; else $(CYGPATH_W) '$(srcdir)/netrc.c'; fi` + +libunittest_a-progress.o: progress.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-progress.o -MD -MP -MF $(DEPDIR)/libunittest_a-progress.Tpo -c -o libunittest_a-progress.o `test -f 'progress.c' || echo '$(srcdir)/'`progress.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-progress.Tpo $(DEPDIR)/libunittest_a-progress.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='progress.c' object='libunittest_a-progress.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-progress.o `test -f 'progress.c' || echo '$(srcdir)/'`progress.c + +libunittest_a-progress.obj: progress.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-progress.obj -MD -MP -MF $(DEPDIR)/libunittest_a-progress.Tpo -c -o libunittest_a-progress.obj `if test -f 'progress.c'; then $(CYGPATH_W) 'progress.c'; else $(CYGPATH_W) '$(srcdir)/progress.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-progress.Tpo $(DEPDIR)/libunittest_a-progress.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='progress.c' object='libunittest_a-progress.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-progress.obj `if test -f 'progress.c'; then $(CYGPATH_W) 'progress.c'; else $(CYGPATH_W) '$(srcdir)/progress.c'; fi` + +libunittest_a-ptimer.o: ptimer.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-ptimer.o -MD -MP -MF $(DEPDIR)/libunittest_a-ptimer.Tpo -c -o libunittest_a-ptimer.o `test -f 'ptimer.c' || echo '$(srcdir)/'`ptimer.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-ptimer.Tpo $(DEPDIR)/libunittest_a-ptimer.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='ptimer.c' object='libunittest_a-ptimer.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-ptimer.o `test -f 'ptimer.c' || echo '$(srcdir)/'`ptimer.c + +libunittest_a-ptimer.obj: ptimer.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-ptimer.obj -MD -MP -MF $(DEPDIR)/libunittest_a-ptimer.Tpo -c -o libunittest_a-ptimer.obj `if test -f 'ptimer.c'; then $(CYGPATH_W) 'ptimer.c'; else $(CYGPATH_W) '$(srcdir)/ptimer.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-ptimer.Tpo $(DEPDIR)/libunittest_a-ptimer.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='ptimer.c' object='libunittest_a-ptimer.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-ptimer.obj `if test -f 'ptimer.c'; then $(CYGPATH_W) 'ptimer.c'; else $(CYGPATH_W) '$(srcdir)/ptimer.c'; fi` + +libunittest_a-recur.o: recur.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-recur.o -MD -MP -MF $(DEPDIR)/libunittest_a-recur.Tpo -c -o libunittest_a-recur.o `test -f 'recur.c' || echo '$(srcdir)/'`recur.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-recur.Tpo $(DEPDIR)/libunittest_a-recur.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='recur.c' object='libunittest_a-recur.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-recur.o `test -f 'recur.c' || echo '$(srcdir)/'`recur.c + +libunittest_a-recur.obj: recur.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-recur.obj -MD -MP -MF $(DEPDIR)/libunittest_a-recur.Tpo -c -o libunittest_a-recur.obj `if test -f 'recur.c'; then $(CYGPATH_W) 'recur.c'; else $(CYGPATH_W) '$(srcdir)/recur.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-recur.Tpo $(DEPDIR)/libunittest_a-recur.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='recur.c' object='libunittest_a-recur.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-recur.obj `if test -f 'recur.c'; then $(CYGPATH_W) 'recur.c'; else $(CYGPATH_W) '$(srcdir)/recur.c'; fi` + +libunittest_a-res.o: res.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-res.o -MD -MP -MF $(DEPDIR)/libunittest_a-res.Tpo -c -o libunittest_a-res.o `test -f 'res.c' || echo '$(srcdir)/'`res.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-res.Tpo $(DEPDIR)/libunittest_a-res.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='res.c' object='libunittest_a-res.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-res.o `test -f 'res.c' || echo '$(srcdir)/'`res.c + +libunittest_a-res.obj: res.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-res.obj -MD -MP -MF $(DEPDIR)/libunittest_a-res.Tpo -c -o libunittest_a-res.obj `if test -f 'res.c'; then $(CYGPATH_W) 'res.c'; else $(CYGPATH_W) '$(srcdir)/res.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-res.Tpo $(DEPDIR)/libunittest_a-res.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='res.c' object='libunittest_a-res.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-res.obj `if test -f 'res.c'; then $(CYGPATH_W) 'res.c'; else $(CYGPATH_W) '$(srcdir)/res.c'; fi` + +libunittest_a-retr.o: retr.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-retr.o -MD -MP -MF $(DEPDIR)/libunittest_a-retr.Tpo -c -o libunittest_a-retr.o `test -f 'retr.c' || echo '$(srcdir)/'`retr.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-retr.Tpo $(DEPDIR)/libunittest_a-retr.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='retr.c' object='libunittest_a-retr.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-retr.o `test -f 'retr.c' || echo '$(srcdir)/'`retr.c + +libunittest_a-retr.obj: retr.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-retr.obj -MD -MP -MF $(DEPDIR)/libunittest_a-retr.Tpo -c -o libunittest_a-retr.obj `if test -f 'retr.c'; then $(CYGPATH_W) 'retr.c'; else $(CYGPATH_W) '$(srcdir)/retr.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-retr.Tpo $(DEPDIR)/libunittest_a-retr.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='retr.c' object='libunittest_a-retr.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-retr.obj `if test -f 'retr.c'; then $(CYGPATH_W) 'retr.c'; else $(CYGPATH_W) '$(srcdir)/retr.c'; fi` + +libunittest_a-spider.o: spider.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-spider.o -MD -MP -MF $(DEPDIR)/libunittest_a-spider.Tpo -c -o libunittest_a-spider.o `test -f 'spider.c' || echo '$(srcdir)/'`spider.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-spider.Tpo $(DEPDIR)/libunittest_a-spider.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='spider.c' object='libunittest_a-spider.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-spider.o `test -f 'spider.c' || echo '$(srcdir)/'`spider.c + +libunittest_a-spider.obj: spider.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-spider.obj -MD -MP -MF $(DEPDIR)/libunittest_a-spider.Tpo -c -o libunittest_a-spider.obj `if test -f 'spider.c'; then $(CYGPATH_W) 'spider.c'; else $(CYGPATH_W) '$(srcdir)/spider.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-spider.Tpo $(DEPDIR)/libunittest_a-spider.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='spider.c' object='libunittest_a-spider.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-spider.obj `if test -f 'spider.c'; then $(CYGPATH_W) 'spider.c'; else $(CYGPATH_W) '$(srcdir)/spider.c'; fi` + +libunittest_a-url.o: url.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-url.o -MD -MP -MF $(DEPDIR)/libunittest_a-url.Tpo -c -o libunittest_a-url.o `test -f 'url.c' || echo '$(srcdir)/'`url.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-url.Tpo $(DEPDIR)/libunittest_a-url.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='url.c' object='libunittest_a-url.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-url.o `test -f 'url.c' || echo '$(srcdir)/'`url.c + +libunittest_a-url.obj: url.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-url.obj -MD -MP -MF $(DEPDIR)/libunittest_a-url.Tpo -c -o libunittest_a-url.obj `if test -f 'url.c'; then $(CYGPATH_W) 'url.c'; else $(CYGPATH_W) '$(srcdir)/url.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-url.Tpo $(DEPDIR)/libunittest_a-url.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='url.c' object='libunittest_a-url.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-url.obj `if test -f 'url.c'; then $(CYGPATH_W) 'url.c'; else $(CYGPATH_W) '$(srcdir)/url.c'; fi` + +libunittest_a-warc.o: warc.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-warc.o -MD -MP -MF $(DEPDIR)/libunittest_a-warc.Tpo -c -o libunittest_a-warc.o `test -f 'warc.c' || echo '$(srcdir)/'`warc.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-warc.Tpo $(DEPDIR)/libunittest_a-warc.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='warc.c' object='libunittest_a-warc.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-warc.o `test -f 'warc.c' || echo '$(srcdir)/'`warc.c + +libunittest_a-warc.obj: warc.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-warc.obj -MD -MP -MF $(DEPDIR)/libunittest_a-warc.Tpo -c -o libunittest_a-warc.obj `if test -f 'warc.c'; then $(CYGPATH_W) 'warc.c'; else $(CYGPATH_W) '$(srcdir)/warc.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-warc.Tpo $(DEPDIR)/libunittest_a-warc.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='warc.c' object='libunittest_a-warc.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-warc.obj `if test -f 'warc.c'; then $(CYGPATH_W) 'warc.c'; else $(CYGPATH_W) '$(srcdir)/warc.c'; fi` + +libunittest_a-xattr.o: xattr.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-xattr.o -MD -MP -MF $(DEPDIR)/libunittest_a-xattr.Tpo -c -o libunittest_a-xattr.o `test -f 'xattr.c' || echo '$(srcdir)/'`xattr.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-xattr.Tpo $(DEPDIR)/libunittest_a-xattr.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='xattr.c' object='libunittest_a-xattr.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-xattr.o `test -f 'xattr.c' || echo '$(srcdir)/'`xattr.c + +libunittest_a-xattr.obj: xattr.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-xattr.obj -MD -MP -MF $(DEPDIR)/libunittest_a-xattr.Tpo -c -o libunittest_a-xattr.obj `if test -f 'xattr.c'; then $(CYGPATH_W) 'xattr.c'; else $(CYGPATH_W) '$(srcdir)/xattr.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-xattr.Tpo $(DEPDIR)/libunittest_a-xattr.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='xattr.c' object='libunittest_a-xattr.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-xattr.obj `if test -f 'xattr.c'; then $(CYGPATH_W) 'xattr.c'; else $(CYGPATH_W) '$(srcdir)/xattr.c'; fi` + +libunittest_a-utils.o: utils.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-utils.o -MD -MP -MF $(DEPDIR)/libunittest_a-utils.Tpo -c -o libunittest_a-utils.o `test -f 'utils.c' || echo '$(srcdir)/'`utils.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-utils.Tpo $(DEPDIR)/libunittest_a-utils.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='utils.c' object='libunittest_a-utils.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-utils.o `test -f 'utils.c' || echo '$(srcdir)/'`utils.c + +libunittest_a-utils.obj: utils.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-utils.obj -MD -MP -MF $(DEPDIR)/libunittest_a-utils.Tpo -c -o libunittest_a-utils.obj `if test -f 'utils.c'; then $(CYGPATH_W) 'utils.c'; else $(CYGPATH_W) '$(srcdir)/utils.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-utils.Tpo $(DEPDIR)/libunittest_a-utils.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='utils.c' object='libunittest_a-utils.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-utils.obj `if test -f 'utils.c'; then $(CYGPATH_W) 'utils.c'; else $(CYGPATH_W) '$(srcdir)/utils.c'; fi` + +libunittest_a-exits.o: exits.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-exits.o -MD -MP -MF $(DEPDIR)/libunittest_a-exits.Tpo -c -o libunittest_a-exits.o `test -f 'exits.c' || echo '$(srcdir)/'`exits.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-exits.Tpo $(DEPDIR)/libunittest_a-exits.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='exits.c' object='libunittest_a-exits.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-exits.o `test -f 'exits.c' || echo '$(srcdir)/'`exits.c + +libunittest_a-exits.obj: exits.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-exits.obj -MD -MP -MF $(DEPDIR)/libunittest_a-exits.Tpo -c -o libunittest_a-exits.obj `if test -f 'exits.c'; then $(CYGPATH_W) 'exits.c'; else $(CYGPATH_W) '$(srcdir)/exits.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-exits.Tpo $(DEPDIR)/libunittest_a-exits.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='exits.c' object='libunittest_a-exits.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-exits.obj `if test -f 'exits.c'; then $(CYGPATH_W) 'exits.c'; else $(CYGPATH_W) '$(srcdir)/exits.c'; fi` + +libunittest_a-build_info.o: build_info.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-build_info.o -MD -MP -MF $(DEPDIR)/libunittest_a-build_info.Tpo -c -o libunittest_a-build_info.o `test -f 'build_info.c' || echo '$(srcdir)/'`build_info.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-build_info.Tpo $(DEPDIR)/libunittest_a-build_info.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='build_info.c' object='libunittest_a-build_info.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-build_info.o `test -f 'build_info.c' || echo '$(srcdir)/'`build_info.c + +libunittest_a-build_info.obj: build_info.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-build_info.obj -MD -MP -MF $(DEPDIR)/libunittest_a-build_info.Tpo -c -o libunittest_a-build_info.obj `if test -f 'build_info.c'; then $(CYGPATH_W) 'build_info.c'; else $(CYGPATH_W) '$(srcdir)/build_info.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-build_info.Tpo $(DEPDIR)/libunittest_a-build_info.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='build_info.c' object='libunittest_a-build_info.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-build_info.obj `if test -f 'build_info.c'; then $(CYGPATH_W) 'build_info.c'; else $(CYGPATH_W) '$(srcdir)/build_info.c'; fi` + +libunittest_a-iri.o: iri.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-iri.o -MD -MP -MF $(DEPDIR)/libunittest_a-iri.Tpo -c -o libunittest_a-iri.o `test -f 'iri.c' || echo '$(srcdir)/'`iri.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-iri.Tpo $(DEPDIR)/libunittest_a-iri.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='iri.c' object='libunittest_a-iri.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-iri.o `test -f 'iri.c' || echo '$(srcdir)/'`iri.c + +libunittest_a-iri.obj: iri.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-iri.obj -MD -MP -MF $(DEPDIR)/libunittest_a-iri.Tpo -c -o libunittest_a-iri.obj `if test -f 'iri.c'; then $(CYGPATH_W) 'iri.c'; else $(CYGPATH_W) '$(srcdir)/iri.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-iri.Tpo $(DEPDIR)/libunittest_a-iri.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='iri.c' object='libunittest_a-iri.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-iri.obj `if test -f 'iri.c'; then $(CYGPATH_W) 'iri.c'; else $(CYGPATH_W) '$(srcdir)/iri.c'; fi` + +libunittest_a-metalink.o: metalink.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-metalink.o -MD -MP -MF $(DEPDIR)/libunittest_a-metalink.Tpo -c -o libunittest_a-metalink.o `test -f 'metalink.c' || echo '$(srcdir)/'`metalink.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-metalink.Tpo $(DEPDIR)/libunittest_a-metalink.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='metalink.c' object='libunittest_a-metalink.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-metalink.o `test -f 'metalink.c' || echo '$(srcdir)/'`metalink.c + +libunittest_a-metalink.obj: metalink.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-metalink.obj -MD -MP -MF $(DEPDIR)/libunittest_a-metalink.Tpo -c -o libunittest_a-metalink.obj `if test -f 'metalink.c'; then $(CYGPATH_W) 'metalink.c'; else $(CYGPATH_W) '$(srcdir)/metalink.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-metalink.Tpo $(DEPDIR)/libunittest_a-metalink.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='metalink.c' object='libunittest_a-metalink.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-metalink.obj `if test -f 'metalink.c'; then $(CYGPATH_W) 'metalink.c'; else $(CYGPATH_W) '$(srcdir)/metalink.c'; fi` + +libunittest_a-version.o: version.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-version.o -MD -MP -MF $(DEPDIR)/libunittest_a-version.Tpo -c -o libunittest_a-version.o `test -f 'version.c' || echo '$(srcdir)/'`version.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-version.Tpo $(DEPDIR)/libunittest_a-version.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='version.c' object='libunittest_a-version.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-version.o `test -f 'version.c' || echo '$(srcdir)/'`version.c + +libunittest_a-version.obj: version.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libunittest_a-version.obj -MD -MP -MF $(DEPDIR)/libunittest_a-version.Tpo -c -o libunittest_a-version.obj `if test -f 'version.c'; then $(CYGPATH_W) 'version.c'; else $(CYGPATH_W) '$(srcdir)/version.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libunittest_a-version.Tpo $(DEPDIR)/libunittest_a-version.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='version.c' object='libunittest_a-version.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libunittest_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libunittest_a-version.obj `if test -f 'version.c'; then $(CYGPATH_W) 'version.c'; else $(CYGPATH_W) '$(srcdir)/version.c'; fi` + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) $(check_LIBRARIES) +check: check-am +all-am: Makefile $(PROGRAMS) config.h +installdirs: + for dir in "$(DESTDIR)$(bindir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES) + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-binPROGRAMS clean-checkLIBRARIES clean-generic \ + mostlyclean-am + +distclean: distclean-am + -rm -f $(DEPDIR)/ftp-opie.Po + -rm -f $(DEPDIR)/gnutls.Po + -rm -f $(DEPDIR)/http-ntlm.Po + -rm -f $(DEPDIR)/mswindows.Po + -rm -f $(DEPDIR)/openssl.Po + -rm -f ./$(DEPDIR)/build_info.Po + -rm -f ./$(DEPDIR)/connect.Po + -rm -f ./$(DEPDIR)/convert.Po + -rm -f ./$(DEPDIR)/cookies.Po + -rm -f ./$(DEPDIR)/css-url.Po + -rm -f ./$(DEPDIR)/css_.Po + -rm -f ./$(DEPDIR)/exits.Po + -rm -f ./$(DEPDIR)/ftp-basic.Po + -rm -f ./$(DEPDIR)/ftp-ls.Po + -rm -f ./$(DEPDIR)/ftp.Po + -rm -f ./$(DEPDIR)/hash.Po + -rm -f ./$(DEPDIR)/host.Po + -rm -f ./$(DEPDIR)/hsts.Po + -rm -f ./$(DEPDIR)/html-parse.Po + -rm -f ./$(DEPDIR)/html-url.Po + -rm -f ./$(DEPDIR)/http.Po + -rm -f ./$(DEPDIR)/init.Po + -rm -f ./$(DEPDIR)/iri.Po + -rm -f ./$(DEPDIR)/libunittest_a-build_info.Po + -rm -f ./$(DEPDIR)/libunittest_a-connect.Po + -rm -f ./$(DEPDIR)/libunittest_a-convert.Po + -rm -f ./$(DEPDIR)/libunittest_a-cookies.Po + -rm -f ./$(DEPDIR)/libunittest_a-css-url.Po + -rm -f ./$(DEPDIR)/libunittest_a-css_.Po + -rm -f ./$(DEPDIR)/libunittest_a-exits.Po + -rm -f ./$(DEPDIR)/libunittest_a-ftp-basic.Po + -rm -f ./$(DEPDIR)/libunittest_a-ftp-ls.Po + -rm -f ./$(DEPDIR)/libunittest_a-ftp.Po + -rm -f ./$(DEPDIR)/libunittest_a-hash.Po + -rm -f ./$(DEPDIR)/libunittest_a-host.Po + -rm -f ./$(DEPDIR)/libunittest_a-hsts.Po + -rm -f ./$(DEPDIR)/libunittest_a-html-parse.Po + -rm -f ./$(DEPDIR)/libunittest_a-html-url.Po + -rm -f ./$(DEPDIR)/libunittest_a-http.Po + -rm -f ./$(DEPDIR)/libunittest_a-init.Po + -rm -f ./$(DEPDIR)/libunittest_a-iri.Po + -rm -f ./$(DEPDIR)/libunittest_a-log.Po + -rm -f ./$(DEPDIR)/libunittest_a-main.Po + -rm -f ./$(DEPDIR)/libunittest_a-metalink.Po + -rm -f ./$(DEPDIR)/libunittest_a-netrc.Po + -rm -f ./$(DEPDIR)/libunittest_a-progress.Po + -rm -f ./$(DEPDIR)/libunittest_a-ptimer.Po + -rm -f ./$(DEPDIR)/libunittest_a-recur.Po + -rm -f ./$(DEPDIR)/libunittest_a-res.Po + -rm -f ./$(DEPDIR)/libunittest_a-retr.Po + -rm -f ./$(DEPDIR)/libunittest_a-spider.Po + -rm -f ./$(DEPDIR)/libunittest_a-url.Po + -rm -f ./$(DEPDIR)/libunittest_a-utils.Po + -rm -f ./$(DEPDIR)/libunittest_a-version.Po + -rm -f ./$(DEPDIR)/libunittest_a-warc.Po + -rm -f ./$(DEPDIR)/libunittest_a-xattr.Po + -rm -f ./$(DEPDIR)/log.Po + -rm -f ./$(DEPDIR)/main.Po + -rm -f ./$(DEPDIR)/metalink.Po + -rm -f ./$(DEPDIR)/netrc.Po + -rm -f ./$(DEPDIR)/progress.Po + -rm -f ./$(DEPDIR)/ptimer.Po + -rm -f ./$(DEPDIR)/recur.Po + -rm -f ./$(DEPDIR)/res.Po + -rm -f ./$(DEPDIR)/retr.Po + -rm -f ./$(DEPDIR)/spider.Po + -rm -f ./$(DEPDIR)/url.Po + -rm -f ./$(DEPDIR)/utils.Po + -rm -f ./$(DEPDIR)/version.Po + -rm -f ./$(DEPDIR)/warc.Po + -rm -f ./$(DEPDIR)/xattr.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-hdr distclean-local distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: install-binPROGRAMS + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f $(DEPDIR)/ftp-opie.Po + -rm -f $(DEPDIR)/gnutls.Po + -rm -f $(DEPDIR)/http-ntlm.Po + -rm -f $(DEPDIR)/mswindows.Po + -rm -f $(DEPDIR)/openssl.Po + -rm -f ./$(DEPDIR)/build_info.Po + -rm -f ./$(DEPDIR)/connect.Po + -rm -f ./$(DEPDIR)/convert.Po + -rm -f ./$(DEPDIR)/cookies.Po + -rm -f ./$(DEPDIR)/css-url.Po + -rm -f ./$(DEPDIR)/css_.Po + -rm -f ./$(DEPDIR)/exits.Po + -rm -f ./$(DEPDIR)/ftp-basic.Po + -rm -f ./$(DEPDIR)/ftp-ls.Po + -rm -f ./$(DEPDIR)/ftp.Po + -rm -f ./$(DEPDIR)/hash.Po + -rm -f ./$(DEPDIR)/host.Po + -rm -f ./$(DEPDIR)/hsts.Po + -rm -f ./$(DEPDIR)/html-parse.Po + -rm -f ./$(DEPDIR)/html-url.Po + -rm -f ./$(DEPDIR)/http.Po + -rm -f ./$(DEPDIR)/init.Po + -rm -f ./$(DEPDIR)/iri.Po + -rm -f ./$(DEPDIR)/libunittest_a-build_info.Po + -rm -f ./$(DEPDIR)/libunittest_a-connect.Po + -rm -f ./$(DEPDIR)/libunittest_a-convert.Po + -rm -f ./$(DEPDIR)/libunittest_a-cookies.Po + -rm -f ./$(DEPDIR)/libunittest_a-css-url.Po + -rm -f ./$(DEPDIR)/libunittest_a-css_.Po + -rm -f ./$(DEPDIR)/libunittest_a-exits.Po + -rm -f ./$(DEPDIR)/libunittest_a-ftp-basic.Po + -rm -f ./$(DEPDIR)/libunittest_a-ftp-ls.Po + -rm -f ./$(DEPDIR)/libunittest_a-ftp.Po + -rm -f ./$(DEPDIR)/libunittest_a-hash.Po + -rm -f ./$(DEPDIR)/libunittest_a-host.Po + -rm -f ./$(DEPDIR)/libunittest_a-hsts.Po + -rm -f ./$(DEPDIR)/libunittest_a-html-parse.Po + -rm -f ./$(DEPDIR)/libunittest_a-html-url.Po + -rm -f ./$(DEPDIR)/libunittest_a-http.Po + -rm -f ./$(DEPDIR)/libunittest_a-init.Po + -rm -f ./$(DEPDIR)/libunittest_a-iri.Po + -rm -f ./$(DEPDIR)/libunittest_a-log.Po + -rm -f ./$(DEPDIR)/libunittest_a-main.Po + -rm -f ./$(DEPDIR)/libunittest_a-metalink.Po + -rm -f ./$(DEPDIR)/libunittest_a-netrc.Po + -rm -f ./$(DEPDIR)/libunittest_a-progress.Po + -rm -f ./$(DEPDIR)/libunittest_a-ptimer.Po + -rm -f ./$(DEPDIR)/libunittest_a-recur.Po + -rm -f ./$(DEPDIR)/libunittest_a-res.Po + -rm -f ./$(DEPDIR)/libunittest_a-retr.Po + -rm -f ./$(DEPDIR)/libunittest_a-spider.Po + -rm -f ./$(DEPDIR)/libunittest_a-url.Po + -rm -f ./$(DEPDIR)/libunittest_a-utils.Po + -rm -f ./$(DEPDIR)/libunittest_a-version.Po + -rm -f ./$(DEPDIR)/libunittest_a-warc.Po + -rm -f ./$(DEPDIR)/libunittest_a-xattr.Po + -rm -f ./$(DEPDIR)/log.Po + -rm -f ./$(DEPDIR)/main.Po + -rm -f ./$(DEPDIR)/metalink.Po + -rm -f ./$(DEPDIR)/netrc.Po + -rm -f ./$(DEPDIR)/progress.Po + -rm -f ./$(DEPDIR)/ptimer.Po + -rm -f ./$(DEPDIR)/recur.Po + -rm -f ./$(DEPDIR)/res.Po + -rm -f ./$(DEPDIR)/retr.Po + -rm -f ./$(DEPDIR)/spider.Po + -rm -f ./$(DEPDIR)/url.Po + -rm -f ./$(DEPDIR)/utils.Po + -rm -f ./$(DEPDIR)/version.Po + -rm -f ./$(DEPDIR)/warc.Po + -rm -f ./$(DEPDIR)/xattr.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-binPROGRAMS + +.MAKE: all check-am install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-binPROGRAMS clean-checkLIBRARIES clean-generic \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-hdr distclean-local distclean-tags \ + distdir dvi dvi-am html html-am info info-am install \ + install-am install-binPROGRAMS install-data install-data-am \ + install-dvi install-dvi-am install-exec install-exec-am \ + install-html install-html-am install-info install-info-am \ + install-man install-pdf install-pdf-am install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic pdf pdf-am \ + ps ps-am tags tags-am uninstall uninstall-am \ + uninstall-binPROGRAMS + +.PRECIOUS: Makefile + + +../lib/libgnu.a: + cd ../lib && $(MAKE) $(AM_MAKEFLAGS) + +build_info.c: $(srcdir)/Makefile.am $(srcdir)/build_info.c.in + if test -n "$(VPATH)"; then cp "$(srcdir)/build_info.c.in" .; fi + $(PERL) "$(top_srcdir)/build-aux/build_info.pl" \ + "$(top_builddir)/src/build_info.c" + if test -n "$(VPATH)"; then rm -f build_info.c.in; fi +version.c: $(wget_SOURCES) ../lib/libgnu.a + echo '/* version.c */' > $@ + echo '/* Autogenerated by Makefile - DO NOT EDIT */' >> $@ + echo '' >> $@ + echo '#include "version.h"' >> $@ + echo 'const char *version_string = "@VERSION@";' >> $@ + echo 'const char *compilation_string = "'$(COMPILE)'";' \ + | $(ESCAPEQUOTE) >> $@ + echo 'const char *link_string = "'$(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) $(LIBS) $(wget_LDADD)'";' \ + | $(ESCAPEQUOTE) >> $@ + +css.c: $(srcdir)/css.l + $(LEX) $(LFLAGS) -o$@ $^ + +css_.c: css.c + echo '#include "wget.h"' > $@ + cat css.c >> $@ + +distclean-local: + rm -f css.c css_.c + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/build_info.c b/src/build_info.c new file mode 100644 index 0000000..bf6e6a9 --- /dev/null +++ b/src/build_info.c @@ -0,0 +1,101 @@ +/* Autogenerated by build_info.pl - DO NOT EDIT */ + +/* This stores global variables that are initialized with + preprocessor declarations for output with the --version flag. + + Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, + 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, Inc. */ + +#include "wget.h" +#include <stdio.h> +#include "version.h" + +const char *compiled_features[] = +{ + +#if defined HAVE_LIBCARES + "+cares", +#else + "-cares", +#endif + +#if defined ENABLE_DIGEST + "+digest", +#else + "-digest", +#endif + +#if defined HAVE_GPGME + "+gpgme", +#else + "-gpgme", +#endif + +#if defined HAVE_SSL + "+https", +#else + "-https", +#endif + +#if defined ENABLE_IPV6 + "+ipv6", +#else + "-ipv6", +#endif + +#if defined ENABLE_IRI + "+iri", +#else + "-iri", +#endif + +#if SIZEOF_OFF_T >= 8 || defined WINDOWS + "+large-file", +#else + "-large-file", +#endif + +#if defined HAVE_METALINK + "+metalink", +#else + "-metalink", +#endif + +#if defined ENABLE_NLS + "+nls", +#else + "-nls", +#endif + +#if defined ENABLE_NTLM + "+ntlm", +#else + "-ntlm", +#endif + +#if defined ENABLE_OPIE + "+opie", +#else + "-opie", +#endif + +#if defined HAVE_LIBPSL + "+psl", +#else + "-psl", +#endif + +#if defined HAVE_LIBSSL || defined HAVE_LIBSSL32 + "+ssl/openssl", +#elif defined HAVE_LIBGNUTLS + "+ssl/gnutls", +#else + "-ssl", +#endif + + + /* sentinel value */ + NULL +}; + + diff --git a/src/build_info.c.in b/src/build_info.c.in new file mode 100644 index 0000000..c7493e9 --- /dev/null +++ b/src/build_info.c.in @@ -0,0 +1,18 @@ +digest defined ENABLE_DIGEST +https defined HAVE_SSL +ipv6 defined ENABLE_IPV6 +iri defined ENABLE_IRI +large-file SIZEOF_OFF_T >= 8 || defined WINDOWS + +nls defined ENABLE_NLS +ntlm defined ENABLE_NTLM +opie defined ENABLE_OPIE +psl defined HAVE_LIBPSL +cares defined HAVE_LIBCARES + +metalink defined HAVE_METALINK +gpgme defined HAVE_GPGME + +ssl choice: + openssl defined HAVE_LIBSSL || defined HAVE_LIBSSL32 + gnutls defined HAVE_LIBGNUTLS diff --git a/src/config.h.in b/src/config.h.in new file mode 100644 index 0000000..936d100 --- /dev/null +++ b/src/config.h.in @@ -0,0 +1,2685 @@ +/* src/config.h.in. Generated from configure.ac by autoheader. */ + +/* Define if building universal (internal helper macro) */ +#undef AC_APPLE_UNIVERSAL_BUILD + +/* Define to the number of bits in type 'ptrdiff_t'. */ +#undef BITSIZEOF_PTRDIFF_T + +/* Define to the number of bits in type 'sig_atomic_t'. */ +#undef BITSIZEOF_SIG_ATOMIC_T + +/* Define to the number of bits in type 'size_t'. */ +#undef BITSIZEOF_SIZE_T + +/* Define to the number of bits in type 'wchar_t'. */ +#undef BITSIZEOF_WCHAR_T + +/* Define to the number of bits in type 'wint_t'. */ +#undef BITSIZEOF_WINT_T + +/* Define to 1 if using `alloca.c'. */ +#undef C_ALLOCA + +/* Define as the bit index in the word where to find bit 0 of the exponent of + 'double'. */ +#undef DBL_EXPBIT0_BIT + +/* Define as the word index where to find the exponent of 'double'. */ +#undef DBL_EXPBIT0_WORD + +/* the name of the file descriptor member of DIR */ +#undef DIR_FD_MEMBER_NAME + +#ifdef DIR_FD_MEMBER_NAME +# define DIR_TO_FD(Dir_p) ((Dir_p)->DIR_FD_MEMBER_NAME) +#else +# define DIR_TO_FD(Dir_p) -1 +#endif + + +/* Define to 1 if // is a file system root distinct from /. */ +#undef DOUBLE_SLASH_IS_DISTINCT_ROOT + +/* Define if struct dirent has a member d_ino that actually works. */ +#undef D_INO_IN_DIRENT + +/* Define if you want the debug output support compiled in. */ +#undef ENABLE_DEBUG + +/* Define if you want the HTTP Digest Authorization compiled in. */ +#undef ENABLE_DIGEST + +/* Define if IPv6 support is enabled. */ +#undef ENABLE_IPV6 + +/* Define if IRI support is enabled. */ +#undef ENABLE_IRI + +/* Define to 1 if translation of program messages to the user's native + language is requested. */ +#undef ENABLE_NLS + +/* Define if you want the NTLM authorization support compiled in. */ +#undef ENABLE_NTLM + +/* Define if you want Opie support for FTP compiled in. */ +#undef ENABLE_OPIE + +/* Define if you want file meta-data storing into POSIX Extended Attributes + compiled in. */ +#undef ENABLE_XATTR + +/* Define this to 1 if F_DUPFD behavior does not match POSIX */ +#undef FCNTL_DUPFD_BUGGY + +/* Define to nothing if C supports flexible array members, and to 1 if it does + not. That way, with a declaration like 'struct s { int n; short + d[FLEXIBLE_ARRAY_MEMBER]; };', the struct hack can be used with pre-C99 + compilers. Use 'FLEXSIZEOF (struct s, d, N * sizeof (short))' to calculate + the size in bytes of such a struct containing an N-element array. */ +#undef FLEXIBLE_ARRAY_MEMBER + +/* Define to 1 if fopen() fails to recognize a trailing slash. */ +#undef FOPEN_TRAILING_SLASH_BUG + +/* Define to 1 if the system's ftello function has the Solaris bug. */ +#undef FTELLO_BROKEN_AFTER_SWITCHING_FROM_READ_TO_WRITE + +/* Define to 1 if fflush is known to work on stdin as per POSIX.1-2008, 0 if + fflush is known to not work, -1 if unknown. */ +#undef FUNC_FFLUSH_STDIN + +/* Define to 1 if mkdir mistakenly creates a directory given with a trailing + dot component. */ +#undef FUNC_MKDIR_DOT_BUG + +/* Define to 1 if nl_langinfo (YESEXPR) returns a non-empty string. */ +#undef FUNC_NL_LANGINFO_YESEXPR_WORKS + +/* Define to 1 if realpath() can malloc memory, always gives an absolute path, + and handles trailing slash correctly. */ +#undef FUNC_REALPATH_WORKS + +/* Define to 1 if futimesat mishandles a NULL file name. */ +#undef FUTIMESAT_NULL_BUG + +/* Define to 1 if this is a fuzzing build */ +#undef FUZZING + +/* Define to the type of elements in the array set by `getgroups'. Usually + this is either `int' or `gid_t'. */ +#undef GETGROUPS_T + +/* Define this to 1 if getgroups(0,NULL) does not return the number of groups. + */ +#undef GETGROUPS_ZERO_BUG + +/* Define this to 'void' or 'struct timezone' to match the system's + declaration of the second argument to gettimeofday. */ +#undef GETTIMEOFDAY_TIMEZONE + +/* Compile Gnulib crypto stream ops. */ +#undef GL_COMPILE_CRYPTO_STREAM + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module canonicalize shall be considered present. */ +#undef GNULIB_CANONICALIZE + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module dirname shall be considered present. */ +#undef GNULIB_DIRNAME + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module fdopendir shall be considered present. */ +#undef GNULIB_FDOPENDIR + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module fd-safer-flag shall be considered present. */ +#undef GNULIB_FD_SAFER_FLAG + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module fflush shall be considered present. */ +#undef GNULIB_FFLUSH + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module fnmatch-gnu shall be considered present. */ +#undef GNULIB_FNMATCH_GNU + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module fopen-gnu shall be considered present. */ +#undef GNULIB_FOPEN_GNU + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module fscanf shall be considered present. */ +#undef GNULIB_FSCANF + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module getcwd shall be considered present. */ +#undef GNULIB_GETCWD + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module isblank shall be considered present. */ +#undef GNULIB_ISBLANK + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module lock shall be considered present. */ +#undef GNULIB_LOCK + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module mkostemp shall be considered present. */ +#undef GNULIB_MKOSTEMP + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module msvc-nothrow shall be considered present. */ +#undef GNULIB_MSVC_NOTHROW + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module openat shall be considered present. */ +#undef GNULIB_OPENAT + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module pipe2-safer shall be considered present. */ +#undef GNULIB_PIPE2_SAFER + +/* Define to 1 if printf and friends should be labeled with attribute + "__gnu_printf__" instead of "__printf__" */ +#undef GNULIB_PRINTF_ATTRIBUTE_FLAVOR_GNU + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module scanf shall be considered present. */ +#undef GNULIB_SCANF + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module sigpipe shall be considered present. */ +#undef GNULIB_SIGPIPE + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module snprintf shall be considered present. */ +#undef GNULIB_SNPRINTF + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module strerror shall be considered present. */ +#undef GNULIB_STRERROR + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module strerror_r-posix shall be considered present. */ +#undef GNULIB_STRERROR_R_POSIX + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module tempname shall be considered present. */ +#undef GNULIB_TEMPNAME + +/* Define to 1 when the gnulib module accept should be tested. */ +#undef GNULIB_TEST_ACCEPT + +/* Define to 1 when the gnulib module access should be tested. */ +#undef GNULIB_TEST_ACCESS + +/* Define to 1 when the gnulib module bind should be tested. */ +#undef GNULIB_TEST_BIND + +/* Define to 1 when the gnulib module btowc should be tested. */ +#undef GNULIB_TEST_BTOWC + +/* Define to 1 when the gnulib module canonicalize should be tested. */ +#undef GNULIB_TEST_CANONICALIZE + +/* Define to 1 when the gnulib module canonicalize_file_name should be tested. + */ +#undef GNULIB_TEST_CANONICALIZE_FILE_NAME + +/* Define to 1 when the gnulib module chdir should be tested. */ +#undef GNULIB_TEST_CHDIR + +/* Define to 1 when the gnulib module cloexec should be tested. */ +#undef GNULIB_TEST_CLOEXEC + +/* Define to 1 when the gnulib module close should be tested. */ +#undef GNULIB_TEST_CLOSE + +/* Define to 1 when the gnulib module closedir should be tested. */ +#undef GNULIB_TEST_CLOSEDIR + +/* Define to 1 when the gnulib module connect should be tested. */ +#undef GNULIB_TEST_CONNECT + +/* Define to 1 when the gnulib module dirfd should be tested. */ +#undef GNULIB_TEST_DIRFD + +/* Define to 1 when the gnulib module dup should be tested. */ +#undef GNULIB_TEST_DUP + +/* Define to 1 when the gnulib module dup2 should be tested. */ +#undef GNULIB_TEST_DUP2 + +/* Define to 1 when the gnulib module environ should be tested. */ +#undef GNULIB_TEST_ENVIRON + +/* Define to 1 when the gnulib module fchdir should be tested. */ +#undef GNULIB_TEST_FCHDIR + +/* Define to 1 when the gnulib module fcntl should be tested. */ +#undef GNULIB_TEST_FCNTL + +/* Define to 1 when the gnulib module fdopendir should be tested. */ +#undef GNULIB_TEST_FDOPENDIR + +/* Define to 1 when the gnulib module fflush should be tested. */ +#undef GNULIB_TEST_FFLUSH + +/* Define to 1 when the gnulib module fnmatch should be tested. */ +#undef GNULIB_TEST_FNMATCH + +/* Define to 1 when the gnulib module fopen should be tested. */ +#undef GNULIB_TEST_FOPEN + +/* Define to 1 when the gnulib module fpurge should be tested. */ +#undef GNULIB_TEST_FPURGE + +/* Define to 1 when the gnulib module free-posix should be tested. */ +#undef GNULIB_TEST_FREE_POSIX + +/* Define to 1 when the gnulib module fseek should be tested. */ +#undef GNULIB_TEST_FSEEK + +/* Define to 1 when the gnulib module fseeko should be tested. */ +#undef GNULIB_TEST_FSEEKO + +/* Define to 1 when the gnulib module fstat should be tested. */ +#undef GNULIB_TEST_FSTAT + +/* Define to 1 when the gnulib module fstatat should be tested. */ +#undef GNULIB_TEST_FSTATAT + +/* Define to 1 when the gnulib module ftell should be tested. */ +#undef GNULIB_TEST_FTELL + +/* Define to 1 when the gnulib module ftello should be tested. */ +#undef GNULIB_TEST_FTELLO + +/* Define to 1 when the gnulib module futimens should be tested. */ +#undef GNULIB_TEST_FUTIMENS + +/* Define to 1 when the gnulib module getaddrinfo should be tested. */ +#undef GNULIB_TEST_GETADDRINFO + +/* Define to 1 when the gnulib module getcwd should be tested. */ +#undef GNULIB_TEST_GETCWD + +/* Define to 1 when the gnulib module getdelim should be tested. */ +#undef GNULIB_TEST_GETDELIM + +/* Define to 1 when the gnulib module getdtablesize should be tested. */ +#undef GNULIB_TEST_GETDTABLESIZE + +/* Define to 1 when the gnulib module getgroups should be tested. */ +#undef GNULIB_TEST_GETGROUPS + +/* Define to 1 when the gnulib module getline should be tested. */ +#undef GNULIB_TEST_GETLINE + +/* Define to 1 when the gnulib module getopt-posix should be tested. */ +#undef GNULIB_TEST_GETOPT_POSIX + +/* Define to 1 when the gnulib module getpass should be tested. */ +#undef GNULIB_TEST_GETPASS + +/* Define to 1 when the gnulib module getpeername should be tested. */ +#undef GNULIB_TEST_GETPEERNAME + +/* Define to 1 when the gnulib module getrandom should be tested. */ +#undef GNULIB_TEST_GETRANDOM + +/* Define to 1 when the gnulib module getsockname should be tested. */ +#undef GNULIB_TEST_GETSOCKNAME + +/* Define to 1 when the gnulib module gettimeofday should be tested. */ +#undef GNULIB_TEST_GETTIMEOFDAY + +/* Define to 1 when the gnulib module group-member should be tested. */ +#undef GNULIB_TEST_GROUP_MEMBER + +/* Define to 1 when the gnulib module ioctl should be tested. */ +#undef GNULIB_TEST_IOCTL + +/* Define to 1 when the gnulib module iswblank should be tested. */ +#undef GNULIB_TEST_ISWBLANK + +/* Define to 1 when the gnulib module iswdigit should be tested. */ +#undef GNULIB_TEST_ISWDIGIT + +/* Define to 1 when the gnulib module iswxdigit should be tested. */ +#undef GNULIB_TEST_ISWXDIGIT + +/* Define to 1 when the gnulib module link should be tested. */ +#undef GNULIB_TEST_LINK + +/* Define to 1 when the gnulib module listen should be tested. */ +#undef GNULIB_TEST_LISTEN + +/* Define to 1 when the gnulib module localeconv should be tested. */ +#undef GNULIB_TEST_LOCALECONV + +/* Define to 1 when the gnulib module lseek should be tested. */ +#undef GNULIB_TEST_LSEEK + +/* Define to 1 when the gnulib module lstat should be tested. */ +#undef GNULIB_TEST_LSTAT + +/* Define to 1 when the gnulib module malloc-posix should be tested. */ +#undef GNULIB_TEST_MALLOC_POSIX + +/* Define to 1 when the gnulib module mbrtowc should be tested. */ +#undef GNULIB_TEST_MBRTOWC + +/* Define to 1 when the gnulib module mbsinit should be tested. */ +#undef GNULIB_TEST_MBSINIT + +/* Define to 1 when the gnulib module mbsrtowcs should be tested. */ +#undef GNULIB_TEST_MBSRTOWCS + +/* Define to 1 when the gnulib module mbtowc should be tested. */ +#undef GNULIB_TEST_MBTOWC + +/* Define to 1 when the gnulib module memchr should be tested. */ +#undef GNULIB_TEST_MEMCHR + +/* Define to 1 when the gnulib module mempcpy should be tested. */ +#undef GNULIB_TEST_MEMPCPY + +/* Define to 1 when the gnulib module memrchr should be tested. */ +#undef GNULIB_TEST_MEMRCHR + +/* Define to 1 when the gnulib module mkdir should be tested. */ +#undef GNULIB_TEST_MKDIR + +/* Define to 1 when the gnulib module mkostemp should be tested. */ +#undef GNULIB_TEST_MKOSTEMP + +/* Define to 1 when the gnulib module mkstemp should be tested. */ +#undef GNULIB_TEST_MKSTEMP + +/* Define to 1 when the gnulib module mktime should be tested. */ +#undef GNULIB_TEST_MKTIME + +/* Define to 1 when the gnulib module nanosleep should be tested. */ +#undef GNULIB_TEST_NANOSLEEP + +/* Define to 1 when the gnulib module nl_langinfo should be tested. */ +#undef GNULIB_TEST_NL_LANGINFO + +/* Define to 1 when the gnulib module open should be tested. */ +#undef GNULIB_TEST_OPEN + +/* Define to 1 when the gnulib module openat should be tested. */ +#undef GNULIB_TEST_OPENAT + +/* Define to 1 when the gnulib module opendir should be tested. */ +#undef GNULIB_TEST_OPENDIR + +/* Define to 1 when the gnulib module pipe should be tested. */ +#undef GNULIB_TEST_PIPE + +/* Define to 1 when the gnulib module pipe2 should be tested. */ +#undef GNULIB_TEST_PIPE2 + +/* Define to 1 when the gnulib module posix_spawn should be tested. */ +#undef GNULIB_TEST_POSIX_SPAWN + +/* Define to 1 when the gnulib module posix_spawnattr_destroy should be + tested. */ +#undef GNULIB_TEST_POSIX_SPAWNATTR_DESTROY + +/* Define to 1 when the gnulib module posix_spawnattr_init should be tested. + */ +#undef GNULIB_TEST_POSIX_SPAWNATTR_INIT + +/* Define to 1 when the gnulib module posix_spawnattr_setflags should be + tested. */ +#undef GNULIB_TEST_POSIX_SPAWNATTR_SETFLAGS + +/* Define to 1 when the gnulib module posix_spawnattr_setpgroup should be + tested. */ +#undef GNULIB_TEST_POSIX_SPAWNATTR_SETPGROUP + +/* Define to 1 when the gnulib module posix_spawnattr_setsigmask should be + tested. */ +#undef GNULIB_TEST_POSIX_SPAWNATTR_SETSIGMASK + +/* Define to 1 when the gnulib module posix_spawnp should be tested. */ +#undef GNULIB_TEST_POSIX_SPAWNP + +/* Define to 1 when the gnulib module posix_spawn_file_actions_addchdir should + be tested. */ +#undef GNULIB_TEST_POSIX_SPAWN_FILE_ACTIONS_ADDCHDIR + +/* Define to 1 when the gnulib module posix_spawn_file_actions_addclose should + be tested. */ +#undef GNULIB_TEST_POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE + +/* Define to 1 when the gnulib module posix_spawn_file_actions_adddup2 should + be tested. */ +#undef GNULIB_TEST_POSIX_SPAWN_FILE_ACTIONS_ADDDUP2 + +/* Define to 1 when the gnulib module posix_spawn_file_actions_addopen should + be tested. */ +#undef GNULIB_TEST_POSIX_SPAWN_FILE_ACTIONS_ADDOPEN + +/* Define to 1 when the gnulib module posix_spawn_file_actions_destroy should + be tested. */ +#undef GNULIB_TEST_POSIX_SPAWN_FILE_ACTIONS_DESTROY + +/* Define to 1 when the gnulib module posix_spawn_file_actions_init should be + tested. */ +#undef GNULIB_TEST_POSIX_SPAWN_FILE_ACTIONS_INIT + +/* Define to 1 when the gnulib module raise should be tested. */ +#undef GNULIB_TEST_RAISE + +/* Define to 1 when the gnulib module rawmemchr should be tested. */ +#undef GNULIB_TEST_RAWMEMCHR + +/* Define to 1 when the gnulib module readdir should be tested. */ +#undef GNULIB_TEST_READDIR + +/* Define to 1 when the gnulib module readlink should be tested. */ +#undef GNULIB_TEST_READLINK + +/* Define to 1 when the gnulib module realloc-posix should be tested. */ +#undef GNULIB_TEST_REALLOC_POSIX + +/* Define to 1 when the gnulib module recv should be tested. */ +#undef GNULIB_TEST_RECV + +/* Define to 1 when the gnulib module rewinddir should be tested. */ +#undef GNULIB_TEST_REWINDDIR + +/* Define to 1 when the gnulib module secure_getenv should be tested. */ +#undef GNULIB_TEST_SECURE_GETENV + +/* Define to 1 when the gnulib module select should be tested. */ +#undef GNULIB_TEST_SELECT + +/* Define to 1 when the gnulib module send should be tested. */ +#undef GNULIB_TEST_SEND + +/* Define to 1 when the gnulib module setlocale_null should be tested. */ +#undef GNULIB_TEST_SETLOCALE_NULL + +/* Define to 1 when the gnulib module setsockopt should be tested. */ +#undef GNULIB_TEST_SETSOCKOPT + +/* Define to 1 when the gnulib module sigaction should be tested. */ +#undef GNULIB_TEST_SIGACTION + +/* Define to 1 when the gnulib module sigprocmask should be tested. */ +#undef GNULIB_TEST_SIGPROCMASK + +/* Define to 1 when the gnulib module snprintf should be tested. */ +#undef GNULIB_TEST_SNPRINTF + +/* Define to 1 when the gnulib module socket should be tested. */ +#undef GNULIB_TEST_SOCKET + +/* Define to 1 when the gnulib module stat should be tested. */ +#undef GNULIB_TEST_STAT + +/* Define to 1 when the gnulib module stpcpy should be tested. */ +#undef GNULIB_TEST_STPCPY + +/* Define to 1 when the gnulib module strchrnul should be tested. */ +#undef GNULIB_TEST_STRCHRNUL + +/* Define to 1 when the gnulib module strdup should be tested. */ +#undef GNULIB_TEST_STRDUP + +/* Define to 1 when the gnulib module strerror should be tested. */ +#undef GNULIB_TEST_STRERROR + +/* Define to 1 when the gnulib module strerror_r should be tested. */ +#undef GNULIB_TEST_STRERROR_R + +/* Define to 1 when the gnulib module strndup should be tested. */ +#undef GNULIB_TEST_STRNDUP + +/* Define to 1 when the gnulib module strnlen should be tested. */ +#undef GNULIB_TEST_STRNLEN + +/* Define to 1 when the gnulib module strpbrk should be tested. */ +#undef GNULIB_TEST_STRPBRK + +/* Define to 1 when the gnulib module strptime should be tested. */ +#undef GNULIB_TEST_STRPTIME + +/* Define to 1 when the gnulib module strtok_r should be tested. */ +#undef GNULIB_TEST_STRTOK_R + +/* Define to 1 when the gnulib module strtoll should be tested. */ +#undef GNULIB_TEST_STRTOLL + +/* Define to 1 when the gnulib module symlink should be tested. */ +#undef GNULIB_TEST_SYMLINK + +/* Define to 1 when the gnulib module timegm should be tested. */ +#undef GNULIB_TEST_TIMEGM + +/* Define to 1 when the gnulib module time_r should be tested. */ +#undef GNULIB_TEST_TIME_R + +/* Define to 1 when the gnulib module uninorm/u8-normalize should be tested. + */ +#undef GNULIB_TEST_UNINORM_U8_NORMALIZE + +/* Define to 1 when the gnulib module unlink should be tested. */ +#undef GNULIB_TEST_UNLINK + +/* Define to 1 when the gnulib module utime should be tested. */ +#undef GNULIB_TEST_UTIME + +/* Define to 1 when the gnulib module vasprintf should be tested. */ +#undef GNULIB_TEST_VASPRINTF + +/* Define to 1 when the gnulib module vsnprintf should be tested. */ +#undef GNULIB_TEST_VSNPRINTF + +/* Define to 1 when the gnulib module waitpid should be tested. */ +#undef GNULIB_TEST_WAITPID + +/* Define to 1 when the gnulib module wcrtomb should be tested. */ +#undef GNULIB_TEST_WCRTOMB + +/* Define to 1 when the gnulib module wcwidth should be tested. */ +#undef GNULIB_TEST_WCWIDTH + +/* Define to 1 when the gnulib module wmemchr should be tested. */ +#undef GNULIB_TEST_WMEMCHR + +/* Define to 1 when the gnulib module wmempcpy should be tested. */ +#undef GNULIB_TEST_WMEMPCPY + +/* Define to 1 when the gnulib module write should be tested. */ +#undef GNULIB_TEST_WRITE + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module unistr/u8-mbtouc-unsafe shall be considered + present. */ +#undef GNULIB_UNISTR_U8_MBTOUC_UNSAFE + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module unistr/u8-uctomb shall be considered present. */ +#undef GNULIB_UNISTR_U8_UCTOMB + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module xalloc shall be considered present. */ +#undef GNULIB_XALLOC + +/* Define to a C preprocessor expression that evaluates to 1 or 0, depending + whether the gnulib module xalloc-die shall be considered present. */ +#undef GNULIB_XALLOC_DIE + +/* Define to 1 if you have 'alloca' after including <alloca.h>, a header that + may be supplied by this distribution. */ +#undef HAVE_ALLOCA + +/* Define to 1 if <alloca.h> works. */ +#undef HAVE_ALLOCA_H + +/* Define to 1 if you have the <arpa/inet.h> header file. */ +#undef HAVE_ARPA_INET_H + +/* Define to 1 if you have the <bcrypt.h> header file. */ +#undef HAVE_BCRYPT_H + +/* Define to 1 if you have the <bp-sym.h> header file. */ +#undef HAVE_BP_SYM_H + +/* Define to 1 if you have the `btowc' function. */ +#undef HAVE_BTOWC + +/* Define to 1 if nanosleep mishandles large arguments. */ +#undef HAVE_BUG_BIG_NANOSLEEP + +/* Define to 1 if you have the <byteswap.h> header file. */ +#undef HAVE_BYTESWAP_H + +/* Define to 1 if you have the `canonicalize_file_name' function. */ +#undef HAVE_CANONICALIZE_FILE_NAME + +/* Define to 1 if you have the `catgets' function. */ +#undef HAVE_CATGETS + +/* Define to 1 if you have the Mac OS X function CFLocaleCopyCurrent in the + CoreFoundation framework. */ +#undef HAVE_CFLOCALECOPYCURRENT + +/* Define to 1 if you have the Mac OS X function CFPreferencesCopyAppValue in + the CoreFoundation framework. */ +#undef HAVE_CFPREFERENCESCOPYAPPVALUE + +/* Define to 1 if you have the `clock_gettime' function. */ +#undef HAVE_CLOCK_GETTIME + +/* Define to 1 if you have the `clock_settime' function. */ +#undef HAVE_CLOCK_SETTIME + +/* Define to 1 if you have the `closedir' function. */ +#undef HAVE_CLOSEDIR + +/* Define to 1 if you have the `confstr' function. */ +#undef HAVE_CONFSTR + +/* Define to 1 if you have the <crtdefs.h> header file. */ +#undef HAVE_CRTDEFS_H + +/* Define if the GNU dcgettext() function is already present or preinstalled. + */ +#undef HAVE_DCGETTEXT + +/* Define to 1 if you have the declaration of `alarm', and to 0 if you don't. + */ +#undef HAVE_DECL_ALARM + +/* Define to 1 if you have the declaration of `clearerr_unlocked', and to 0 if + you don't. */ +#undef HAVE_DECL_CLEARERR_UNLOCKED + +/* Define to 1 if you have the declaration of `dirfd', and to 0 if you don't. + */ +#undef HAVE_DECL_DIRFD + +/* Define to 1 if you have the declaration of `ecvt', and to 0 if you don't. + */ +#undef HAVE_DECL_ECVT + +/* Define to 1 if you have the declaration of `execvpe', and to 0 if you + don't. */ +#undef HAVE_DECL_EXECVPE + +/* Define to 1 if you have the declaration of `fchdir', and to 0 if you don't. + */ +#undef HAVE_DECL_FCHDIR + +/* Define to 1 if you have the declaration of `fcloseall', and to 0 if you + don't. */ +#undef HAVE_DECL_FCLOSEALL + +/* Define to 1 if you have the declaration of `fcvt', and to 0 if you don't. + */ +#undef HAVE_DECL_FCVT + +/* Define to 1 if you have the declaration of `fdopendir', and to 0 if you + don't. */ +#undef HAVE_DECL_FDOPENDIR + +/* Define to 1 if you have the declaration of `feof_unlocked', and to 0 if you + don't. */ +#undef HAVE_DECL_FEOF_UNLOCKED + +/* Define to 1 if you have the declaration of `ferror_unlocked', and to 0 if + you don't. */ +#undef HAVE_DECL_FERROR_UNLOCKED + +/* Define to 1 if you have the declaration of `fflush_unlocked', and to 0 if + you don't. */ +#undef HAVE_DECL_FFLUSH_UNLOCKED + +/* Define to 1 if you have the declaration of `fgets_unlocked', and to 0 if + you don't. */ +#undef HAVE_DECL_FGETS_UNLOCKED + +/* Define to 1 if you have the declaration of `flockfile', and to 0 if you + don't. */ +#undef HAVE_DECL_FLOCKFILE + +/* Define to 1 if you have the declaration of `fpurge', and to 0 if you don't. + */ +#undef HAVE_DECL_FPURGE + +/* Define to 1 if you have the declaration of `fputc_unlocked', and to 0 if + you don't. */ +#undef HAVE_DECL_FPUTC_UNLOCKED + +/* Define to 1 if you have the declaration of `fputs_unlocked', and to 0 if + you don't. */ +#undef HAVE_DECL_FPUTS_UNLOCKED + +/* Define to 1 if you have the declaration of `fread_unlocked', and to 0 if + you don't. */ +#undef HAVE_DECL_FREAD_UNLOCKED + +/* Define to 1 if you have the declaration of `freeaddrinfo', and to 0 if you + don't. */ +#undef HAVE_DECL_FREEADDRINFO + +/* Define to 1 if you have the declaration of `fseeko', and to 0 if you don't. + */ +#undef HAVE_DECL_FSEEKO + +/* Define to 1 if you have the declaration of `ftello', and to 0 if you don't. + */ +#undef HAVE_DECL_FTELLO + +/* Define to 1 if you have the declaration of `funlockfile', and to 0 if you + don't. */ +#undef HAVE_DECL_FUNLOCKFILE + +/* Define to 1 if you have the declaration of `fwrite_unlocked', and to 0 if + you don't. */ +#undef HAVE_DECL_FWRITE_UNLOCKED + +/* Define to 1 if you have the declaration of `gai_strerror', and to 0 if you + don't. */ +#undef HAVE_DECL_GAI_STRERROR + +/* Define to 1 if you have the declaration of `gai_strerrorA', and to 0 if you + don't. */ +#undef HAVE_DECL_GAI_STRERRORA + +/* Define to 1 if you have the declaration of `gcvt', and to 0 if you don't. + */ +#undef HAVE_DECL_GCVT + +/* Define to 1 if you have the declaration of `getaddrinfo', and to 0 if you + don't. */ +#undef HAVE_DECL_GETADDRINFO + +/* Define to 1 if you have the declaration of `getchar_unlocked', and to 0 if + you don't. */ +#undef HAVE_DECL_GETCHAR_UNLOCKED + +/* Define to 1 if you have the declaration of `getcwd', and to 0 if you don't. + */ +#undef HAVE_DECL_GETCWD + +/* Define to 1 if you have the declaration of `getc_unlocked', and to 0 if you + don't. */ +#undef HAVE_DECL_GETC_UNLOCKED + +/* Define to 1 if you have the declaration of `getdelim', and to 0 if you + don't. */ +#undef HAVE_DECL_GETDELIM + +/* Define to 1 if you have the declaration of `getdtablesize', and to 0 if you + don't. */ +#undef HAVE_DECL_GETDTABLESIZE + +/* Define to 1 if you have the declaration of `getline', and to 0 if you + don't. */ +#undef HAVE_DECL_GETLINE + +/* Define to 1 if you have the declaration of `getnameinfo', and to 0 if you + don't. */ +#undef HAVE_DECL_GETNAMEINFO + +/* Define to 1 if you have the declaration of `h_errno', and to 0 if you + don't. */ +#undef HAVE_DECL_H_ERRNO + +/* Define to 1 if you have the declaration of `inet_ntop', and to 0 if you + don't. */ +#undef HAVE_DECL_INET_NTOP + +/* Define to 1 if you have the declaration of `isblank', and to 0 if you + don't. */ +#undef HAVE_DECL_ISBLANK + +/* Define to 1 if you have the declaration of `iswblank', and to 0 if you + don't. */ +#undef HAVE_DECL_ISWBLANK + +/* Define to 1 if you have the declaration of `localtime_r', and to 0 if you + don't. */ +#undef HAVE_DECL_LOCALTIME_R + +/* Define to 1 if you have the declaration of `mbrtowc', and to 0 if you + don't. */ +#undef HAVE_DECL_MBRTOWC + +/* Define to 1 if you have the declaration of `mbsinit', and to 0 if you + don't. */ +#undef HAVE_DECL_MBSINIT + +/* Define to 1 if you have the declaration of `mbsrtowcs', and to 0 if you + don't. */ +#undef HAVE_DECL_MBSRTOWCS + +/* Define to 1 if you have the declaration of `memrchr', and to 0 if you + don't. */ +#undef HAVE_DECL_MEMRCHR + +/* Define to 1 if you have the declaration of `program_invocation_name', and + to 0 if you don't. */ +#undef HAVE_DECL_PROGRAM_INVOCATION_NAME + +/* Define to 1 if you have the declaration of `program_invocation_short_name', + and to 0 if you don't. */ +#undef HAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME + +/* Define to 1 if you have the declaration of `putchar_unlocked', and to 0 if + you don't. */ +#undef HAVE_DECL_PUTCHAR_UNLOCKED + +/* Define to 1 if you have the declaration of `putc_unlocked', and to 0 if you + don't. */ +#undef HAVE_DECL_PUTC_UNLOCKED + +/* Define to 1 if you have the declaration of `snprintf', and to 0 if you + don't. */ +#undef HAVE_DECL_SNPRINTF + +/* Define to 1 if you have the declaration of `strdup', and to 0 if you don't. + */ +#undef HAVE_DECL_STRDUP + +/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you + don't. */ +#undef HAVE_DECL_STRERROR_R + +/* Define to 1 if you have the declaration of `strncasecmp', and to 0 if you + don't. */ +#undef HAVE_DECL_STRNCASECMP + +/* Define to 1 if you have the declaration of `strndup', and to 0 if you + don't. */ +#undef HAVE_DECL_STRNDUP + +/* Define to 1 if you have the declaration of `strnlen', and to 0 if you + don't. */ +#undef HAVE_DECL_STRNLEN + +/* Define to 1 if you have the declaration of `strtok_r', and to 0 if you + don't. */ +#undef HAVE_DECL_STRTOK_R + +/* Define to 1 if you have the declaration of `towlower', and to 0 if you + don't. */ +#undef HAVE_DECL_TOWLOWER + +/* Define to 1 if you have the declaration of `vsnprintf', and to 0 if you + don't. */ +#undef HAVE_DECL_VSNPRINTF + +/* Define to 1 if you have the declaration of `wcrtomb', and to 0 if you + don't. */ +#undef HAVE_DECL_WCRTOMB + +/* Define to 1 if you have the declaration of `wcsdup', and to 0 if you don't. + */ +#undef HAVE_DECL_WCSDUP + +/* Define to 1 if you have the declaration of `wcwidth', and to 0 if you + don't. */ +#undef HAVE_DECL_WCWIDTH + +/* Define to 1 if you have the declaration of `_fseeki64', and to 0 if you + don't. */ +#undef HAVE_DECL__FSEEKI64 + +/* Define to 1 if you have the declaration of `_snprintf', and to 0 if you + don't. */ +#undef HAVE_DECL__SNPRINTF + +/* Define to 1 if you have the declaration of `__argv', and to 0 if you don't. + */ +#undef HAVE_DECL___ARGV + +/* Define to 1 if you have the declaration of `__fsetlocking', and to 0 if you + don't. */ +#undef HAVE_DECL___FSETLOCKING + +/* Define to 1 if you have the <dirent.h> header file. */ +#undef HAVE_DIRENT_H + +/* Define to 1 if you have the `dirfd' function. */ +#undef HAVE_DIRFD + +/* Define to 1 if you have the <dlfcn.h> header file. */ +#undef HAVE_DLFCN_H + +/* Define to 1 if you have the `drand48' function. */ +#undef HAVE_DRAND48 + +/* Define if you have the declaration of environ. */ +#undef HAVE_ENVIRON_DECL + +/* Define to 1 if you have the `faccessat' function. */ +#undef HAVE_FACCESSAT + +/* Define to 1 if you have the `fchdir' function. */ +#undef HAVE_FCHDIR + +/* Define to 1 if you have the `fcntl' function. */ +#undef HAVE_FCNTL + +/* Define to 1 if you have the `fdopendir' function. */ +#undef HAVE_FDOPENDIR + +/* Define to 1 if you have the <features.h> header file. */ +#undef HAVE_FEATURES_H + +/* Define to 1 if you have the `flock' function. */ +#undef HAVE_FLOCK + +/* Define to 1 if you have the `flockfile' function. */ +#undef HAVE_FLOCKFILE + +/* Define to 1 if you have the `fmemopen' function. */ +#undef HAVE_FMEMOPEN + +/* Define to 1 if you have the `fnmatch' function. */ +#undef HAVE_FNMATCH + +/* Define to 1 if you have the <fnmatch.h> header file. */ +#undef HAVE_FNMATCH_H + +/* Define to 1 if you have the `fpurge' function. */ +#undef HAVE_FPURGE + +/* Define to 1 if fseeko (and presumably ftello) exists and is declared. */ +#undef HAVE_FSEEKO + +/* Define to 1 if you have the `fstatat' function. */ +#undef HAVE_FSTATAT + +/* Define to 1 if you have the `ftello' function. */ +#undef HAVE_FTELLO + +/* Define to 1 if you have the `funlockfile' function. */ +#undef HAVE_FUNLOCKFILE + +/* Define to 1 if you have the `futimens' function. */ +#undef HAVE_FUTIMENS + +/* Define to 1 if you have the `futimes' function. */ +#undef HAVE_FUTIMES + +/* Define to 1 if you have the `futimesat' function. */ +#undef HAVE_FUTIMESAT + +/* Define to 1 if getaddrinfo exists, or to 0 otherwise. */ +#undef HAVE_GETADDRINFO + +/* Define to 1 if getcwd works, but with shorter paths than is generally + tested with the replacement. */ +#undef HAVE_GETCWD_SHORTER + +/* Define to 1 if you have the `getdelim' function. */ +#undef HAVE_GETDELIM + +/* Define to 1 if you have the `getdtablesize' function. */ +#undef HAVE_GETDTABLESIZE + +/* Define to 1 if you have the `getegid' function. */ +#undef HAVE_GETEGID + +/* Define to 1 if you have the `geteuid' function. */ +#undef HAVE_GETEUID + +/* Define to 1 if you have the `getexecname' function. */ +#undef HAVE_GETEXECNAME + +/* Define to 1 if you have the `getgid' function. */ +#undef HAVE_GETGID + +/* Define to 1 if your system has a working `getgroups' function. */ +#undef HAVE_GETGROUPS + +/* Define to 1 if you have the `gethostbyname' function. */ +#undef HAVE_GETHOSTBYNAME + +/* Define to 1 if you have the <getopt.h> header file. */ +#undef HAVE_GETOPT_H + +/* Define to 1 if you have the `getopt_long_only' function. */ +#undef HAVE_GETOPT_LONG_ONLY + +/* Define to 1 if you have the `getpagesize' function. */ +#undef HAVE_GETPAGESIZE + +/* Define to 1 if you have the `getpass' function. */ +#undef HAVE_GETPASS + +/* Define to 1 if you have the `getprogname' function. */ +#undef HAVE_GETPROGNAME + +/* Define to 1 if you have the `getrandom' function. */ +#undef HAVE_GETRANDOM + +/* Define to 1 if you have the `getservbyname' function. */ +#undef HAVE_GETSERVBYNAME + +/* Define if the GNU gettext() function is already present or preinstalled. */ +#undef HAVE_GETTEXT + +/* Define to 1 if you have the `gettimeofday' function. */ +#undef HAVE_GETTIMEOFDAY + +/* Define to 1 if you have the `getuid' function. */ +#undef HAVE_GETUID + +/* Define to 1 if you have the `gnutls_priority_set_direct' function. */ +#undef HAVE_GNUTLS_PRIORITY_SET_DIRECT + +/* Define if GPGME is available. */ +#undef HAVE_GPGME + +/* Define if you have the iconv() function and it works. */ +#undef HAVE_ICONV + +/* Define to 1 if you have the <iconv.h> header file. */ +#undef HAVE_ICONV_H + +/* Define to 1 if you have the `inet_ntop' function. */ +#undef HAVE_INET_NTOP + +/* Define to 1 if the compiler supports one of the keywords 'inline', + '__inline__', '__inline' and effectively inlines functions marked as such. + */ +#undef HAVE_INLINE + +/* Define to 1 if the system has the type `int64_t'. */ +#undef HAVE_INT64_T + +/* Define if you have the 'intmax_t' type in <stdint.h> or <inttypes.h>. */ +#undef HAVE_INTMAX_T + +/* Define to 1 if the system has the type `intptr_t'. */ +#undef HAVE_INTPTR_T + +/* Define to 1 if you have the <inttypes.h> header file. */ +#undef HAVE_INTTYPES_H + +/* Define if <inttypes.h> exists, doesn't clash with <sys/types.h>, and + declares uintmax_t. */ +#undef HAVE_INTTYPES_H_WITH_UINTMAX + +/* Define to 1 if you have the `ioctl' function. */ +#undef HAVE_IOCTL + +/* Define to 1 if <sys/socket.h> defines AF_INET. */ +#undef HAVE_IPV4 + +/* Define to 1 if <sys/socket.h> defines AF_INET6. */ +#undef HAVE_IPV6 + +/* Define to 1 if you have the `isatty' function. */ +#undef HAVE_ISATTY + +/* Define to 1 if you have the `isblank' function. */ +#undef HAVE_ISBLANK + +/* Define to 1 if you have the `issetugid' function. */ +#undef HAVE_ISSETUGID + +/* Define to 1 if you have the `iswblank' function. */ +#undef HAVE_ISWBLANK + +/* Define to 1 if you have the `iswcntrl' function. */ +#undef HAVE_ISWCNTRL + +/* Define to 1 if you have the `iswctype' function. */ +#undef HAVE_ISWCTYPE + +/* Define if you have <langinfo.h> and nl_langinfo(CODESET). */ +#undef HAVE_LANGINFO_CODESET + +/* Define to 1 if you have the <langinfo.h> header file. */ +#undef HAVE_LANGINFO_H + +/* Define if libcares is available. */ +#undef HAVE_LIBCARES + +/* Define to 1 if you have the `dl' library (-ldl). */ +#undef HAVE_LIBDL + +/* Define to 1 if you have the `eay32' library (-leay32). */ +#undef HAVE_LIBEAY32 + +/* Define if you have the libgnutls library. */ +#undef HAVE_LIBGNUTLS + +/* Define to 1 if you have the <libintl.h> header file. */ +#undef HAVE_LIBINTL_H + +/* Define if libpcre is available. */ +#undef HAVE_LIBPCRE + +/* Define if libpcre2 is available. */ +#undef HAVE_LIBPCRE2 + +/* PSL support enabled */ +#undef HAVE_LIBPSL + +/* Define if you have the libssl library. */ +#undef HAVE_LIBSSL + +/* Define to 1 if you have the `ssl32' library (-lssl32). */ +#undef HAVE_LIBSSL32 + +/* Define if you have the libunistring library. */ +#undef HAVE_LIBUNISTRING + +/* Define if using libuuid. */ +#undef HAVE_LIBUUID + +/* Define if using zlib. */ +#undef HAVE_LIBZ + +/* Define to 1 if the bcrypt library is guaranteed to be present. */ +#undef HAVE_LIB_BCRYPT + +/* Define to 1 if you have the <limits.h> header file. */ +#undef HAVE_LIMITS_H + +/* Define to 1 if you have the `link' function. */ +#undef HAVE_LINK + +/* Define to 1 if you have 'struct sockaddr_alg' defined. */ +#undef HAVE_LINUX_IF_ALG_H + +/* Define to 1 if you have the `localtime_r' function. */ +#undef HAVE_LOCALTIME_R + +/* Define to 1 if the system has the type 'long long int'. */ +#undef HAVE_LONG_LONG_INT + +/* Define to 1 if you have the `lstat' function. */ +#undef HAVE_LSTAT + +/* Define to 1 if you have the `lutimes' function. */ +#undef HAVE_LUTIMES + +/* Define to 1 if you have the <malloc.h> header file. */ +#undef HAVE_MALLOC_H + +/* Define if the 'malloc' function is POSIX compliant. */ +#undef HAVE_MALLOC_POSIX + +/* Define to 1 if mmap()'s MAP_ANONYMOUS flag is available after including + config.h and <sys/mman.h>. */ +#undef HAVE_MAP_ANONYMOUS + +/* Define to 1 if you have the `mbrtowc' function. */ +#undef HAVE_MBRTOWC + +/* Define to 1 if you have the `mbsinit' function. */ +#undef HAVE_MBSINIT + +/* Define to 1 if you have the `mbsrtowcs' function. */ +#undef HAVE_MBSRTOWCS + +/* Define to 1 if <wchar.h> declares mbstate_t. */ +#undef HAVE_MBSTATE_T + +/* Define to 1 if you have the `mbtowc' function. */ +#undef HAVE_MBTOWC + +/* Define to 1 if you have the `mempcpy' function. */ +#undef HAVE_MEMPCPY + +/* Define to 1 if you have the `memrchr' function. */ +#undef HAVE_MEMRCHR + +/* Define if using metalink. */ +#undef HAVE_METALINK + +/* Define to 1 if getcwd minimally works, that is, its result can be trusted + when it succeeds. */ +#undef HAVE_MINIMALLY_WORKING_GETCWD + +/* Define to 1 if you have the <minix/config.h> header file. */ +#undef HAVE_MINIX_CONFIG_H + +/* Define to 1 if <limits.h> defines the MIN and MAX macros. */ +#undef HAVE_MINMAX_IN_LIMITS_H + +/* Define to 1 if <sys/param.h> defines the MIN and MAX macros. */ +#undef HAVE_MINMAX_IN_SYS_PARAM_H + +/* Define to 1 if you have the `mkostemp' function. */ +#undef HAVE_MKOSTEMP + +/* Define to 1 if you have the `mkstemp' function. */ +#undef HAVE_MKSTEMP + +/* Define to 1 if you have a working `mmap' system call. */ +#undef HAVE_MMAP + +/* Define to 1 if you have the `mprotect' function. */ +#undef HAVE_MPROTECT + +/* Define to 1 on MSVC platforms that have the "invalid parameter handler" + concept. */ +#undef HAVE_MSVC_INVALID_PARAMETER_HANDLER + +/* Define to 1 if you have the <netdb.h> header file. */ +#undef HAVE_NETDB_H + +/* Define to 1 if you have the <netinet/in.h> header file. */ +#undef HAVE_NETINET_IN_H + +/* Use libnettle */ +#undef HAVE_NETTLE + +/* Define to 1 if you have the `nl_langinfo' function. */ +#undef HAVE_NL_LANGINFO + +/* Define to 1 if you have the `openat' function. */ +#undef HAVE_OPENAT + +/* Define to 1 if you have the `opendir' function. */ +#undef HAVE_OPENDIR + +/* Define to 1 if libcrypto is used for MD5. */ +#undef HAVE_OPENSSL_MD5 + +/* Define to 1 if you have the <openssl/md5.h> header file. */ +#undef HAVE_OPENSSL_MD5_H + +/* Define to 1 if libcrypto is used for SHA1. */ +#undef HAVE_OPENSSL_SHA1 + +/* Define to 1 if libcrypto is used for SHA256. */ +#undef HAVE_OPENSSL_SHA256 + +/* Define to 1 if libcrypto is used for SHA512. */ +#undef HAVE_OPENSSL_SHA512 + +/* Define to 1 if you have the <openssl/sha.h> header file. */ +#undef HAVE_OPENSSL_SHA_H + +/* Define to 1 if getcwd works, except it sometimes fails when it shouldn't, + setting errno to ERANGE, ENAMETOOLONG, or ENOENT. */ +#undef HAVE_PARTLY_WORKING_GETCWD + +/* Define to 1 if you have the `pathconf' function. */ +#undef HAVE_PATHCONF + +/* Define to 1 if you have the <paths.h> header file. */ +#undef HAVE_PATHS_H + +/* Define to 1 if you have the `pipe' function. */ +#undef HAVE_PIPE + +/* Define to 1 if you have the `pipe2' function. */ +#undef HAVE_PIPE2 + +/* Define to 1 if you have the `posix_spawn' function. */ +#undef HAVE_POSIX_SPAWN + +/* Define to 1 if the system has the type `posix_spawnattr_t'. */ +#undef HAVE_POSIX_SPAWNATTR_T + +/* Define to 1 if you have the `posix_spawn_file_actions_addchdir' function. + */ +#undef HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCHDIR + +/* Define to 1 if you have the `posix_spawn_file_actions_addchdir_np' + function. */ +#undef HAVE_POSIX_SPAWN_FILE_ACTIONS_ADDCHDIR_NP + +/* Define to 1 if the system has the type `posix_spawn_file_actions_t'. */ +#undef HAVE_POSIX_SPAWN_FILE_ACTIONS_T + +/* Define to 1 if you have the `psl_latest' function. */ +#undef HAVE_PSL_LATEST + +/* Define if you have the <pthread.h> header and the POSIX threads API. */ +#undef HAVE_PTHREAD_API + +/* Define if the <pthread.h> defines PTHREAD_MUTEX_RECURSIVE. */ +#undef HAVE_PTHREAD_MUTEX_RECURSIVE + +/* Define if the POSIX multithreading library has read/write locks. */ +#undef HAVE_PTHREAD_RWLOCK + +/* Define if the 'pthread_rwlock_rdlock' function prefers a writer to a + reader. */ +#undef HAVE_PTHREAD_RWLOCK_RDLOCK_PREFER_WRITER + +/* Define to 1 if you have the <pwd.h> header file. */ +#undef HAVE_PWD_H + +/* Define to 1 if you have the `raise' function. */ +#undef HAVE_RAISE + +/* Define to 1 if you have the `random' function. */ +#undef HAVE_RANDOM + +/* Define to 1 if you have the `RAND_egd' function. */ +#undef HAVE_RAND_EGD + +/* Define to 1 if you have the `rawmemchr' function. */ +#undef HAVE_RAWMEMCHR + +/* Define to 1 if you have the `readdir' function. */ +#undef HAVE_READDIR + +/* Define to 1 if you have the `readlink' function. */ +#undef HAVE_READLINK + +/* Define if the 'realloc' function is POSIX compliant. */ +#undef HAVE_REALLOC_POSIX + +/* Define to 1 if you have the `realpath' function. */ +#undef HAVE_REALPATH + +/* Define to 1 if you have the `rewinddir' function. */ +#undef HAVE_REWINDDIR + +/* Define to 1 if the system has the type `sa_family_t'. */ +#undef HAVE_SA_FAMILY_T + +/* Define to 1 if you have the <sched.h> header file. */ +#undef HAVE_SCHED_H + +/* Define to 1 if you have the `sched_setparam' function. */ +#undef HAVE_SCHED_SETPARAM + +/* Define to 1 if you have the `sched_setscheduler' function. */ +#undef HAVE_SCHED_SETSCHEDULER + +/* Define to 1 if you have the <sdkddkver.h> header file. */ +#undef HAVE_SDKDDKVER_H + +/* Define to 1 if you have the `secure_getenv' function. */ +#undef HAVE_SECURE_GETENV + +/* Define to 1 if you have the `setdtablesize' function. */ +#undef HAVE_SETDTABLESIZE + +/* Define to 1 if you have the `setegid' function. */ +#undef HAVE_SETEGID + +/* Define to 1 if you have the `seteuid' function. */ +#undef HAVE_SETEUID + +/* Define to 1 if you have the `shutdown' function. */ +#undef HAVE_SHUTDOWN + +/* Define to 1 if you have the `sigaction' function. */ +#undef HAVE_SIGACTION + +/* Define to 1 if you have the `sigaltstack' function. */ +#undef HAVE_SIGALTSTACK + +/* Define to 1 if you have the `sigblock' function. */ +#undef HAVE_SIGBLOCK + +/* Define to 1 if the system has the type `siginfo_t'. */ +#undef HAVE_SIGINFO_T + +/* Define to 1 if you have the `siginterrupt' function. */ +#undef HAVE_SIGINTERRUPT + +/* Define to 1 if 'sig_atomic_t' is a signed integer type. */ +#undef HAVE_SIGNED_SIG_ATOMIC_T + +/* Define to 1 if 'wchar_t' is a signed integer type. */ +#undef HAVE_SIGNED_WCHAR_T + +/* Define to 1 if 'wint_t' is a signed integer type. */ +#undef HAVE_SIGNED_WINT_T + +/* Define to 1 if you have the `sigsetjmp' function. */ +#undef HAVE_SIGSETJMP + +/* Define to 1 if the system has the type `sigset_t'. */ +#undef HAVE_SIGSET_T + +/* Define to 1 if the system has the type `sig_atomic_t'. */ +#undef HAVE_SIG_ATOMIC_T + +/* Define to 1 if you have the `sleep' function. */ +#undef HAVE_SLEEP + +/* Define to 1 if you have the `snprintf' function. */ +#undef HAVE_SNPRINTF + +/* Define if the return value of the snprintf function is the number of of + bytes (excluding the terminating NUL) that would have been produced if the + buffer had been large enough. */ +#undef HAVE_SNPRINTF_RETVAL_C99 + +/* Define if the string produced by the snprintf function is always NUL + terminated. */ +#undef HAVE_SNPRINTF_TRUNCATION_C99 + +/* Define if struct sockaddr_in6 has the sin6_scope_id member */ +#undef HAVE_SOCKADDR_IN6_SCOPE_ID + +/* Define to 1 if you have the <spawn.h> header file. */ +#undef HAVE_SPAWN_H + +/* Define to 1 if stdbool.h conforms to C99. */ +#undef HAVE_STDBOOL_H + +/* Define to 1 if you have the <stdint.h> header file. */ +#undef HAVE_STDINT_H + +/* Define if <stdint.h> exists, doesn't clash with <sys/types.h>, and declares + uintmax_t. */ +#undef HAVE_STDINT_H_WITH_UINTMAX + +/* Define to 1 if you have the <stdio_ext.h> header file. */ +#undef HAVE_STDIO_EXT_H + +/* Define to 1 if you have the <stdio.h> header file. */ +#undef HAVE_STDIO_H + +/* Define to 1 if you have the <stdlib.h> header file. */ +#undef HAVE_STDLIB_H + +/* Define to 1 if you have the `stpcpy' function. */ +#undef HAVE_STPCPY + +/* Define to 1 if you have the `strcasecmp' function. */ +#undef HAVE_STRCASECMP + +/* Define to 1 if you have the `strchrnul' function. */ +#undef HAVE_STRCHRNUL + +/* Define to 1 if you have the `strdup' function. */ +#undef HAVE_STRDUP + +/* Define to 1 if you have the `strerror_r' function. */ +#undef HAVE_STRERROR_R + +/* Define to 1 if you have the <strings.h> header file. */ +#undef HAVE_STRINGS_H + +/* Define to 1 if you have the <string.h> header file. */ +#undef HAVE_STRING_H + +/* Define to 1 if you have the `strlcpy' function. */ +#undef HAVE_STRLCPY + +/* Define to 1 if you have the `strncasecmp' function. */ +#undef HAVE_STRNCASECMP + +/* Define to 1 if you have the `strndup' function. */ +#undef HAVE_STRNDUP + +/* Define to 1 if you have the `strnlen' function. */ +#undef HAVE_STRNLEN + +/* Define to 1 if you have the `strpbrk' function. */ +#undef HAVE_STRPBRK + +/* Define to 1 if you have the `strptime' function. */ +#undef HAVE_STRPTIME + +/* Define to 1 if you have the `strtok_r' function. */ +#undef HAVE_STRTOK_R + +/* Define to 1 if you have the `strtol' function. */ +#undef HAVE_STRTOL + +/* Define to 1 if you have the `strtoll' function. */ +#undef HAVE_STRTOLL + +/* Define to 1 if the system has the type `struct addrinfo'. */ +#undef HAVE_STRUCT_ADDRINFO + +/* Define to 1 if `l_type' is a member of `struct flock'. */ +#undef HAVE_STRUCT_FLOCK_L_TYPE + +/* Define to 1 if `decimal_point' is a member of `struct lconv'. */ +#undef HAVE_STRUCT_LCONV_DECIMAL_POINT + +/* Define to 1 if `sa_sigaction' is a member of `struct sigaction'. */ +#undef HAVE_STRUCT_SIGACTION_SA_SIGACTION + +/* Define to 1 if the system has the type `struct sockaddr_in6'. */ +#undef HAVE_STRUCT_SOCKADDR_IN6 + +/* Define to 1 if `sa_len' is a member of `struct sockaddr'. */ +#undef HAVE_STRUCT_SOCKADDR_SA_LEN + +/* Define to 1 if the system has the type `struct sockaddr_storage'. */ +#undef HAVE_STRUCT_SOCKADDR_STORAGE + +/* Define to 1 if `ss_family' is a member of `struct sockaddr_storage'. */ +#undef HAVE_STRUCT_SOCKADDR_STORAGE_SS_FAMILY + +/* Define to 1 if `st_atimensec' is a member of `struct stat'. */ +#undef HAVE_STRUCT_STAT_ST_ATIMENSEC + +/* Define to 1 if `st_atimespec.tv_nsec' is a member of `struct stat'. */ +#undef HAVE_STRUCT_STAT_ST_ATIMESPEC_TV_NSEC + +/* Define to 1 if `st_atim.st__tim.tv_nsec' is a member of `struct stat'. */ +#undef HAVE_STRUCT_STAT_ST_ATIM_ST__TIM_TV_NSEC + +/* Define to 1 if `st_atim.tv_nsec' is a member of `struct stat'. */ +#undef HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC + +/* Define to 1 if `st_birthtimensec' is a member of `struct stat'. */ +#undef HAVE_STRUCT_STAT_ST_BIRTHTIMENSEC + +/* Define to 1 if `st_birthtimespec.tv_nsec' is a member of `struct stat'. */ +#undef HAVE_STRUCT_STAT_ST_BIRTHTIMESPEC_TV_NSEC + +/* Define to 1 if `st_birthtim.tv_nsec' is a member of `struct stat'. */ +#undef HAVE_STRUCT_STAT_ST_BIRTHTIM_TV_NSEC + +/* Define to 1 if you have the `symlink' function. */ +#undef HAVE_SYMLINK + +/* Define to 1 if you have the <sys/bitypes.h> header file. */ +#undef HAVE_SYS_BITYPES_H + +/* Define to 1 if you have the <sys/cdefs.h> header file. */ +#undef HAVE_SYS_CDEFS_H + +/* Define to 1 if you have the <sys/file.h> header file. */ +#undef HAVE_SYS_FILE_H + +/* Define to 1 if you have the <sys/inttypes.h> header file. */ +#undef HAVE_SYS_INTTYPES_H + +/* Define to 1 if you have the <sys/ioctl.h> header file. */ +#undef HAVE_SYS_IOCTL_H + +/* Define to 1 if you have the <sys/mman.h> header file. */ +#undef HAVE_SYS_MMAN_H + +/* Define to 1 if you have the <sys/param.h> header file. */ +#undef HAVE_SYS_PARAM_H + +/* Define to 1 if you have the <sys/random.h> header file. */ +#undef HAVE_SYS_RANDOM_H + +/* Define to 1 if you have the <sys/select.h> header file. */ +#undef HAVE_SYS_SELECT_H + +/* Define to 1 if you have the <sys/single_threaded.h> header file. */ +#undef HAVE_SYS_SINGLE_THREADED_H + +/* Define to 1 if you have the <sys/socket.h> header file. */ +#undef HAVE_SYS_SOCKET_H + +/* Define to 1 if you have the <sys/stat.h> header file. */ +#undef HAVE_SYS_STAT_H + +/* Define to 1 if you have the <sys/time.h> header file. */ +#undef HAVE_SYS_TIME_H + +/* Define to 1 if you have the <sys/types.h> header file. */ +#undef HAVE_SYS_TYPES_H + +/* Define to 1 if you have the <sys/uio.h> header file. */ +#undef HAVE_SYS_UIO_H + +/* Define to 1 if you have the <sys/utime.h> header file. */ +#undef HAVE_SYS_UTIME_H + +/* Define to 1 if you have the <sys/wait.h> header file. */ +#undef HAVE_SYS_WAIT_H + +/* Define to 1 if the system has the 'tcgetattr' function. */ +#undef HAVE_TCGETATTR + +/* Define to 1 if the system has the 'tcsetattr' function. */ +#undef HAVE_TCSETATTR + +/* Define to 1 if you have the <termios.h> header file. */ +#undef HAVE_TERMIOS_H + +/* Define to 1 if you have the `thrd_create' function. */ +#undef HAVE_THRD_CREATE + +/* Define to 1 if you have the <threads.h> header file. */ +#undef HAVE_THREADS_H + +/* Define to 1 if you have the `timegm' function. */ +#undef HAVE_TIMEGM + +/* Define if struct tm has the tm_gmtoff member. */ +#undef HAVE_TM_GMTOFF + +/* Define to 1 if you have the `towlower' function. */ +#undef HAVE_TOWLOWER + +/* Define to 1 if the system has the type `uint32_t'. */ +#undef HAVE_UINT32_T + +/* Define to 1 if the system has the type `uintptr_t'. */ +#undef HAVE_UINTPTR_T + +/* Define to 1 if you have the <unistd.h> header file. */ +#undef HAVE_UNISTD_H + +/* Define to 1 if the system has the type 'unsigned long long int'. */ +#undef HAVE_UNSIGNED_LONG_LONG_INT + +/* Define to 1 if you have the `usleep' function. */ +#undef HAVE_USLEEP + +/* Define to 1 if you have the `utime' function. */ +#undef HAVE_UTIME + +/* Define to 1 if you have the `utimensat' function. */ +#undef HAVE_UTIMENSAT + +/* Define to 1 if you have the <utime.h> header file. */ +#undef HAVE_UTIME_H + +/* Define if uuid_create is available. */ +#undef HAVE_UUID_CREATE + +/* Define if you have a global __progname variable */ +#undef HAVE_VAR___PROGNAME + +/* Define to 1 if you have the `vasnprintf' function. */ +#undef HAVE_VASNPRINTF + +/* Define to 1 if you have the `vasprintf' function. */ +#undef HAVE_VASPRINTF + +/* Define to 1 if you have the `vfork' function. */ +#undef HAVE_VFORK + +/* Define to 1 or 0, depending whether the compiler supports simple visibility + declarations. */ +#undef HAVE_VISIBILITY + +/* Define to 1 if you have the `vsnprintf' function. */ +#undef HAVE_VSNPRINTF + +/* Define to 1 if you have the `waitid' function. */ +#undef HAVE_WAITID + +/* Define to 1 if you have the <wchar.h> header file. */ +#undef HAVE_WCHAR_H + +/* Define if you have the 'wchar_t' type. */ +#undef HAVE_WCHAR_T + +/* Define to 1 if you have the `wcrtomb' function. */ +#undef HAVE_WCRTOMB + +/* Define to 1 if you have the `wcslen' function. */ +#undef HAVE_WCSLEN + +/* Define to 1 if you have the `wcsnlen' function. */ +#undef HAVE_WCSNLEN + +/* Define to 1 if you have the <wctype.h> header file. */ +#undef HAVE_WCTYPE_H + +/* Define to 1 if you have the `wcwidth' function. */ +#undef HAVE_WCWIDTH + +/* Define to 1 if the compiler and linker support weak declarations of + symbols. */ +#undef HAVE_WEAK_SYMBOLS + +/* Define to 1 if you have the <winsock2.h> header file. */ +#undef HAVE_WINSOCK2_H + +/* Define if you have the 'wint_t' type. */ +#undef HAVE_WINT_T + +/* Define to 1 if you have the `wmempcpy' function. */ +#undef HAVE_WMEMPCPY + +/* Define to 1 if fstatat (..., 0) works. For example, it does not work in AIX + 7.1. */ +#undef HAVE_WORKING_FSTATAT_ZERO_FLAG + +/* Define to 1 if O_NOATIME works. */ +#undef HAVE_WORKING_O_NOATIME + +/* Define to 1 if O_NOFOLLOW works. */ +#undef HAVE_WORKING_O_NOFOLLOW + +/* Define if utimes works properly. */ +#undef HAVE_WORKING_UTIMES + +/* Define to 1 if you have the <ws2tcpip.h> header file. */ +#undef HAVE_WS2TCPIP_H + +/* Define to 1 if you have the <xlocale.h> header file. */ +#undef HAVE_XLOCALE_H + +/* Define to 1 if the system has the type `_Bool'. */ +#undef HAVE__BOOL + +/* Define to 1 if you have the `_fseeki64' function. */ +#undef HAVE__FSEEKI64 + +/* Define to 1 if you have the `_ftelli64' function. */ +#undef HAVE__FTELLI64 + +/* Define to 1 if you have the `_set_invalid_parameter_handler' function. */ +#undef HAVE__SET_INVALID_PARAMETER_HANDLER + +/* Define to 1 if the compiler supports __builtin_expect, + and to 2 if <builtins.h> does. */ +#undef HAVE___BUILTIN_EXPECT +#ifndef HAVE___BUILTIN_EXPECT +# define __builtin_expect(e, c) (e) +#elif HAVE___BUILTIN_EXPECT == 2 +# include <builtins.h> +#endif + + +/* Define to 1 if you have the `__fpurge' function. */ +#undef HAVE___FPURGE + +/* Define to 1 if you have the `__freading' function. */ +#undef HAVE___FREADING + +/* Define to 1 if you have the `__fsetlocking' function. */ +#undef HAVE___FSETLOCKING + +/* Define to 1 if the compiler supports the keyword '__inline'. */ +#undef HAVE___INLINE + +/* Define to 1 if you have the `__secure_getenv' function. */ +#undef HAVE___SECURE_GETENV + +/* Define to 1 if you have the `__xpg_strerror_r' function. */ +#undef HAVE___XPG_STRERROR_R + +/* Define as const if the declaration of iconv() needs const. */ +#undef ICONV_CONST + +/* Define to 1 if lseek does not detect pipes. */ +#undef LSEEK_PIPE_BROKEN + +/* Define to 1 if 'lstat' dereferences a symlink specified with a trailing + slash. */ +#undef LSTAT_FOLLOWS_SLASHED_SYMLINK + +/* If malloc(0) is != NULL, define this to 1. Otherwise define this to 0. */ +#undef MALLOC_0_IS_NONNULL + +/* Define to a substitute value for mmap()'s MAP_ANONYMOUS flag. */ +#undef MAP_ANONYMOUS + +/* Define if the mbrtowc function does not return (size_t) -2 for empty input. + */ +#undef MBRTOWC_EMPTY_INPUT_BUG + +/* Define if the mbrtowc function may signal encoding errors in the C locale. + */ +#undef MBRTOWC_IN_C_LOCALE_MAYBE_EILSEQ + +/* Define if the mbrtowc function has the NULL pwc argument bug. */ +#undef MBRTOWC_NULL_ARG1_BUG + +/* Define if the mbrtowc function has the NULL string argument bug. */ +#undef MBRTOWC_NULL_ARG2_BUG + +/* Define if the mbrtowc function does not return 0 for a NUL character. */ +#undef MBRTOWC_NUL_RETVAL_BUG + +/* Define if the mbrtowc function returns a wrong return value. */ +#undef MBRTOWC_RETVAL_BUG + +/* Define if the mbrtowc function stores a wide character when reporting + incomplete input. */ +#undef MBRTOWC_STORES_INCOMPLETE_BUG + +/* Use GNU style printf and scanf. */ +#ifndef __USE_MINGW_ANSI_STDIO +# undef __USE_MINGW_ANSI_STDIO +#endif + + +/* Define if the compilation of mktime.c should define 'mktime_internal'. */ +#undef NEED_MKTIME_INTERNAL + +/* Define if the compilation of mktime.c should define 'mktime' with the + native Windows TZ workaround. */ +#undef NEED_MKTIME_WINDOWS + +/* Define if the compilation of mktime.c should define 'mktime' with the + algorithmic workarounds. */ +#undef NEED_MKTIME_WORKING + +/* Define to 1 if nl_langinfo is multithread-safe. */ +#undef NL_LANGINFO_MTSAFE + +/* Define to 1 if open() fails to recognize a trailing slash. */ +#undef OPEN_TRAILING_SLASH_BUG + +/* Define to be the name of the operating system. */ +#undef OS_TYPE + +/* Name of package */ +#undef PACKAGE + +/* Define to the address where bug reports for this package should be sent. */ +#undef PACKAGE_BUGREPORT + +/* Define to the full name of this package. */ +#undef PACKAGE_NAME + +/* Define to the full name and version of this package. */ +#undef PACKAGE_STRING + +/* Define to the one symbol short name of this package. */ +#undef PACKAGE_TARNAME + +/* Define to the home page for this package. */ +#undef PACKAGE_URL + +/* Define to the version of this package. */ +#undef PACKAGE_VERSION + +/* Define to the type that is the result of default argument promotions of + type mode_t. */ +#undef PROMOTED_MODE_T + +/* Define if the pthread_in_use() detection is hard. */ +#undef PTHREAD_IN_USE_DETECTION_HARD + +/* Define to l, ll, u, ul, ull, etc., as suitable for constants of type + 'ptrdiff_t'. */ +#undef PTRDIFF_T_SUFFIX + +/* Define to 1 if readlink fails to recognize a trailing slash. */ +#undef READLINK_TRAILING_SLASH_BUG + +/* Define to 1 if readlink sets errno instead of truncating a too-long link. + */ +#undef READLINK_TRUNCATE_BUG + +/* Define to 1 if gnulib's dirfd() replacement is used. */ +#undef REPLACE_DIRFD + +/* Define to 1 if gnulib's fchdir() replacement is used. */ +#undef REPLACE_FCHDIR + +/* Define to 1 if stat needs help when passed a file name with a trailing + slash */ +#undef REPLACE_FUNC_STAT_FILE + +/* Define to 1 if utime needs help when passed a file name with a trailing + slash */ +#undef REPLACE_FUNC_UTIME_FILE + +/* Define if nl_langinfo exists but is overridden by gnulib. */ +#undef REPLACE_NL_LANGINFO + +/* Define to 1 if open() should work around the inability to open a directory. + */ +#undef REPLACE_OPEN_DIRECTORY + +/* Define if gnulib uses its own posix_spawn and posix_spawnp functions. */ +#undef REPLACE_POSIX_SPAWN + +/* Define to 1 if strerror(0) does not return a message implying success. */ +#undef REPLACE_STRERROR_0 + +/* Define if vasnprintf exists but is overridden by gnulib. */ +#undef REPLACE_VASNPRINTF + +/* Define to 1 if setlocale (LC_ALL, NULL) is multithread-safe. */ +#undef SETLOCALE_NULL_ALL_MTSAFE + +/* Define to 1 if setlocale (category, NULL) is multithread-safe. */ +#undef SETLOCALE_NULL_ONE_MTSAFE + +/* File name of the Bourne shell. */ +#if (defined _WIN32 && !defined __CYGWIN__) || defined __CYGWIN__ || defined __ANDROID__ +/* Omit the directory part because + - For native Windows programs in a Cygwin environment, the Cygwin mounts + are not visible. + - For 32-bit Cygwin programs in a 64-bit Cygwin environment, the Cygwin + mounts are not visible. + - On Android, /bin/sh does not exist. It's /system/bin/sh instead. */ +# define BOURNE_SHELL "sh" +#else +# define BOURNE_SHELL "/bin/sh" +#endif + +/* Define to l, ll, u, ul, ull, etc., as suitable for constants of type + 'sig_atomic_t'. */ +#undef SIG_ATOMIC_T_SUFFIX + +/* The size of `long', as computed by sizeof. */ +#undef SIZEOF_LONG + +/* The size of `off_t', as computed by sizeof. */ +#undef SIZEOF_OFF_T + +/* Define as the maximum value of type 'size_t', if the system doesn't define + it. */ +#ifndef SIZE_MAX +# undef SIZE_MAX +#endif + +/* Define to l, ll, u, ul, ull, etc., as suitable for constants of type + 'size_t'. */ +#undef SIZE_T_SUFFIX + +/* If using the C implementation of alloca, define if you know the + direction of stack growth for your system; otherwise it will be + automatically deduced at runtime. + STACK_DIRECTION > 0 => grows toward higher addresses + STACK_DIRECTION < 0 => grows toward lower addresses + STACK_DIRECTION = 0 => direction of growth unknown */ +#undef STACK_DIRECTION + +/* Define to 1 if the `S_IS*' macros in <sys/stat.h> do not work properly. */ +#undef STAT_MACROS_BROKEN + +/* Define to 1 if all of the C90 standard headers exist (not just the ones + required in a freestanding environment). This macro is provided for + backward compatibility; new code need not use it. */ +#undef STDC_HEADERS + +/* Define to 1 if strerror_r returns char *. */ +#undef STRERROR_R_CHAR_P + +/* Define to 1 if time_t is signed. */ +#undef TIME_T_IS_SIGNED + +/* Define to 1 if the type of the st_atim member of a struct stat is struct + timespec. */ +#undef TYPEOF_STRUCT_STAT_ST_ATIM_IS_STRUCT_TIMESPEC + +/* Define to 1 if unlink() on a parent directory may succeed */ +#undef UNLINK_PARENT_BUG + +/* Define to the prefix of C symbols at the assembler and linker level, either + an underscore or empty. */ +#undef USER_LABEL_PREFIX + +/* Define if the combination of the ISO C and POSIX multithreading APIs can be + used. */ +#undef USE_ISOC_AND_POSIX_THREADS + +/* Define if the ISO C multithreading library can be used. */ +#undef USE_ISOC_THREADS + +/* Define to 1 if you want to use the Linux kernel cryptographic API. */ +#undef USE_LINUX_CRYPTO_API + +/* Define if the POSIX multithreading library can be used. */ +#undef USE_POSIX_THREADS + +/* Define if references to the POSIX multithreading library should be made + weak. */ +#undef USE_POSIX_THREADS_WEAK + +/* Enable extensions on AIX 3, Interix. */ +#ifndef _ALL_SOURCE +# undef _ALL_SOURCE +#endif +/* Enable general extensions on macOS. */ +#ifndef _DARWIN_C_SOURCE +# undef _DARWIN_C_SOURCE +#endif +/* Enable general extensions on Solaris. */ +#ifndef __EXTENSIONS__ +# undef __EXTENSIONS__ +#endif +/* Enable GNU extensions on systems that have them. */ +#ifndef _GNU_SOURCE +# undef _GNU_SOURCE +#endif +/* Enable X/Open compliant socket functions that do not require linking + with -lxnet on HP-UX 11.11. */ +#ifndef _HPUX_ALT_XOPEN_SOCKET_API +# undef _HPUX_ALT_XOPEN_SOCKET_API +#endif +/* Identify the host operating system as Minix. + This macro does not affect the system headers' behavior. + A future release of Autoconf may stop defining this macro. */ +#ifndef _MINIX +# undef _MINIX +#endif +/* Enable general extensions on NetBSD. + Enable NetBSD compatibility extensions on Minix. */ +#ifndef _NETBSD_SOURCE +# undef _NETBSD_SOURCE +#endif +/* Enable OpenBSD compatibility extensions on NetBSD. + Oddly enough, this does nothing on OpenBSD. */ +#ifndef _OPENBSD_SOURCE +# undef _OPENBSD_SOURCE +#endif +/* Define to 1 if needed for POSIX-compatible behavior. */ +#ifndef _POSIX_SOURCE +# undef _POSIX_SOURCE +#endif +/* Define to 2 if needed for POSIX-compatible behavior. */ +#ifndef _POSIX_1_SOURCE +# undef _POSIX_1_SOURCE +#endif +/* Enable POSIX-compatible threading on Solaris. */ +#ifndef _POSIX_PTHREAD_SEMANTICS +# undef _POSIX_PTHREAD_SEMANTICS +#endif +/* Enable extensions specified by ISO/IEC TS 18661-5:2014. */ +#ifndef __STDC_WANT_IEC_60559_ATTRIBS_EXT__ +# undef __STDC_WANT_IEC_60559_ATTRIBS_EXT__ +#endif +/* Enable extensions specified by ISO/IEC TS 18661-1:2014. */ +#ifndef __STDC_WANT_IEC_60559_BFP_EXT__ +# undef __STDC_WANT_IEC_60559_BFP_EXT__ +#endif +/* Enable extensions specified by ISO/IEC TS 18661-2:2015. */ +#ifndef __STDC_WANT_IEC_60559_DFP_EXT__ +# undef __STDC_WANT_IEC_60559_DFP_EXT__ +#endif +/* Enable extensions specified by ISO/IEC TS 18661-4:2015. */ +#ifndef __STDC_WANT_IEC_60559_FUNCS_EXT__ +# undef __STDC_WANT_IEC_60559_FUNCS_EXT__ +#endif +/* Enable extensions specified by ISO/IEC TS 18661-3:2015. */ +#ifndef __STDC_WANT_IEC_60559_TYPES_EXT__ +# undef __STDC_WANT_IEC_60559_TYPES_EXT__ +#endif +/* Enable extensions specified by ISO/IEC TR 24731-2:2010. */ +#ifndef __STDC_WANT_LIB_EXT2__ +# undef __STDC_WANT_LIB_EXT2__ +#endif +/* Enable extensions specified by ISO/IEC 24747:2009. */ +#ifndef __STDC_WANT_MATH_SPEC_FUNCS__ +# undef __STDC_WANT_MATH_SPEC_FUNCS__ +#endif +/* Enable extensions on HP NonStop. */ +#ifndef _TANDEM_SOURCE +# undef _TANDEM_SOURCE +#endif +/* Enable X/Open extensions. Define to 500 only if necessary + to make mbstate_t available. */ +#ifndef _XOPEN_SOURCE +# undef _XOPEN_SOURCE +#endif + + +/* Define to 1 if you want getc etc. to use unlocked I/O if available. + Unlocked I/O can improve performance in unithreaded apps, but it is not + safe for multithreaded apps. */ +#undef USE_UNLOCKED_IO + +/* Define if the native Windows multithreading API can be used. */ +#undef USE_WINDOWS_THREADS + +/* Version number of package */ +#undef VERSION + +/* Define to l, ll, u, ul, ull, etc., as suitable for constants of type + 'wchar_t'. */ +#undef WCHAR_T_SUFFIX + +/* Define if the wcrtomb function does not work in the C locale. */ +#undef WCRTOMB_C_LOCALE_BUG + +/* Define if the wcrtomb function has an incorrect return value. */ +#undef WCRTOMB_RETVAL_BUG + +/* Define if WSAStartup is needed. */ +#undef WINDOWS_SOCKETS + +/* Define to l, ll, u, ul, ull, etc., as suitable for constants of type + 'wint_t'. */ +#undef WINT_T_SUFFIX + +/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most + significant byte first (like Motorola and SPARC, unlike Intel). */ +#if defined AC_APPLE_UNIVERSAL_BUILD +# if defined __BIG_ENDIAN__ +# define WORDS_BIGENDIAN 1 +# endif +#else +# ifndef WORDS_BIGENDIAN +# undef WORDS_BIGENDIAN +# endif +#endif + +/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a + `char[]'. */ +#undef YYTEXT_POINTER + +/* Number of bits in a file offset, on hosts where this is settable. */ +#undef _FILE_OFFSET_BITS + +/* True if the compiler says it groks GNU C version MAJOR.MINOR. */ +#if defined __GNUC__ && defined __GNUC_MINOR__ +# define _GL_GNUC_PREREQ(major, minor) \ + ((major) < __GNUC__ + ((minor) <= __GNUC_MINOR__)) +#else +# define _GL_GNUC_PREREQ(major, minor) 0 +#endif + + +/* Define to 1 to make fseeko visible on some hosts (e.g. glibc 2.2). */ +#undef _LARGEFILE_SOURCE + +/* Define for large files, on AIX-style hosts. */ +#undef _LARGE_FILES + +/* Define to 1 on Solaris. */ +#undef _LCONV_C99 + +/* The _Noreturn keyword of C11. */ +#ifndef _Noreturn +# if (defined __cplusplus \ + && ((201103 <= __cplusplus && !(__GNUC__ == 4 && __GNUC_MINOR__ == 7)) \ + || (defined _MSC_VER && 1900 <= _MSC_VER)) \ + && 0) + /* [[noreturn]] is not practically usable, because with it the syntax + extern _Noreturn void func (...); + would not be valid; such a declaration would only be valid with 'extern' + and '_Noreturn' swapped, or without the 'extern' keyword. However, some + AIX system header files and several gnulib header files use precisely + this syntax with 'extern'. */ +# define _Noreturn [[noreturn]] +# elif ((!defined __cplusplus || defined __clang__) \ + && (201112 <= (defined __STDC_VERSION__ ? __STDC_VERSION__ : 0) \ + || _GL_GNUC_PREREQ (4, 7) \ + || (defined __apple_build_version__ \ + ? 6000000 <= __apple_build_version__ \ + : 3 < __clang_major__ + (5 <= __clang_minor__)))) + /* _Noreturn works as-is. */ +# elif _GL_GNUC_PREREQ (2, 8) || defined __clang__ || 0x5110 <= __SUNPRO_C +# define _Noreturn __attribute__ ((__noreturn__)) +# elif 1200 <= (defined _MSC_VER ? _MSC_VER : 0) +# define _Noreturn __declspec (noreturn) +# else +# define _Noreturn +# endif +#endif + + +/* Define to 1 in order to get the POSIX compatible declarations of socket + functions. */ +#undef _POSIX_PII_SOCKET + +/* Define if you want <regex.h> to include <limits.h>, so that it consistently + overrides <limits.h>'s RE_DUP_MAX. */ +#undef _REGEX_INCLUDE_LIMITS_H + +/* Define if you want regoff_t to be at least as wide POSIX requires. */ +#undef _REGEX_LARGE_OFFSETS + +/* For standard stat data types on VMS. */ +#undef _USE_STD_STAT + +/* Define to rpl_ if the getopt replacement functions and variables should be + used. */ +#undef __GETOPT_PREFIX + +/* Define to 1 if the system <stdint.h> predates C++11. */ +#undef __STDC_CONSTANT_MACROS + +/* Define to 1 if the system <stdint.h> predates C++11. */ +#undef __STDC_LIMIT_MACROS + +/* The _GL_ASYNC_SAFE marker should be attached to functions that are + signal handlers (for signals other than SIGABRT, SIGPIPE) or can be + invoked from such signal handlers. Such functions have some restrictions: + * All functions that it calls should be marked _GL_ASYNC_SAFE as well, + or should be listed as async-signal-safe in POSIX + <https://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_04> + section 2.4.3. Note that malloc(), sprintf(), and fwrite(), in + particular, are NOT async-signal-safe. + * All memory locations (variables and struct fields) that these functions + access must be marked 'volatile'. This holds for both read and write + accesses. Otherwise the compiler might optimize away stores to and + reads from such locations that occur in the program, depending on its + data flow analysis. For example, when the program contains a loop + that is intended to inspect a variable set from within a signal handler + while (!signal_occurred) + ; + the compiler is allowed to transform this into an endless loop if the + variable 'signal_occurred' is not declared 'volatile'. + Additionally, recall that: + * A signal handler should not modify errno (except if it is a handler + for a fatal signal and ends by raising the same signal again, thus + provoking the termination of the process). If it invokes a function + that may clobber errno, it needs to save and restore the value of + errno. */ +#define _GL_ASYNC_SAFE + + +/* Attributes. */ +#ifdef __has_attribute +# define _GL_HAS_ATTRIBUTE(attr) __has_attribute (__##attr##__) +#else +# define _GL_HAS_ATTRIBUTE(attr) _GL_ATTR_##attr +# define _GL_ATTR_alloc_size _GL_GNUC_PREREQ (4, 3) +# define _GL_ATTR_always_inline _GL_GNUC_PREREQ (3, 2) +# define _GL_ATTR_artificial _GL_GNUC_PREREQ (4, 3) +# define _GL_ATTR_cold _GL_GNUC_PREREQ (4, 3) +# define _GL_ATTR_const _GL_GNUC_PREREQ (2, 95) +# define _GL_ATTR_deprecated _GL_GNUC_PREREQ (3, 1) +# define _GL_ATTR_diagnose_if 0 +# define _GL_ATTR_error _GL_GNUC_PREREQ (4, 3) +# define _GL_ATTR_externally_visible _GL_GNUC_PREREQ (4, 1) +# define _GL_ATTR_fallthrough _GL_GNUC_PREREQ (7, 0) +# define _GL_ATTR_format _GL_GNUC_PREREQ (2, 7) +# define _GL_ATTR_leaf _GL_GNUC_PREREQ (4, 6) +# ifdef _ICC +# define _GL_ATTR_may_alias 0 +# else +# define _GL_ATTR_may_alias _GL_GNUC_PREREQ (3, 3) +# endif +# define _GL_ATTR_malloc _GL_GNUC_PREREQ (3, 0) +# define _GL_ATTR_noinline _GL_GNUC_PREREQ (3, 1) +# define _GL_ATTR_nonnull _GL_GNUC_PREREQ (3, 3) +# define _GL_ATTR_nonstring _GL_GNUC_PREREQ (8, 0) +# define _GL_ATTR_nothrow _GL_GNUC_PREREQ (3, 3) +# define _GL_ATTR_packed _GL_GNUC_PREREQ (2, 7) +# define _GL_ATTR_pure _GL_GNUC_PREREQ (2, 96) +# define _GL_ATTR_returns_nonnull _GL_GNUC_PREREQ (4, 9) +# define _GL_ATTR_sentinel _GL_GNUC_PREREQ (4, 0) +# define _GL_ATTR_unused _GL_GNUC_PREREQ (2, 7) +# define _GL_ATTR_warn_unused_result _GL_GNUC_PREREQ (3, 4) +#endif + + +#if _GL_HAS_ATTRIBUTE (alloc_size) +# define _GL_ATTRIBUTE_ALLOC_SIZE(args) __attribute__ ((__alloc_size__ args)) +#else +# define _GL_ATTRIBUTE_ALLOC_SIZE(args) +#endif + +#if _GL_HAS_ATTRIBUTE (always_inline) +# define _GL_ATTRIBUTE_ALWAYS_INLINE __attribute__ ((__always_inline__)) +#else +# define _GL_ATTRIBUTE_ALWAYS_INLINE +#endif + +#if _GL_HAS_ATTRIBUTE (artificial) +# define _GL_ATTRIBUTE_ARTIFICIAL __attribute__ ((__artificial__)) +#else +# define _GL_ATTRIBUTE_ARTIFICIAL +#endif + +/* Avoid __attribute__ ((cold)) on MinGW; see thread starting at + <https://lists.gnu.org/r/emacs-devel/2019-04/msg01152.html>. + Also, Oracle Studio 12.6 requires 'cold' not '__cold__'. */ +#if _GL_HAS_ATTRIBUTE (cold) && !defined __MINGW32__ +# ifndef __SUNPRO_C +# define _GL_ATTRIBUTE_COLD __attribute__ ((__cold__)) +# else +# define _GL_ATTRIBUTE_COLD __attribute__ ((cold)) +# endif +#else +# define _GL_ATTRIBUTE_COLD +#endif + +#if _GL_HAS_ATTRIBUTE (const) +# define _GL_ATTRIBUTE_CONST __attribute__ ((__const__)) +#else +# define _GL_ATTRIBUTE_CONST +#endif + +#if 201710L < __STDC_VERSION__ +# define _GL_ATTRIBUTE_DEPRECATED [[__deprecated__]] +#elif _GL_HAS_ATTRIBUTE (deprecated) +# define _GL_ATTRIBUTE_DEPRECATED __attribute__ ((__deprecated__)) +#else +# define _GL_ATTRIBUTE_DEPRECATED +#endif + +#if _GL_HAS_ATTRIBUTE (error) +# define _GL_ATTRIBUTE_ERROR(msg) __attribute__ ((__error__ (msg))) +# define _GL_ATTRIBUTE_WARNING(msg) __attribute__ ((__warning__ (msg))) +#elif _GL_HAS_ATTRIBUTE (diagnose_if) +# define _GL_ATTRIBUTE_ERROR(msg) __attribute__ ((__diagnose_if__ (1, msg, "error"))) +# define _GL_ATTRIBUTE_WARNING(msg) __attribute__ ((__diagnose_if__ (1, msg, "warning"))) +#else +# define _GL_ATTRIBUTE_ERROR(msg) +# define _GL_ATTRIBUTE_WARNING(msg) +#endif + +#if _GL_HAS_ATTRIBUTE (externally_visible) +# define _GL_ATTRIBUTE_EXTERNALLY_VISIBLE __attribute__ ((externally_visible)) +#else +# define _GL_ATTRIBUTE_EXTERNALLY_VISIBLE +#endif + +/* FALLTHROUGH is special, because it always expands to something. */ +#if 201710L < __STDC_VERSION__ +# define _GL_ATTRIBUTE_FALLTHROUGH [[__fallthrough__]] +#elif _GL_HAS_ATTRIBUTE (fallthrough) +# define _GL_ATTRIBUTE_FALLTHROUGH __attribute__ ((__fallthrough__)) +#else +# define _GL_ATTRIBUTE_FALLTHROUGH ((void) 0) +#endif + +#if _GL_HAS_ATTRIBUTE (format) +# define _GL_ATTRIBUTE_FORMAT(spec) __attribute__ ((__format__ spec)) +#else +# define _GL_ATTRIBUTE_FORMAT(spec) +#endif + +#if _GL_HAS_ATTRIBUTE (leaf) +# define _GL_ATTRIBUTE_LEAF __attribute__ ((__leaf__)) +#else +# define _GL_ATTRIBUTE_LEAF +#endif + +/* Oracle Studio 12.6 mishandles may_alias despite __has_attribute OK. */ +#if _GL_HAS_ATTRIBUTE (may_alias) && !defined __SUNPRO_C +# define _GL_ATTRIBUTE_MAY_ALIAS __attribute__ ((__may_alias__)) +#else +# define _GL_ATTRIBUTE_MAY_ALIAS +#endif + +#if 201710L < __STDC_VERSION__ +# define _GL_ATTRIBUTE_MAYBE_UNUSED [[__maybe_unused__]] +#elif _GL_HAS_ATTRIBUTE (unused) +# define _GL_ATTRIBUTE_MAYBE_UNUSED __attribute__ ((__unused__)) +#else +# define _GL_ATTRIBUTE_MAYBE_UNUSED +#endif +/* Earlier spellings of this macro. */ +#define _GL_UNUSED _GL_ATTRIBUTE_MAYBE_UNUSED +#define _UNUSED_PARAMETER_ _GL_ATTRIBUTE_MAYBE_UNUSED + +#if _GL_HAS_ATTRIBUTE (malloc) +# define _GL_ATTRIBUTE_MALLOC __attribute__ ((__malloc__)) +#else +# define _GL_ATTRIBUTE_MALLOC +#endif + +#if 201710L < __STDC_VERSION__ +# define _GL_ATTRIBUTE_NODISCARD [[__nodiscard__]] +#elif _GL_HAS_ATTRIBUTE (warn_unused_result) +# define _GL_ATTRIBUTE_NODISCARD __attribute__ ((__warn_unused_result__)) +#else +# define _GL_ATTRIBUTE_NODISCARD +#endif + +#if _GL_HAS_ATTRIBUTE (noinline) +# define _GL_ATTRIBUTE_NOINLINE __attribute__ ((__noinline__)) +#else +# define _GL_ATTRIBUTE_NOINLINE +#endif + +#if _GL_HAS_ATTRIBUTE (nonnull) +# define _GL_ATTRIBUTE_NONNULL(args) __attribute__ ((__nonnull__ args)) +#else +# define _GL_ATTRIBUTE_NONNULL(args) +#endif + +#if _GL_HAS_ATTRIBUTE (nonstring) +# define _GL_ATTRIBUTE_NONSTRING __attribute__ ((__nonstring__)) +#else +# define _GL_ATTRIBUTE_NONSTRING +#endif + +/* There is no _GL_ATTRIBUTE_NORETURN; use _Noreturn instead. */ + +#if _GL_HAS_ATTRIBUTE (nothrow) && !defined __cplusplus +# define _GL_ATTRIBUTE_NOTHROW __attribute__ ((__nothrow__)) +#else +# define _GL_ATTRIBUTE_NOTHROW +#endif + +#if _GL_HAS_ATTRIBUTE (packed) +# define _GL_ATTRIBUTE_PACKED __attribute__ ((__packed__)) +#else +# define _GL_ATTRIBUTE_PACKED +#endif + +#if _GL_HAS_ATTRIBUTE (pure) +# define _GL_ATTRIBUTE_PURE __attribute__ ((__pure__)) +#else +# define _GL_ATTRIBUTE_PURE +#endif + +#if _GL_HAS_ATTRIBUTE (returns_nonnull) +# define _GL_ATTRIBUTE_RETURNS_NONNULL __attribute__ ((__returns_nonnull__)) +#else +# define _GL_ATTRIBUTE_RETURNS_NONNULL +#endif + +#if _GL_HAS_ATTRIBUTE (sentinel) +# define _GL_ATTRIBUTE_SENTINEL(pos) __attribute__ ((__sentinel__ pos)) +#else +# define _GL_ATTRIBUTE_SENTINEL(pos) +#endif + + +/* To support C++ as well as C, use _GL_UNUSED_LABEL with trailing ';'. */ +#if !defined __cplusplus || _GL_GNUC_PREREQ (4, 5) +# define _GL_UNUSED_LABEL _GL_ATTRIBUTE_MAYBE_UNUSED +#else +# define _GL_UNUSED_LABEL +#endif + + +/* Define to empty if `const' does not conform to ANSI C. */ +#undef const + +/* Define as 'access' if you don't have the eaccess() function. */ +#undef eaccess + +/* Please see the Gnulib manual for how to use these macros. + + Suppress extern inline with HP-UX cc, as it appears to be broken; see + <http://lists.gnu.org/archive/html/bug-texinfo/2013-02/msg00030.html>. + + Suppress extern inline with Sun C in standards-conformance mode, as it + mishandles inline functions that call each other. E.g., for 'inline void f + (void) { } inline void g (void) { f (); }', c99 incorrectly complains + 'reference to static identifier "f" in extern inline function'. + This bug was observed with Sun C 5.12 SunOS_i386 2011/11/16. + + Suppress extern inline (with or without __attribute__ ((__gnu_inline__))) + on configurations that mistakenly use 'static inline' to implement + functions or macros in standard C headers like <ctype.h>. For example, + if isdigit is mistakenly implemented via a static inline function, + a program containing an extern inline function that calls isdigit + may not work since the C standard prohibits extern inline functions + from calling static functions. This bug is known to occur on: + + OS X 10.8 and earlier; see: + http://lists.gnu.org/archive/html/bug-gnulib/2012-12/msg00023.html + + DragonFly; see + http://muscles.dragonflybsd.org/bulk/bleeding-edge-potential/latest-per-pkg/ah-tty-0.3.12.log + + FreeBSD; see: + http://lists.gnu.org/archive/html/bug-gnulib/2014-07/msg00104.html + + OS X 10.9 has a macro __header_inline indicating the bug is fixed for C and + for clang but remains for g++; see <http://trac.macports.org/ticket/41033>. + Assume DragonFly and FreeBSD will be similar. */ +#if (((defined __APPLE__ && defined __MACH__) \ + || defined __DragonFly__ || defined __FreeBSD__) \ + && (defined __header_inline \ + ? (defined __cplusplus && defined __GNUC_STDC_INLINE__ \ + && ! defined __clang__) \ + : ((! defined _DONT_USE_CTYPE_INLINE_ \ + && (defined __GNUC__ || defined __cplusplus)) \ + || (defined _FORTIFY_SOURCE && 0 < _FORTIFY_SOURCE \ + && defined __GNUC__ && ! defined __cplusplus)))) +# define _GL_EXTERN_INLINE_STDHEADER_BUG +#endif +#if ((__GNUC__ \ + ? defined __GNUC_STDC_INLINE__ && __GNUC_STDC_INLINE__ \ + : (199901L <= __STDC_VERSION__ \ + && !defined __HP_cc \ + && !(defined __SUNPRO_C && __STDC__))) \ + && !defined _GL_EXTERN_INLINE_STDHEADER_BUG) +# define _GL_INLINE inline +# define _GL_EXTERN_INLINE extern inline +# define _GL_EXTERN_INLINE_IN_USE +#elif (2 < __GNUC__ + (7 <= __GNUC_MINOR__) && !defined __STRICT_ANSI__ \ + && !defined _GL_EXTERN_INLINE_STDHEADER_BUG) +# if defined __GNUC_GNU_INLINE__ && __GNUC_GNU_INLINE__ + /* __gnu_inline__ suppresses a GCC 4.2 diagnostic. */ +# define _GL_INLINE extern inline __attribute__ ((__gnu_inline__)) +# else +# define _GL_INLINE extern inline +# endif +# define _GL_EXTERN_INLINE extern +# define _GL_EXTERN_INLINE_IN_USE +#else +# define _GL_INLINE static _GL_UNUSED +# define _GL_EXTERN_INLINE static _GL_UNUSED +#endif + +#if 4 < __GNUC__ + (6 <= __GNUC_MINOR__) +# if defined __GNUC_STDC_INLINE__ && __GNUC_STDC_INLINE__ +# define _GL_INLINE_HEADER_CONST_PRAGMA +# else +# define _GL_INLINE_HEADER_CONST_PRAGMA \ + _Pragma ("GCC diagnostic ignored \"-Wsuggest-attribute=const\"") +# endif + /* Suppress GCC's bogus "no previous prototype for 'FOO'" + and "no previous declaration for 'FOO'" diagnostics, + when FOO is an inline function in the header; see + <http://gcc.gnu.org/bugzilla/show_bug.cgi?id=54113>. */ +# define _GL_INLINE_HEADER_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wmissing-prototypes\"") \ + _Pragma ("GCC diagnostic ignored \"-Wmissing-declarations\"") \ + _GL_INLINE_HEADER_CONST_PRAGMA +# define _GL_INLINE_HEADER_END \ + _Pragma ("GCC diagnostic pop") +#else +# define _GL_INLINE_HEADER_BEGIN +# define _GL_INLINE_HEADER_END +#endif + +/* Define to `int' if <sys/types.h> doesn't define. */ +#undef gid_t + +/* Define to `__inline__' or `__inline' if that's what the C compiler + calls it, or to nothing if 'inline' is not supported under any name. */ +#ifndef __cplusplus +#undef inline +#endif + +/* Define to long or long long if <stdint.h> and <inttypes.h> don't define. */ +#undef intmax_t + +/* Work around a bug in Apple GCC 4.0.1 build 5465: In C99 mode, it supports + the ISO C 99 semantics of 'extern inline' (unlike the GNU C semantics of + earlier versions), but does not display it by setting __GNUC_STDC_INLINE__. + __APPLE__ && __MACH__ test for Mac OS X. + __APPLE_CC__ tests for the Apple compiler and its version. + __STDC_VERSION__ tests for the C99 mode. */ +#if defined __APPLE__ && defined __MACH__ && __APPLE_CC__ >= 5465 && !defined __cplusplus && __STDC_VERSION__ >= 199901L && !defined __GNUC_STDC_INLINE__ +# define __GNUC_STDC_INLINE__ 1 +#endif + +/* Define to a type if <wchar.h> does not define. */ +#undef mbstate_t + +/* _GL_CMP (n1, n2) performs a three-valued comparison on n1 vs. n2, where + n1 and n2 are expressions without side effects, that evaluate to real + numbers (excluding NaN). + It returns + 1 if n1 > n2 + 0 if n1 == n2 + -1 if n1 < n2 + The naïve code (n1 > n2 ? 1 : n1 < n2 ? -1 : 0) produces a conditional + jump with nearly all GCC versions up to GCC 10. + This variant (n1 < n2 ? -1 : n1 > n2) produces a conditional with many + GCC versions up to GCC 9. + The better code (n1 > n2) - (n1 < n2) from Hacker's Delight § 2-9 + avoids conditional jumps in all GCC versions >= 3.4. */ +#define _GL_CMP(n1, n2) (((n1) > (n2)) - ((n1) < (n2))) + + +/* Define to the real name of the mktime_internal function. */ +#undef mktime_internal + +/* Define to `int' if <sys/types.h> does not define. */ +#undef mode_t + +/* Define to the type of st_nlink in struct stat, or a supertype. */ +#undef nlink_t + +/* Define as a signed integer type capable of holding a process identifier. */ +#undef pid_t + +/* Define as the type of the result of subtracting two pointers, if the system + doesn't define it. */ +#undef ptrdiff_t + +/* Define to rpl_re_comp if the replacement should be used. */ +#undef re_comp + +/* Define to rpl_re_compile_fastmap if the replacement should be used. */ +#undef re_compile_fastmap + +/* Define to rpl_re_compile_pattern if the replacement should be used. */ +#undef re_compile_pattern + +/* Define to rpl_re_exec if the replacement should be used. */ +#undef re_exec + +/* Define to rpl_re_match if the replacement should be used. */ +#undef re_match + +/* Define to rpl_re_match_2 if the replacement should be used. */ +#undef re_match_2 + +/* Define to rpl_re_search if the replacement should be used. */ +#undef re_search + +/* Define to rpl_re_search_2 if the replacement should be used. */ +#undef re_search_2 + +/* Define to rpl_re_set_registers if the replacement should be used. */ +#undef re_set_registers + +/* Define to rpl_re_set_syntax if the replacement should be used. */ +#undef re_set_syntax + +/* Define to rpl_re_syntax_options if the replacement should be used. */ +#undef re_syntax_options + +/* Define to rpl_regcomp if the replacement should be used. */ +#undef regcomp + +/* Define to rpl_regerror if the replacement should be used. */ +#undef regerror + +/* Define to rpl_regexec if the replacement should be used. */ +#undef regexec + +/* Define to rpl_regfree if the replacement should be used. */ +#undef regfree + +/* Define to the equivalent of the C99 'restrict' keyword, or to + nothing if this is not supported. Do not define if restrict is + supported only directly. */ +#undef restrict +/* Work around a bug in older versions of Sun C++, which did not + #define __restrict__ or support _Restrict or __restrict__ + even though the corresponding Sun C compiler ended up with + "#define restrict _Restrict" or "#define restrict __restrict__" + in the previous line. This workaround can be removed once + we assume Oracle Developer Studio 12.5 (2016) or later. */ +#if defined __SUNPRO_CC && !defined __RESTRICT && !defined __restrict__ +# define _Restrict +# define __restrict__ +#endif + +/* Define as an integer type suitable for memory locations that can be + accessed atomically even in the presence of asynchronous signals. */ +#undef sig_atomic_t + +/* Define to `unsigned int' if <sys/types.h> does not define. */ +#undef size_t + +/* type to use in place of socklen_t if not defined */ +#undef socklen_t + +/* Define as a signed type of the same size as size_t. */ +#undef ssize_t + +/* Define to `int' if <sys/types.h> doesn't define. */ +#undef uid_t + +/* Define to empty if the keyword `volatile' does not work. Warning: valid + code using `volatile' can become incorrect without. Disable with care. */ +#undef volatile diff --git a/src/connect.c b/src/connect.c new file mode 100644 index 0000000..ad7ba0b --- /dev/null +++ b/src/connect.c @@ -0,0 +1,1084 @@ +/* Establishing and handling network connections. + Copyright (C) 1995-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include "exits.h" +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <assert.h> + +#include <sys/socket.h> +#include <sys/select.h> + +#ifndef WINDOWS +# ifdef __VMS +# include "vms_ip.h" +# else /* def __VMS */ +# include <netdb.h> +# endif /* def __VMS [else] */ +# include <netinet/in.h> +# ifndef __BEOS__ +# include <arpa/inet.h> +# endif +#endif /* not WINDOWS */ + +#include <errno.h> +#include <string.h> +#include <sys/time.h> + +#include "utils.h" +#include "host.h" +#include "connect.h" +#include "hash.h" + +#include <stdint.h> + +/* Define sockaddr_storage where unavailable (presumably on IPv4-only + hosts). */ + +#ifndef ENABLE_IPV6 +# ifndef HAVE_STRUCT_SOCKADDR_STORAGE +# define sockaddr_storage sockaddr_in +# endif +#endif /* ENABLE_IPV6 */ + +/* Fill SA as per the data in IP and PORT. SA should point to struct + sockaddr_storage if ENABLE_IPV6 is defined, to struct sockaddr_in + otherwise. */ + +static void +sockaddr_set_data (struct sockaddr *sa, const ip_address *ip, int port) +{ + switch (ip->family) + { + case AF_INET: + { + struct sockaddr_in *sin = (struct sockaddr_in *)sa; + xzero (*sin); + sin->sin_family = AF_INET; + sin->sin_port = htons (port); + sin->sin_addr = ip->data.d4; + break; + } +#ifdef ENABLE_IPV6 + case AF_INET6: + { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sa; + xzero (*sin6); + sin6->sin6_family = AF_INET6; + sin6->sin6_port = htons (port); + sin6->sin6_addr = ip->data.d6; +#ifdef HAVE_SOCKADDR_IN6_SCOPE_ID + sin6->sin6_scope_id = ip->ipv6_scope; +#endif + break; + } +#endif /* ENABLE_IPV6 */ + default: + abort (); + } +} + +/* Get the data of SA, specifically the IP address and the port. If + you're not interested in one or the other information, pass NULL as + the pointer. */ + +static void +sockaddr_get_data (const struct sockaddr *sa, ip_address *ip, int *port) +{ + switch (sa->sa_family) + { + case AF_INET: + { + struct sockaddr_in *sin = (struct sockaddr_in *)sa; + if (ip) + { + ip->family = AF_INET; + ip->data.d4 = sin->sin_addr; + } + if (port) + *port = ntohs (sin->sin_port); + break; + } +#ifdef ENABLE_IPV6 + case AF_INET6: + { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sa; + if (ip) + { + ip->family = AF_INET6; + ip->data.d6 = sin6->sin6_addr; +#ifdef HAVE_SOCKADDR_IN6_SCOPE_ID + ip->ipv6_scope = sin6->sin6_scope_id; +#endif + } + if (port) + *port = ntohs (sin6->sin6_port); + break; + } +#endif + default: + abort (); + } +} + +/* Return the size of the sockaddr structure depending on its + family. */ + +static socklen_t +sockaddr_size (const struct sockaddr *sa) +{ + switch (sa->sa_family) + { + case AF_INET: + return sizeof (struct sockaddr_in); +#ifdef ENABLE_IPV6 + case AF_INET6: + return sizeof (struct sockaddr_in6); +#endif + default: + abort (); + } +} + +/* Resolve the bind address specified via --bind-address and store it + to SA. The resolved value is stored in a static variable and + reused after the first invocation of this function. + + Returns true on success, false on failure. */ + +static bool +resolve_bind_address (struct sockaddr *sa) +{ + struct address_list *al; + + /* Make sure this is called only once. opt.bind_address doesn't + change during a Wget run. */ + static bool called, should_bind; + static ip_address ip; + if (called) + { + if (should_bind) + sockaddr_set_data (sa, &ip, 0); + return should_bind; + } + called = true; + + al = lookup_host (opt.bind_address, LH_BIND | LH_SILENT); + if (!al) + { + /* #### We should be able to print the error message here. */ + logprintf (LOG_NOTQUIET, + _("%s: unable to resolve bind address %s; disabling bind.\n"), + exec_name, quote (opt.bind_address)); + should_bind = false; + return false; + } + + /* Pick the first address in the list and use it as bind address. + Perhaps we should try multiple addresses in succession, but I + don't think that's necessary in practice. */ + ip = *address_list_address_at (al, 0); + address_list_release (al); + + sockaddr_set_data (sa, &ip, 0); + should_bind = true; + return true; +} + +struct cwt_context { + int fd; + const struct sockaddr *addr; + socklen_t addrlen; + int result; +}; + +static void +connect_with_timeout_callback (void *arg) +{ + struct cwt_context *ctx = (struct cwt_context *)arg; + ctx->result = connect (ctx->fd, ctx->addr, ctx->addrlen); +} + +/* Like connect, but specifies a timeout. If connecting takes longer + than TIMEOUT seconds, -1 is returned and errno is set to + ETIMEDOUT. */ + +static int +connect_with_timeout (int fd, const struct sockaddr *addr, socklen_t addrlen, + double timeout) +{ + struct cwt_context ctx; + ctx.fd = fd; + ctx.addr = addr; + ctx.addrlen = addrlen; + + if (run_with_timeout (timeout, connect_with_timeout_callback, &ctx)) + { + errno = ETIMEDOUT; + return -1; + } + if (ctx.result == -1 && errno == EINTR) + errno = ETIMEDOUT; + return ctx.result; +} + +/* Connect via TCP to the specified address and port. + + If PRINT is non-NULL, it is the host name to print that we're + connecting to. */ + +int +connect_to_ip (const ip_address *ip, int port, const char *print) +{ + struct sockaddr_storage ss; + struct sockaddr *sa = (struct sockaddr *)&ss; + int sock; + + /* If PRINT is non-NULL, print the "Connecting to..." line, with + PRINT being the host name we're connecting to. */ + if (print) + { + const char *txt_addr = print_address (ip); + if (0 != strcmp (print, txt_addr)) + { + char *str = NULL, *name; + + if (opt.enable_iri && (name = idn_decode ((char *) print)) != NULL) + { + str = aprintf ("%s (%s)", name, print); + xfree (name); + } + + logprintf (LOG_VERBOSE, _("Connecting to %s|%s|:%d... "), + str ? str : escnonprint_uri (print), txt_addr, port); + + xfree (str); + } + else + { + if (ip->family == AF_INET) + logprintf (LOG_VERBOSE, _("Connecting to %s:%d... "), txt_addr, port); +#ifdef ENABLE_IPV6 + else if (ip->family == AF_INET6) + logprintf (LOG_VERBOSE, _("Connecting to [%s]:%d... "), txt_addr, port); +#endif + } + } + + /* Store the sockaddr info to SA. */ + sockaddr_set_data (sa, ip, port); + + /* Create the socket of the family appropriate for the address. */ + sock = socket (sa->sa_family, SOCK_STREAM, 0); + if (sock < 0) + goto err; + +#if defined(ENABLE_IPV6) && defined(IPV6_V6ONLY) + if (opt.ipv6_only) { + int on = 1; + /* In case of error, we will go on anyway... */ + int err = setsockopt (sock, IPPROTO_IPV6, IPV6_V6ONLY, &on, sizeof (on)); + IF_DEBUG + if (err < 0) + DEBUGP (("Failed setting IPV6_V6ONLY: %s", strerror (errno))); + } +#endif + + /* For very small rate limits, set the buffer size (and hence, + hopefully, the kernel's TCP window size) to the per-second limit. + That way we should never have to sleep for more than 1s between + network reads. */ + if (opt.limit_rate && opt.limit_rate < 8192) + { + int bufsize = opt.limit_rate; + if (bufsize < 512) + bufsize = 512; /* avoid pathologically small values */ +#ifdef SO_RCVBUF + if (setsockopt (sock, SOL_SOCKET, SO_RCVBUF, + (void *) &bufsize, (socklen_t) sizeof (bufsize))) + logprintf (LOG_NOTQUIET, _("setsockopt SO_RCVBUF failed: %s\n"), + strerror (errno)); +#endif + /* When we add limit_rate support for writing, which is useful + for POST, we should also set SO_SNDBUF here. */ + } + + if (opt.bind_address) + { + /* Bind the client side of the socket to the requested + address. */ + struct sockaddr_storage bind_ss; + struct sockaddr *bind_sa = (struct sockaddr *)&bind_ss; + if (resolve_bind_address (bind_sa)) + { + if (bind (sock, bind_sa, sockaddr_size (bind_sa)) < 0) + goto err; + } + } + + /* Connect the socket to the remote endpoint. */ + if (connect_with_timeout (sock, sa, sockaddr_size (sa), + opt.connect_timeout) < 0) + goto err; + + /* Success. */ + assert (sock >= 0); + if (print) + logprintf (LOG_VERBOSE, _("connected.\n")); + DEBUGP (("Created socket %d.\n", sock)); + return sock; + + err: + { + /* Protect errno from possible modifications by close and + logprintf. */ + int save_errno = errno; + if (sock >= 0) + { +#ifdef WIN32 + /* If the connection timed out, fd_close will hang in Gnulib's + close_fd_maybe_socket, inside the call to WSAEnumNetworkEvents. */ + if (errno != ETIMEDOUT) +#endif + fd_close (sock); + } + if (print) + logprintf (LOG_NOTQUIET, _("failed: %s.\n"), strerror (errno)); + errno = save_errno; + return -1; + } +} + +/* Connect via TCP to a remote host on the specified port. + + HOST is resolved as an Internet host name. If HOST resolves to + more than one IP address, they are tried in the order returned by + DNS until connecting to one of them succeeds. */ + +int +connect_to_host (const char *host, int port) +{ + int i, start, end; + int sock; + + struct address_list *al = lookup_host (host, 0); + + retry: + if (!al) + { + logprintf (LOG_NOTQUIET, + _("%s: unable to resolve host address %s\n"), + exec_name, quote (host)); + return E_HOST; + } + + address_list_get_bounds (al, &start, &end); + for (i = start; i < end; i++) + { + const ip_address *ip = address_list_address_at (al, i); + sock = connect_to_ip (ip, port, host); + if (sock >= 0) + { + /* Success. */ + address_list_set_connected (al); + address_list_release (al); + return sock; + } + + /* The attempt to connect has failed. Continue with the loop + and try next address. */ + + address_list_set_faulty (al, i); + } + + /* Failed to connect to any of the addresses in AL. */ + + if (address_list_connected_p (al)) + { + /* We connected to AL before, but cannot do so now. That might + indicate that our DNS cache entry for HOST has expired. */ + address_list_release (al); + al = lookup_host (host, LH_REFRESH); + goto retry; + } + address_list_release (al); + + return -1; +} + +/* Create a socket, bind it to local interface BIND_ADDRESS on port + *PORT, set up a listen backlog, and return the resulting socket, or + -1 in case of error. + + BIND_ADDRESS is the address of the interface to bind to. If it is + NULL, the socket is bound to the default address. PORT should + point to the port number that will be used for the binding. If + that number is 0, the system will choose a suitable port, and the + chosen value will be written to *PORT. + + Calling accept() on such a socket waits for and accepts incoming + TCP connections. */ + +int +bind_local (const ip_address *bind_address, int *port) +{ + int sock; + struct sockaddr_storage ss; + struct sockaddr *sa = (struct sockaddr *)&ss; + + /* For setting options with setsockopt. */ + int setopt_val = 1; + void *setopt_ptr = (void *)&setopt_val; + socklen_t setopt_size = sizeof (setopt_val); + + sock = socket (bind_address->family, SOCK_STREAM, 0); + if (sock < 0) + return -1; + +#ifdef SO_REUSEADDR + if (setsockopt (sock, SOL_SOCKET, SO_REUSEADDR, setopt_ptr, setopt_size)) + logprintf (LOG_NOTQUIET, _("setsockopt SO_REUSEADDR failed: %s\n"), + strerror (errno)); +#endif + + xzero (ss); + sockaddr_set_data (sa, bind_address, *port); + if (bind (sock, sa, sockaddr_size (sa)) < 0) + { + fd_close (sock); + return -1; + } + DEBUGP (("Local socket fd %d bound.\n", sock)); + + /* If *PORT is 0, find out which port we've bound to. */ + if (*port == 0) + { + socklen_t addrlen = sockaddr_size (sa); + if (getsockname (sock, sa, &addrlen) < 0) + { + /* If we can't find out the socket's local address ("name"), + something is seriously wrong with the socket, and it's + unusable for us anyway because we must know the chosen + port. */ + fd_close (sock); + return -1; + } + sockaddr_get_data (sa, NULL, port); + DEBUGP (("binding to address %s using port %i.\n", + print_address (bind_address), *port)); + } + if (listen (sock, 1) < 0) + { + fd_close (sock); + return -1; + } + return sock; +} + +/* Like a call to accept(), but with the added check for timeout. + + In other words, accept a client connection on LOCAL_SOCK, and + return the new socket used for communication with the client. + LOCAL_SOCK should have been bound, e.g. using bind_local(). + + The caller is blocked until a connection is established. If no + connection is established for opt.connect_timeout seconds, the + function exits with an error status. */ + +int +accept_connection (int local_sock) +{ + int sock; + + /* We don't need the values provided by accept, but accept + apparently requires them to be present. */ + struct sockaddr_storage ss; + struct sockaddr *sa = (struct sockaddr *)&ss; + socklen_t addrlen = sizeof (ss); + + if (opt.connect_timeout) + { + int test = select_fd (local_sock, opt.connect_timeout, WAIT_FOR_READ); + if (test == 0) + errno = ETIMEDOUT; + if (test <= 0) + return -1; + } + sock = accept (local_sock, sa, &addrlen); + DEBUGP (("Accepted client at socket %d.\n", sock)); + return sock; +} + +/* Get the IP address associated with the connection on FD and store + it to IP. Return true on success, false otherwise. + + If ENDPOINT is ENDPOINT_LOCAL, it returns the address of the local + (client) side of the socket. Else if ENDPOINT is ENDPOINT_PEER, it + returns the address of the remote (peer's) side of the socket. */ + +bool +socket_ip_address (int sock, ip_address *ip, int endpoint) +{ + struct sockaddr_storage storage; + struct sockaddr *sockaddr = (struct sockaddr *) &storage; + socklen_t addrlen = sizeof (storage); + int ret; + + memset (sockaddr, 0, addrlen); + if (endpoint == ENDPOINT_LOCAL) + ret = getsockname (sock, sockaddr, &addrlen); + else if (endpoint == ENDPOINT_PEER) + ret = getpeername (sock, sockaddr, &addrlen); + else + abort (); + if (ret < 0) + return false; + + memset(ip, 0, sizeof(ip_address)); + ip->family = sockaddr->sa_family; + switch (sockaddr->sa_family) + { +#ifdef ENABLE_IPV6 + case AF_INET6: + { + struct sockaddr_in6 *sa6 = (struct sockaddr_in6 *)&storage; + ip->data.d6 = sa6->sin6_addr; +#ifdef HAVE_SOCKADDR_IN6_SCOPE_ID + ip->ipv6_scope = sa6->sin6_scope_id; +#endif + DEBUGP (("conaddr is: %s\n", print_address (ip))); + return true; + } +#endif + case AF_INET: + { + struct sockaddr_in *sa = (struct sockaddr_in *)&storage; + ip->data.d4 = sa->sin_addr; + DEBUGP (("conaddr is: %s\n", print_address (ip))); + return true; + } + default: + abort (); + } +} + +/* Get the socket family of connection on FD and store + Return family type on success, -1 otherwise. + + If ENDPOINT is ENDPOINT_LOCAL, it returns the sock family of the local + (client) side of the socket. Else if ENDPOINT is ENDPOINT_PEER, it + returns the sock family of the remote (peer's) side of the socket. */ + +int +socket_family (int sock, int endpoint) +{ + struct sockaddr_storage storage; + struct sockaddr *sockaddr = (struct sockaddr *) &storage; + socklen_t addrlen = sizeof (storage); + int ret; + + memset (sockaddr, 0, addrlen); + + if (endpoint == ENDPOINT_LOCAL) + ret = getsockname (sock, sockaddr, &addrlen); + else if (endpoint == ENDPOINT_PEER) + ret = getpeername (sock, sockaddr, &addrlen); + else + abort (); + + if (ret < 0) + return -1; + + return sockaddr->sa_family; +} + +/* Return true if the error from the connect code can be considered + retryable. Wget normally retries after errors, but the exception + are the "unsupported protocol" type errors (possible on IPv4/IPv6 + dual family systems) and "connection refused". */ + +bool +retryable_socket_connect_error (int err) +{ + /* Have to guard against some of these values not being defined. + Cannot use a switch statement because some of the values might be + equal. */ + if (false +#ifdef EAFNOSUPPORT + || err == EAFNOSUPPORT +#endif +#ifdef EPFNOSUPPORT + || err == EPFNOSUPPORT +#endif +#ifdef ESOCKTNOSUPPORT /* no, "sockt" is not a typo! */ + || err == ESOCKTNOSUPPORT +#endif +#ifdef EPROTONOSUPPORT + || err == EPROTONOSUPPORT +#endif +#ifdef ENOPROTOOPT + || err == ENOPROTOOPT +#endif + /* Apparently, older versions of Linux and BSD used EINVAL + instead of EAFNOSUPPORT and such. */ + || err == EINVAL + ) + return false; + + if (!opt.retry_connrefused) + if (err == ECONNREFUSED +#ifdef ENETUNREACH + || err == ENETUNREACH /* network is unreachable */ +#endif +#ifdef EHOSTUNREACH + || err == EHOSTUNREACH /* host is unreachable */ +#endif + ) + return false; + + return true; +} + +/* Wait for a single descriptor to become available, timing out after + MAXTIME seconds. Returns 1 if FD is available, 0 for timeout and + -1 for error. The argument WAIT_FOR can be a combination of + WAIT_FOR_READ and WAIT_FOR_WRITE. + + This is a mere convenience wrapper around the select call, and + should be taken as such (for example, it doesn't implement Wget's + 0-timeout-means-no-timeout semantics.) */ + +static int +select_fd_internal (int fd, double maxtime, int wait_for, bool convert_back _GL_UNUSED) +{ + fd_set fdset; + fd_set *rd = NULL, *wr = NULL; + struct timeval tmout; + int result; + + if (fd < 0) + return -1; + + if (fd >= FD_SETSIZE) + { + logprintf (LOG_NOTQUIET, _("Too many fds open. Cannot use select on a fd >= %d\n"), FD_SETSIZE); + exit (WGET_EXIT_GENERIC_ERROR); + } + FD_ZERO (&fdset); + FD_SET (fd, &fdset); + if (wait_for & WAIT_FOR_READ) + rd = &fdset; + if (wait_for & WAIT_FOR_WRITE) + wr = &fdset; + + tmout.tv_sec = (long) maxtime; + tmout.tv_usec = 1000000 * (maxtime - (long) maxtime); + + do + { + result = select (fd + 1, rd, wr, NULL, &tmout); +#ifdef WINDOWS + /* gnulib select() converts blocking sockets to nonblocking in windows. + wget uses blocking sockets so we must convert them back to blocking. */ + if (convert_back) + set_windows_fd_as_blocking_socket (fd); +#endif + } + while (result < 0 && errno == EINTR); + + return result; +} + +int +select_fd (int fd, double maxtime, int wait_for) +{ + return select_fd_internal (fd, maxtime, wait_for, true); +} + +#ifdef WINDOWS +int +select_fd_nb (int fd, double maxtime, int wait_for) +{ + return select_fd_internal (fd, maxtime, wait_for, false); +} +#endif + +/* Return true if the connection to the remote site established + through SOCK is still open. + + Specifically, this function returns true if SOCK is not ready for + reading. This is because, when the connection closes, the socket + is ready for reading because EOF is about to be delivered. A side + effect of this method is that sockets that have pending data are + considered non-open. This is actually a good thing for callers of + this function, where such pending data can only be unwanted + leftover from a previous request. */ + +bool +test_socket_open (int sock) +{ + fd_set check_set; + struct timeval to; + int ret = 0; + + if (sock >= FD_SETSIZE) + { + logprintf (LOG_NOTQUIET, _("Too many fds open. Cannot use select on a fd >= %d\n"), FD_SETSIZE); + exit (WGET_EXIT_GENERIC_ERROR); + } + /* Check if we still have a valid (non-EOF) connection. From Andrew + * Maholski's code in the Unix Socket FAQ. */ + + FD_ZERO (&check_set); + FD_SET (sock, &check_set); + + /* Wait one microsecond */ + to.tv_sec = 0; + to.tv_usec = 1; + + ret = select (sock + 1, &check_set, NULL, NULL, &to); +#ifdef WINDOWS +/* gnulib select() converts blocking sockets to nonblocking in windows. +wget uses blocking sockets so we must convert them back to blocking +*/ + set_windows_fd_as_blocking_socket ( sock ); +#endif + + if ( !ret ) + /* We got a timeout, it means we're still connected. */ + return true; + else + /* Read now would not wait, it means we have either pending data + or EOF/error. */ + return false; +} + +/* Basic socket operations, mostly EINTR wrappers. */ + +static int +sock_read (int fd, char *buf, int bufsize) +{ + int res; + do + res = read (fd, buf, bufsize); + while (res == -1 && errno == EINTR); + return res; +} + +static int +sock_write (int fd, char *buf, int bufsize) +{ + int res; + do + res = write (fd, buf, bufsize); + while (res == -1 && errno == EINTR); + return res; +} + +static int +sock_poll (int fd, double timeout, int wait_for) +{ + return select_fd (fd, timeout, wait_for); +} + +static int +sock_peek (int fd, char *buf, int bufsize) +{ + int res; + do + res = recv (fd, buf, bufsize, MSG_PEEK); + while (res == -1 && errno == EINTR); + return res; +} + +static void +sock_close (int fd) +{ + close (fd); + DEBUGP (("Closed fd %d\n", fd)); +} +#undef read +#undef write +#undef close + +/* Reading and writing from the network. We build around the socket + (file descriptor) API, but support "extended" operations for things + that are not mere file descriptors under the hood, such as SSL + sockets. + + That way the user code can call fd_read(fd, ...) and we'll run read + or SSL_read or whatever is necessary. */ + +static struct hash_table *transport_map; +static unsigned int transport_map_modified_tick; + +struct transport_info { + struct transport_implementation *imp; + void *ctx; +}; + +/* Register the transport layer operations that will be used when + reading, writing, and polling FD. + + This should be used for transport layers like SSL that piggyback on + sockets. FD should otherwise be a real socket, on which you can + call getpeername, etc. */ + +void +fd_register_transport (int fd, struct transport_implementation *imp, void *ctx) +{ + struct transport_info *info; + + /* The file descriptor must be non-negative to be registered. + Negative values are ignored by fd_close(), and -1 cannot be used as + hash key. */ + assert (fd >= 0); + + info = xnew (struct transport_info); + info->imp = imp; + info->ctx = ctx; + if (!transport_map) + transport_map = hash_table_new (0, NULL, NULL); + hash_table_put (transport_map, (void *)(intptr_t) fd, info); + ++transport_map_modified_tick; +} + +/* Return context of the transport registered with + fd_register_transport. This assumes fd_register_transport was + previously called on FD. */ + +void * +fd_transport_context (int fd) +{ + struct transport_info *info = hash_table_get (transport_map, (void *)(intptr_t) fd); + return info ? info->ctx : NULL; +} + +/* When fd_read/fd_write are called multiple times in a loop, they should + remember the INFO pointer instead of fetching it every time. It is + not enough to compare FD to LAST_FD because FD might have been + closed and reopened. modified_tick ensures that changes to + transport_map will not be unnoticed. + + This is a macro because we want the static storage variables to be + per-function. */ + +#define LAZY_RETRIEVE_INFO(info) do { \ + static struct transport_info *last_info; \ + static int last_fd = -1; \ + static unsigned int last_tick; \ + if (!transport_map) \ + info = NULL; \ + else if (last_fd == fd && last_tick == transport_map_modified_tick) \ + info = last_info; \ + else \ + { \ + info = hash_table_get (transport_map, (void *)(intptr_t) fd); \ + last_fd = fd; \ + last_info = info; \ + last_tick = transport_map_modified_tick; \ + } \ +} while (0) + +static bool +poll_internal (int fd, struct transport_info *info, int wf, double timeout) +{ + if (timeout == -1) + timeout = opt.read_timeout; + if (timeout) + { + int test; + if (info && info->imp->poller) + test = info->imp->poller (fd, timeout, wf, info->ctx); + else + test = sock_poll (fd, timeout, wf); + if (test == 0) + errno = ETIMEDOUT; + if (test <= 0) + return false; + } + return true; +} + +/* Read no more than BUFSIZE bytes of data from FD, storing them to + BUF. If TIMEOUT is non-zero, the operation aborts if no data is + received after that many seconds. If TIMEOUT is -1, the value of + opt.timeout is used for TIMEOUT. */ + +int +fd_read (int fd, char *buf, int bufsize, double timeout) +{ + struct transport_info *info; + LAZY_RETRIEVE_INFO (info); + + /* let imp->reader take care about timeout. + (or in worst case timeout can be 2*timeout) */ + if (info && info->imp->reader) + return info->imp->reader (fd, buf, bufsize, info->ctx, timeout); + + if (!poll_internal (fd, info, WAIT_FOR_READ, timeout)) + return -1; + return sock_read (fd, buf, bufsize); +} + +/* Like fd_read, except it provides a "preview" of the data that will + be read by subsequent calls to fd_read. Specifically, it copies no + more than BUFSIZE bytes of the currently available data to BUF and + returns the number of bytes copied. Return values and timeout + semantics are the same as those of fd_read. + + CAVEAT: Do not assume that the first subsequent call to fd_read + will retrieve the same amount of data. Reading can return more or + less data, depending on the TCP implementation and other + circumstances. However, barring an error, it can be expected that + all the peeked data will eventually be read by fd_read. */ + +int +fd_peek (int fd, char *buf, int bufsize, double timeout) +{ + struct transport_info *info; + LAZY_RETRIEVE_INFO (info); + + if (info && info->imp->peeker) + return info->imp->peeker (fd, buf, bufsize, info->ctx, timeout); + + if (!poll_internal (fd, info, WAIT_FOR_READ, timeout)) + return -1; + return sock_peek (fd, buf, bufsize); +} + +/* Write the entire contents of BUF to FD. If TIMEOUT is non-zero, + the operation aborts if no data is received after that many + seconds. If TIMEOUT is -1, the value of opt.timeout is used for + TIMEOUT. */ + +int +fd_write (int fd, char *buf, int bufsize, double timeout) +{ + int res; + struct transport_info *info; + LAZY_RETRIEVE_INFO (info); + + /* `write' may write less than LEN bytes, thus the loop keeps trying + it until all was written, or an error occurred. */ + res = 0; + while (bufsize > 0) + { + if (!poll_internal (fd, info, WAIT_FOR_WRITE, timeout)) + return -1; + if (info && info->imp->writer) + res = info->imp->writer (fd, buf, bufsize, info->ctx); + else + res = sock_write (fd, buf, bufsize); + if (res <= 0) + break; + buf += res; + bufsize -= res; + } + return res; +} + +/* Report the most recent error(s) on FD. This should only be called + after fd_* functions, such as fd_read and fd_write, and only if + they return a negative result. For errors coming from other calls + such as setsockopt or fopen, strerror should continue to be + used. + + If the transport doesn't support error messages or doesn't supply + one, strerror(errno) is returned. The returned error message + should not be used after fd_close has been called. */ + +const char * +fd_errstr (int fd) +{ + /* Don't bother with LAZY_RETRIEVE_INFO, as this will only be called + in case of error, never in a tight loop. */ + struct transport_info *info = NULL; + + if (transport_map) + info = hash_table_get (transport_map, (void *)(intptr_t) fd); + + if (info && info->imp->errstr) + { + const char *err = info->imp->errstr (fd, info->ctx); + if (err) + return err; + /* else, fall through and print the system error. */ + } + return strerror (errno); +} + +/* Close the file descriptor FD. */ + +void +fd_close (int fd) +{ + struct transport_info *info; + if (fd < 0) + return; + + /* Don't use LAZY_RETRIEVE_INFO because fd_close() is only called once + per socket, so that particular optimization wouldn't work. */ + info = NULL; + if (transport_map) + info = hash_table_get (transport_map, (void *)(intptr_t) fd); + + if (info && info->imp->closer) + info->imp->closer (fd, info->ctx); + else + sock_close (fd); + + if (info) + { + hash_table_remove (transport_map, (void *)(intptr_t) fd); + xfree (info); + ++transport_map_modified_tick; + } +} + +#if defined DEBUG_MALLOC || defined TESTING +void +connect_cleanup(void) +{ + if (transport_map) + { + hash_table_iterator iter; + for (hash_table_iterate (transport_map, &iter); hash_table_iter_next (&iter); ) + { + xfree (iter.value); + } + hash_table_destroy (transport_map); + transport_map = NULL; + } +} +#endif diff --git a/src/connect.h b/src/connect.h new file mode 100644 index 0000000..8453559 --- /dev/null +++ b/src/connect.h @@ -0,0 +1,89 @@ +/* Declarations for connect. + Copyright (C) 1996-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef CONNECT_H +#define CONNECT_H + +#include "host.h" /* for definition of ip_address */ + +/* Function declarations */ + +/* Returned by connect_to_host when host name cannot be resolved. */ +enum { + E_HOST = -100 +}; +int connect_to_host (const char *, int); +int connect_to_ip (const ip_address *, int, const char *); + +int bind_local (const ip_address *, int *); +int accept_connection (int); + +enum { + ENDPOINT_LOCAL, + ENDPOINT_PEER +}; +bool socket_ip_address (int, ip_address *, int); +int socket_family (int sock, int endpoint); + +bool retryable_socket_connect_error (int); + +/* Flags for select_fd's WAIT_FOR argument. */ +enum { + WAIT_FOR_READ = 1, + WAIT_FOR_WRITE = 2 +}; +int select_fd (int, double, int); +bool test_socket_open (int); + +struct transport_implementation { + int (*reader) (int, char *, int, void *, double); + int (*writer) (int, char *, int, void *); + int (*poller) (int, double, int, void *); + int (*peeker) (int, char *, int, void *, double); + const char *(*errstr) (int, void *); + void (*closer) (int, void *); +}; + +void fd_register_transport (int, struct transport_implementation *, void *); +void *fd_transport_context (int); +int fd_read (int, char *, int, double); +int fd_write (int, char *, int, double); +int fd_peek (int, char *, int, double); +const char *fd_errstr (int); +void fd_close (int); +void connect_cleanup (void); + +#ifdef WINDOWS +int select_fd_nb (int, double, int); +#else +#define select_fd_nb select_fd +#endif + +#endif /* CONNECT_H */ diff --git a/src/convert.c b/src/convert.c new file mode 100644 index 0000000..f422178 --- /dev/null +++ b/src/convert.c @@ -0,0 +1,1230 @@ +/* Conversion of links to local files. + Copyright (C) 2003-2011, 2014-2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> +#include <assert.h> +#include "convert.h" +#include "url.h" +#include "recur.h" +#include "utils.h" +#include "hash.h" +#include "ptimer.h" +#include "res.h" +#include "html-url.h" +#include "css-url.h" +#include "iri.h" +#include "xstrndup.h" + +static struct hash_table *dl_file_url_map; +struct hash_table *dl_url_file_map; + +/* Set of HTML/CSS files downloaded in this Wget run, used for link + conversion after Wget is done. */ +struct hash_table *downloaded_html_set; +struct hash_table *downloaded_css_set; + +static void convert_links (const char *, struct urlpos *); + + +static void +convert_links_in_hashtable (struct hash_table *downloaded_set, + int is_css, + int *file_count) +{ + int i, cnt = 0; + char *arr[1024], **file_array; + + if (!downloaded_set || (cnt = hash_table_count (downloaded_set)) == 0) + return; + + if (cnt <= (int) countof (arr)) + file_array = arr; + else + file_array = xmalloc (cnt * sizeof (arr[0])); + + string_set_to_array (downloaded_set, file_array); + + for (i = 0; i < cnt; i++) + { + struct urlpos *urls, *cur_url; + char *url; + char *file = file_array[i]; + + /* Determine the URL of the file. get_urls_{html,css} will need + it. */ + url = hash_table_get (dl_file_url_map, file); + if (!url) + { + DEBUGP (("Apparently %s has been removed.\n", file)); + continue; + } + + DEBUGP (("Scanning %s (from %s)\n", file, url)); + + /* Parse the file... */ + urls = is_css ? get_urls_css_file (file, url) : + get_urls_html (file, url, NULL, NULL); + + /* We don't respect meta_disallow_follow here because, even if + the file is not followed, we might still want to convert the + links that have been followed from other files. */ + + for (cur_url = urls; cur_url; cur_url = cur_url->next) + { + char *local_name; + struct url *u; + struct iri *pi; + + if (cur_url->link_base_p) + { + /* Base references have been resolved by our parser, so + we turn the base URL into an empty string. (Perhaps + we should remove the tag entirely?) */ + cur_url->convert = CO_NULLIFY_BASE; + continue; + } + + /* We decide the direction of conversion according to whether + a URL was downloaded. Downloaded URLs will be converted + ABS2REL, whereas non-downloaded will be converted REL2ABS. */ + + pi = iri_new (); + set_uri_encoding (pi, opt.locale, true); + + u = url_parse (cur_url->url->url, NULL, pi, true); + if (!u) + continue; + + local_name = hash_table_get (dl_url_file_map, u->url); + + /* Decide on the conversion type. */ + if (local_name) + { + /* We've downloaded this URL. Convert it to relative + form. We do this even if the URL already is in + relative form, because our directory structure may + not be identical to that on the server (think `-nd', + `--cut-dirs', etc.). If --convert-file-only was passed, + we only convert the basename portion of the URL. */ + cur_url->convert = (opt.convert_file_only ? CO_CONVERT_BASENAME_ONLY : CO_CONVERT_TO_RELATIVE); + cur_url->local_name = xstrdup (local_name); + DEBUGP (("will convert url %s to local %s\n", u->url, local_name)); + } + else + { + /* We haven't downloaded this URL. If it's not already + complete (including a full host name), convert it to + that form, so it can be reached while browsing this + HTML locally. */ + if (!cur_url->link_complete_p) + cur_url->convert = CO_CONVERT_TO_COMPLETE; + cur_url->local_name = NULL; + DEBUGP (("will convert url %s to complete\n", u->url)); + } + + url_free (u); + iri_free (pi); + } + + /* Convert the links in the file. */ + convert_links (file, urls); + ++*file_count; + + /* Free the data. */ + free_urlpos (urls); + } + + if (file_array != arr) + xfree (file_array); +} + +/* This function is called when the retrieval is done to convert the + links that have been downloaded. It has to be called at the end of + the retrieval, because only then does Wget know conclusively which + URLs have been downloaded, and which not, so it can tell which + direction to convert to. + + The "direction" means that the URLs to the files that have been + downloaded get converted to the relative URL which will point to + that file. And the other URLs get converted to the remote URL on + the server. + + All the downloaded HTMLs are kept in downloaded_html_files, and + downloaded URLs in urls_downloaded. All the information is + extracted from these two lists. */ + +void +convert_all_links (void) +{ + double secs; + int file_count = 0; + + struct ptimer *timer = ptimer_new (); + + convert_links_in_hashtable (downloaded_html_set, 0, &file_count); + convert_links_in_hashtable (downloaded_css_set, 1, &file_count); + + secs = ptimer_measure (timer); + logprintf (LOG_VERBOSE, _("Converted links in %d files in %s seconds.\n"), + file_count, print_decimal (secs)); + + ptimer_destroy (timer); +} + +static void write_backup_file (const char *, downloaded_file_t); +static const char *replace_plain (const char*, int, FILE*, const char *); +static const char *replace_attr (const char *, int, FILE *, const char *); +static const char *replace_attr_refresh_hack (const char *, int, FILE *, + const char *, int); +static char *local_quote_string (const char *, bool); +static char *construct_relative (const char *, const char *); +static char *convert_basename (const char *, const struct urlpos *); + +/* Change the links in one file. LINKS is a list of links in the + document, along with their positions and the desired direction of + the conversion. */ +static void +convert_links (const char *file, struct urlpos *links) +{ + struct file_memory *fm; + FILE *fp; + const char *p; + downloaded_file_t downloaded_file_return; + + struct urlpos *link; + int to_url_count = 0, to_file_count = 0; + + logprintf (LOG_VERBOSE, _("Converting links in %s... "), file); + + { + /* First we do a "dry run": go through the list L and see whether + any URL needs to be converted in the first place. If not, just + leave the file alone. */ + int dry_count = 0; + struct urlpos *dry; + for (dry = links; dry; dry = dry->next) + if (dry->convert != CO_NOCONVERT) + ++dry_count; + if (!dry_count) + { + logputs (LOG_VERBOSE, _("nothing to do.\n")); + return; + } + logprintf (LOG_VERBOSE, _("%d.\n"), dry_count); + } + + fm = wget_read_file (file); + if (!fm) + { + logprintf (LOG_NOTQUIET, _("Cannot convert links in %s: %s\n"), + file, strerror (errno)); + return; + } + + downloaded_file_return = downloaded_file (CHECK_FOR_FILE, file); + if (opt.backup_converted && downloaded_file_return) + write_backup_file (file, downloaded_file_return); + + /* Before opening the file for writing, unlink the file. This is + important if the data in FM is mapped. In such case, nulling the + file, which is what fopen() below does, would make us read all + zeroes from the mapped region. */ + if (unlink (file) < 0 && errno != ENOENT) + { + logprintf (LOG_NOTQUIET, _("Unable to delete %s: %s\n"), + quote (file), strerror (errno)); + wget_read_file_free (fm); + return; + } + /* Now open the file for writing. */ + fp = fopen (file, "wb"); + if (!fp) + { + logprintf (LOG_NOTQUIET, _("Cannot convert links in %s: %s\n"), + file, strerror (errno)); + wget_read_file_free (fm); + return; + } + + /* Here we loop through all the URLs in file, replacing those of + them that are downloaded with relative references. */ + p = fm->content; + for (link = links; link; link = link->next) + { + char *url_start = fm->content + link->pos; + + if (link->pos >= fm->length) + { + DEBUGP (("Something strange is going on. Please investigate.")); + break; + } + /* If the URL is not to be converted, skip it. */ + if (link->convert == CO_NOCONVERT) + { + DEBUGP (("Skipping %s at position %d.\n", link->url->url, link->pos)); + continue; + } + + /* Echo the file contents, up to the offending URL's opening + quote, to the outfile. */ + fwrite (p, 1, url_start - p, fp); + p = url_start; + + switch (link->convert) + { + case CO_CONVERT_TO_RELATIVE: + /* Convert absolute URL to relative. */ + if (link->local_name) { + char *newname = construct_relative (file, link->local_name); + char *quoted_newname = local_quote_string (newname, + link->link_css_p); + + if (link->link_css_p || link->link_noquote_html_p) + p = replace_plain (p, link->size, fp, quoted_newname); + else if (!link->link_refresh_p) + p = replace_attr (p, link->size, fp, quoted_newname); + else + p = replace_attr_refresh_hack (p, link->size, fp, quoted_newname, + link->refresh_timeout); + + DEBUGP (("TO_RELATIVE: %s to %s at position %d in %s.\n", + link->url->url, newname, link->pos, file)); + + xfree (newname); + xfree (quoted_newname); + ++to_file_count; + } + break; + case CO_CONVERT_BASENAME_ONLY: + { + char *newname = convert_basename (p, link); + char *quoted_newname = local_quote_string (newname, link->link_css_p); + + if (link->link_css_p || link->link_noquote_html_p) + p = replace_plain (p, link->size, fp, quoted_newname); + else if (!link->link_refresh_p) + p = replace_attr (p, link->size, fp, quoted_newname); + else + p = replace_attr_refresh_hack (p, link->size, fp, quoted_newname, + link->refresh_timeout); + + DEBUGP (("Converted file part only: %s to %s at position %d in %s.\n", + link->url->url, newname, link->pos, file)); + + xfree (newname); + xfree (quoted_newname); + ++to_file_count; + + break; + } + case CO_CONVERT_TO_COMPLETE: + /* Convert the link to absolute URL. */ + { + char *newlink = link->url->url; + char *quoted_newlink = html_quote_string (newlink); + + if (link->link_css_p || link->link_noquote_html_p) + p = replace_plain (p, link->size, fp, newlink); + else if (!link->link_refresh_p) + p = replace_attr (p, link->size, fp, quoted_newlink); + else + p = replace_attr_refresh_hack (p, link->size, fp, quoted_newlink, + link->refresh_timeout); + + DEBUGP (("TO_COMPLETE: <something> to %s at position %d in %s.\n", + newlink, link->pos, file)); + + xfree (quoted_newlink); + ++to_url_count; + break; + } + case CO_NULLIFY_BASE: + /* Change the base href to "". */ + p = replace_attr (p, link->size, fp, ""); + break; + case CO_NOCONVERT: + abort (); + break; + } + } + + /* Output the rest of the file. */ + if (p - fm->content < fm->length) + fwrite (p, 1, fm->length - (p - fm->content), fp); + fclose (fp); + wget_read_file_free (fm); + + logprintf (LOG_VERBOSE, "%d-%d\n", to_file_count, to_url_count); +} + +/* Construct and return a link that points from BASEFILE to LINKFILE. + Both files should be local file names, BASEFILE of the referrering + file, and LINKFILE of the referred file. + + Examples: + + cr("foo", "bar") -> "bar" + cr("A/foo", "A/bar") -> "bar" + cr("A/foo", "A/B/bar") -> "B/bar" + cr("A/X/foo", "A/Y/bar") -> "../Y/bar" + cr("X/", "Y/bar") -> "../Y/bar" (trailing slash does matter in BASE) + + Both files should be absolute or relative, otherwise strange + results might ensue. The function makes no special efforts to + handle "." and ".." in links, so make sure they're not there + (e.g. using path_simplify). */ + +static char * +construct_relative (const char *basefile, const char *linkfile) +{ + char *link; + int basedirs; + const char *b, *l; + int i, start; + + /* First, skip the initial directory components common to both + files. */ + start = 0; + for (b = basefile, l = linkfile; *b == *l && *b != '\0'; ++b, ++l) + { + if (*b == '/') + start = (b - basefile) + 1; + } + basefile += start; + linkfile += start; + + /* With common directories out of the way, the situation we have is + as follows: + b - b1/b2/[...]/bfile + l - l1/l2/[...]/lfile + + The link we're constructing needs to be: + lnk - ../../l1/l2/[...]/lfile + + Where the number of ".."'s equals the number of bN directory + components in B. */ + + /* Count the directory components in B. */ + basedirs = 0; + for (b = basefile; *b; b++) + { + if (*b == '/') + ++basedirs; + } + + if (!basedirs && (b = strpbrk (linkfile, "/:")) && *b == ':') + { + link = xmalloc (2 + strlen (linkfile) + 1); + memcpy (link, "./", 2); + strcpy (link + 2, linkfile); + } + else + { + /* Construct LINK as explained above. */ + link = xmalloc (3 * basedirs + strlen (linkfile) + 1); + for (i = 0; i < basedirs; i++) + memcpy (link + 3 * i, "../", 3); + strcpy (link + 3 * i, linkfile); + } + + return link; +} + +/* Construct and return a "transparent proxy" URL + reflecting changes made by --adjust-extension to the file component + (i.e., "basename") of the original URL, but leaving the "dirname" + of the URL (protocol://hostname... portion) untouched. + + Think: populating a squid cache via a recursive wget scrape, where + changing URLs to work locally with "file://..." is NOT desirable. + + Example: + + if + p = "//foo.com/bar.cgi?xyz" + and + link->local_name = "docroot/foo.com/bar.cgi?xyz.css" + then + + new_construct_func(p, link); + will return + "//foo.com/bar.cgi?xyz.css" + + Essentially, we do s/$(basename orig_url)/$(basename link->local_name)/ +*/ +static char * +convert_basename (const char *p, const struct urlpos *link) +{ + int len = link->size; + char *url = NULL; + char *org_basename = NULL, *local_basename; + char *result = NULL; + + if (*p == '"' || *p == '\'') + { + len -= 2; + p++; + } + + url = xstrndup (p, len); + + org_basename = strrchr (url, '/'); + if (org_basename) + org_basename++; + else + org_basename = url; + + local_basename = link->local_name ? strrchr (link->local_name, '/') : NULL; + if (local_basename) + local_basename++; + else + local_basename = url; + + /* + * If the basenames differ, graft the adjusted basename (local_basename) + * onto the original URL. + */ + if (strcmp (org_basename, local_basename) == 0) + result = url; + else + { + result = uri_merge (url, local_basename); + xfree (url); + } + + return result; +} + +/* Used by write_backup_file to remember which files have been + written. */ +static struct hash_table *converted_files; + +static void +write_backup_file (const char *file, downloaded_file_t downloaded_file_return) +{ + /* Rather than just writing over the original .html file with the + converted version, save the former to *.orig. Note we only do + this for files we've _successfully_ downloaded, so we don't + clobber .orig files sitting around from previous invocations. + On VMS, use "_orig" instead of ".orig". See "wget.h". */ + + if (!converted_files) + converted_files = make_string_hash_table (0); + + /* We can get called twice on the same URL thanks to the + convert_all_links() call in main. If we write the .orig file + each time in such a case, it'll end up containing the first-pass + conversion, not the original file. So, see if we've already been + called on this file. */ + if (!string_set_contains (converted_files, file)) + { + /* Construct the backup filename as the original name plus ".orig". */ + char buf[1024]; + size_t filename_len = strlen (file); + char *filename_plus_orig_suffix; + + if (filename_len < sizeof (buf) - 5) + filename_plus_orig_suffix = buf; + else + filename_plus_orig_suffix = xmalloc (filename_len + 5 + 1); + + /* TODO: hack this to work with css files */ + if (downloaded_file_return == FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED) + { + /* Just write "orig" over "html". We need to do it this way + because when we're checking to see if we've downloaded the + file before (to see if we can skip downloading it), we don't + know if it's a text/html file. Therefore we don't know yet + at that stage that -E is going to cause us to tack on + ".html", so we need to compare vs. the original URL plus + ".orig", not the original URL plus ".html.orig". */ + memcpy (filename_plus_orig_suffix, file, filename_len - 4); + memcpy (filename_plus_orig_suffix + filename_len - 4, "orig", 5); + } + else /* downloaded_file_return == FILE_DOWNLOADED_NORMALLY */ + { + /* Append ".orig" to the name. */ + memcpy (filename_plus_orig_suffix, file, filename_len); + strcpy (filename_plus_orig_suffix + filename_len, ORIG_SFX); + } + + /* Rename <file> to <file>.orig before former gets written over. */ + if (rename (file, filename_plus_orig_suffix) != 0) + logprintf (LOG_NOTQUIET, _("Cannot back up %s as %s: %s\n"), + file, filename_plus_orig_suffix, strerror (errno)); + + if (filename_plus_orig_suffix != buf) + xfree (filename_plus_orig_suffix); + + /* Remember that we've already written a .orig backup for this file. + Note that we never free this memory since we need it till the + convert_all_links() call, which is one of the last things the + program does before terminating. BTW, I'm not sure if it would be + safe to just set 'converted_file_ptr->string' to 'file' below, + rather than making a copy of the string... Another note is that I + thought I could just add a field to the urlpos structure saying + that we'd written a .orig file for this URL, but that didn't work, + so I had to make this separate list. + -- Dan Harkless <wget@harkless.org> + + This [adding a field to the urlpos structure] didn't work + because convert_file() is called from convert_all_links at + the end of the retrieval with a freshly built new urlpos + list. + -- Hrvoje Niksic <hniksic@xemacs.org> + */ + string_set_add (converted_files, file); + } +} + +static bool find_fragment (const char *, int, const char **, const char **); + +/* Replace a string with NEW_TEXT. Ignore quoting. */ +static const char * +replace_plain (const char *p, int size, FILE *fp, const char *new_text) +{ + fputs (new_text, fp); + p += size; + return p; +} + +/* Replace an attribute's original text with NEW_TEXT. */ + +static const char * +replace_attr (const char *p, int size, FILE *fp, const char *new_text) +{ + bool quote_flag = false; + char quote_char = '\"'; /* use "..." for quoting, unless the + original value is quoted, in which + case reuse its quoting char. */ + const char *frag_beg, *frag_end; + + /* Structure of our string is: + "...old-contents..." + <--- size ---> (with quotes) + OR: + ...old-contents... + <--- size --> (no quotes) */ + + if (*p == '\"' || *p == '\'') + { + quote_char = *p; + quote_flag = true; + ++p; + size -= 2; /* disregard opening and closing quote */ + } + putc (quote_char, fp); + fputs (new_text, fp); + + /* Look for fragment identifier, if any. */ + if (find_fragment (p, size, &frag_beg, &frag_end)) + fwrite (frag_beg, 1, frag_end - frag_beg, fp); + p += size; + if (quote_flag) + ++p; + putc (quote_char, fp); + + return p; +} + +/* The same as REPLACE_ATTR, but used when replacing + <meta http-equiv=refresh content="new_text"> because we need to + append "timeout_value; URL=" before the next_text. */ + +static const char * +replace_attr_refresh_hack (const char *p, int size, FILE *fp, + const char *new_text, int timeout) +{ + /* "0; URL=..." */ + char new_with_timeout[1024]; + + if (((unsigned) snprintf ( + new_with_timeout, sizeof (new_with_timeout), + "%d; URL=%s", timeout, new_text)) >= sizeof (new_with_timeout)) + { + // very unlikely fallback using heap memory + char *tmp = aprintf("%d; URL=%s", timeout, new_text); + const char *res = replace_attr (p, size, fp, tmp); + xfree (tmp); + return res; + } + + return replace_attr (p, size, fp, new_with_timeout); +} + +/* Find the first occurrence of '#' in [BEG, BEG+SIZE) that is not + preceded by '&'. If the character is not found, return zero. If + the character is found, return true and set BP and EP to point to + the beginning and end of the region. + + This is used for finding the fragment indentifiers in URLs. */ + +static bool +find_fragment (const char *beg, int size, const char **bp, const char **ep) +{ + const char *end = beg + size; + bool saw_amp = false; + for (; beg < end; beg++) + { + switch (*beg) + { + case '&': + saw_amp = true; + break; + case '#': + if (!saw_amp) + { + *bp = beg; + *ep = end; + return true; + } + /* fallthrough */ + default: + saw_amp = false; + } + } + return false; +} + +/* Quote FILE for use as local reference to an HTML file. + + We quote ? as %3F to avoid passing part of the file name as the + parameter when browsing the converted file through HTTP. However, + it is safe to do this only when `--adjust-extension' is turned on. + This is because converting "index.html?foo=bar" to + "index.html%3Ffoo=bar" would break local browsing, as the latter + isn't even recognized as an HTML file! However, converting + "index.html?foo=bar.html" to "index.html%3Ffoo=bar.html" should be + safe for both local and HTTP-served browsing. + + We always quote "#" as "%23", "%" as "%25" and ";" as "%3B" + because those characters have special meanings in URLs. */ + +static char * +local_quote_string (const char *file, bool no_html_quote) +{ + const char *from; + char *newname, *to, *res; + char buf[1024]; + size_t tolen; + + char *any = strpbrk (file, "?#%;"); + if (!any) + return no_html_quote ? strdup (file) : html_quote_string (file); + + /* Allocate space assuming the worst-case scenario, each character + having to be quoted. */ + tolen = 3 * strlen (file); + if (tolen < sizeof (buf)) + to = newname = buf; + else + to = newname = xmalloc (tolen + 1); + + for (from = file; *from; from++) + switch (*from) + { + case '%': + *to++ = '%'; + *to++ = '2'; + *to++ = '5'; + break; + case '#': + *to++ = '%'; + *to++ = '2'; + *to++ = '3'; + break; + case ';': + *to++ = '%'; + *to++ = '3'; + *to++ = 'B'; + break; + case '?': + if (opt.adjust_extension) + { + *to++ = '%'; + *to++ = '3'; + *to++ = 'F'; + break; + } + /* fallthrough */ + default: + *to++ = *from; + } + *to = '\0'; + + if (newname == buf) + return no_html_quote ? strdup (newname) : html_quote_string (newname); + + if (no_html_quote) + return newname; + + res = html_quote_string (newname); + xfree (newname); + return res; +} + +/* Book-keeping code for dl_file_url_map, dl_url_file_map, + downloaded_html_list, and downloaded_html_set. Other code calls + these functions to let us know that a file has been downloaded. */ + +#define ENSURE_TABLES_EXIST do { \ + if (!dl_file_url_map) \ + dl_file_url_map = make_string_hash_table (0); \ + if (!dl_url_file_map) \ + dl_url_file_map = make_string_hash_table (0); \ +} while (0) + +/* Return true if S1 and S2 are the same, except for "/index.html". + The three cases in which it returns one are (substitute any + substring for "foo"): + + m("foo/index.html", "foo/") ==> 1 + m("foo/", "foo/index.html") ==> 1 + m("foo", "foo/index.html") ==> 1 + m("foo", "foo/" ==> 1 + m("foo", "foo") ==> 1 */ + +static bool +match_except_index (const char *s1, const char *s2) +{ + int i; + const char *lng; + + /* Skip common substring. */ + for (i = 0; *s1 && *s2 && *s1 == *s2; s1++, s2++, i++) + ; + if (i == 0) + /* Strings differ at the very beginning -- bail out. We need to + check this explicitly to avoid `lng - 1' reading outside the + array. */ + return false; + + if (!*s1 && !*s2) + /* Both strings hit EOF -- strings are equal. */ + return true; + else if (*s1 && *s2) + /* Strings are randomly different, e.g. "/foo/bar" and "/foo/qux". */ + return false; + else if (*s1) + /* S1 is the longer one. */ + lng = s1; + else + /* S2 is the longer one. */ + lng = s2; + + /* foo */ /* foo/ */ + /* foo/index.html */ /* or */ /* foo/index.html */ + /* ^ */ /* ^ */ + + if (*lng != '/') + /* The right-hand case. */ + --lng; + + if (*lng == '/' && *(lng + 1) == '\0') + /* foo */ + /* foo/ */ + return true; + + return 0 == strcmp (lng, "/index.html"); +} + +static int +dissociate_urls_from_file_mapper (void *key, void *value, void *arg) +{ + char *mapping_url = (char *)key; + char *mapping_file = (char *)value; + char *file = (char *)arg; + + if (0 == strcmp (mapping_file, file)) + { + hash_table_remove (dl_url_file_map, mapping_url); + xfree (mapping_url); + xfree (mapping_file); + } + + /* Continue mapping. */ + return 0; +} + +/* Remove all associations from various URLs to FILE from dl_url_file_map. */ + +static void +dissociate_urls_from_file (const char *file) +{ + /* Can't use hash_table_iter_* because the table mutates while mapping. */ + hash_table_for_each (dl_url_file_map, dissociate_urls_from_file_mapper, + (char *) file); +} + +/* Register that URL has been successfully downloaded to FILE. This + is used by the link conversion code to convert references to URLs + to references to local files. It is also being used to check if a + URL has already been downloaded. */ + +void +register_download (const char *url, const char *file) +{ + char *old_file, *old_url; + + ENSURE_TABLES_EXIST; + + /* With some forms of retrieval, it is possible, although not likely + or particularly desirable. If both are downloaded, the second + download will override the first one. When that happens, + dissociate the old file name from the URL. */ + + if (hash_table_get_pair (dl_file_url_map, file, &old_file, &old_url)) + { + if (0 == strcmp (url, old_url)) + /* We have somehow managed to download the same URL twice. + Nothing to do. */ + return; + + if (match_except_index (url, old_url) + && !hash_table_contains (dl_url_file_map, url)) + /* The two URLs differ only in the "index.html" ending. For + example, one is "http://www.server.com/", and the other is + "http://www.server.com/index.html". Don't remove the old + one, just add the new one as a non-canonical entry. */ + goto url_only; + + hash_table_remove (dl_file_url_map, file); + xfree (old_file); + xfree (old_url); + + /* Remove all the URLs that point to this file. Yes, there can + be more than one such URL, because we store redirections as + multiple entries in dl_url_file_map. For example, if URL1 + redirects to URL2 which gets downloaded to FILE, we map both + URL1 and URL2 to FILE in dl_url_file_map. (dl_file_url_map + only points to URL2.) When another URL gets loaded to FILE, + we want both URL1 and URL2 dissociated from it. + + This is a relatively expensive operation because it performs + a linear search of the whole hash table, but it should be + called very rarely, only when two URLs resolve to the same + file name, *and* the "<file>.1" extensions are turned off. + In other words, almost never. */ + dissociate_urls_from_file (file); + } + + hash_table_put (dl_file_url_map, xstrdup (file), xstrdup (url)); + + url_only: + /* A URL->FILE mapping is not possible without a FILE->URL mapping. + If the latter were present, it should have been removed by the + above `if'. So we could write: + + assert (!hash_table_contains (dl_url_file_map, url)); + + The above is correct when running in recursive mode where the + same URL always resolves to the same file. But if you do + something like: + + wget URL URL + + then the first URL will resolve to "FILE", and the other to + "FILE.1". In that case, FILE.1 will not be found in + dl_file_url_map, but URL will still point to FILE in + dl_url_file_map. */ + if (hash_table_get_pair (dl_url_file_map, url, &old_url, &old_file)) + { + hash_table_remove (dl_url_file_map, url); + xfree (old_url); + xfree (old_file); + } + + hash_table_put (dl_url_file_map, xstrdup (url), xstrdup (file)); +} + +/* Register that FROM has been redirected to "TO". This assumes that TO + is successfully downloaded and already registered using + register_download() above. */ + +void +register_redirection (const char *from, const char *to) +{ + char *file; + + ENSURE_TABLES_EXIST; + + file = hash_table_get (dl_url_file_map, to); + assert (file != NULL); + if (!hash_table_contains (dl_url_file_map, from)) + hash_table_put (dl_url_file_map, xstrdup (from), xstrdup (file)); +} + +/* Register that the file has been deleted. */ + +void +register_delete_file (const char *file) +{ + char *old_url, *old_file; + + ENSURE_TABLES_EXIST; + + if (!hash_table_get_pair (dl_file_url_map, file, &old_file, &old_url)) + return; + + hash_table_remove (dl_file_url_map, file); + xfree (old_file); + xfree (old_url); + dissociate_urls_from_file (file); +} + +/* Register that FILE is an HTML file that has been downloaded. */ + +void +register_html (const char *file) +{ + if (!downloaded_html_set) + downloaded_html_set = make_string_hash_table (0); + string_set_add (downloaded_html_set, file); +} + +/* Register that FILE is a CSS file that has been downloaded. */ + +void +register_css (const char *file) +{ + if (!downloaded_css_set) + downloaded_css_set = make_string_hash_table (0); + string_set_add (downloaded_css_set, file); +} + +/* Cleanup the data structures associated with this file. */ + +#if defined DEBUG_MALLOC || defined TESTING +static void downloaded_files_free (void); + +void +convert_cleanup (void) +{ + if (dl_file_url_map) + { + free_keys_and_values (dl_file_url_map); + hash_table_destroy (dl_file_url_map); + dl_file_url_map = NULL; + } + if (dl_url_file_map) + { + free_keys_and_values (dl_url_file_map); + hash_table_destroy (dl_url_file_map); + dl_url_file_map = NULL; + } + if (downloaded_html_set) + string_set_free (downloaded_html_set); + if (downloaded_css_set) + string_set_free (downloaded_css_set); + downloaded_files_free (); + if (converted_files) + string_set_free (converted_files); +} +#endif + +/* Book-keeping code for downloaded files that enables extension + hacks. */ + +/* This table should really be merged with dl_file_url_map and + downloaded_html_files. This was originally a list, but I changed + it to a hash table because it was actually taking a lot of time to + find things in it. */ + +static struct hash_table *downloaded_files_hash; + +/* We're storing "modes" of type downloaded_file_t in the hash table. + However, our hash tables only accept pointers for keys and values. + So when we need a pointer, we use the address of a + downloaded_file_t variable of static storage. */ + +static downloaded_file_t * +downloaded_mode_to_ptr (downloaded_file_t mode) +{ + static downloaded_file_t + v1 = FILE_NOT_ALREADY_DOWNLOADED, + v2 = FILE_DOWNLOADED_NORMALLY, + v3 = FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, + v4 = CHECK_FOR_FILE; + + switch (mode) + { + case FILE_NOT_ALREADY_DOWNLOADED: + return &v1; + case FILE_DOWNLOADED_NORMALLY: + return &v2; + case FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED: + return &v3; + case CHECK_FOR_FILE: + return &v4; + } + return NULL; +} + +/* Remembers which files have been downloaded. In the standard case, + should be called with mode == FILE_DOWNLOADED_NORMALLY for each + file we actually download successfully (i.e. not for ones we have + failures on or that we skip due to -N). + + When we've downloaded a file and tacked on a ".html" extension due + to -E, call this function with + FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED rather than + FILE_DOWNLOADED_NORMALLY. + + If you just want to check if a file has been previously added + without adding it, call with mode == CHECK_FOR_FILE. Please be + sure to call this function with local filenames, not remote + URLs. */ + +downloaded_file_t +downloaded_file (downloaded_file_t mode, const char *file) +{ + downloaded_file_t *ptr; + + if (mode == CHECK_FOR_FILE) + { + if (!downloaded_files_hash) + return FILE_NOT_ALREADY_DOWNLOADED; + ptr = hash_table_get (downloaded_files_hash, file); + if (!ptr) + return FILE_NOT_ALREADY_DOWNLOADED; + return *ptr; + } + + if (!downloaded_files_hash) + downloaded_files_hash = make_string_hash_table (0); + + ptr = hash_table_get (downloaded_files_hash, file); + if (ptr) + return *ptr; + + ptr = downloaded_mode_to_ptr (mode); + hash_table_put (downloaded_files_hash, xstrdup (file), ptr); + + return FILE_NOT_ALREADY_DOWNLOADED; +} + +#if defined DEBUG_MALLOC || defined TESTING +static void +downloaded_files_free (void) +{ + if (downloaded_files_hash) + { + hash_table_iterator iter; + for (hash_table_iterate (downloaded_files_hash, &iter); + hash_table_iter_next (&iter); + ) + xfree (iter.key); + hash_table_destroy (downloaded_files_hash); + downloaded_files_hash = NULL; + } +} +#endif + +/* The function returns the pointer to the malloc-ed quoted version of + string s. It will recognize and quote numeric and special graphic + entities, as per RFC1866: + + `&' -> `&' + `<' -> `<' + `>' -> `>' + `"' -> `"' + SP -> ` ' + + No other entities are recognized or replaced. */ +char * +html_quote_string (const char *s) +{ + const char *b = s; + char *p, *res; + int i; + + /* Pass through the string, and count the new size. */ + for (i = 0; *s; s++, i++) + { + if (*s == '&') + i += 4; /* `amp;' */ + else if (*s == '<' || *s == '>') + i += 3; /* `lt;' and `gt;' */ + else if (*s == '\"') + i += 5; /* `quot;' */ + else if (*s == ' ') + i += 4; /* #32; */ + } + res = xmalloc (i + 1); + s = b; + for (p = res; *s; s++) + { + switch (*s) + { + case '&': + *p++ = '&'; + *p++ = 'a'; + *p++ = 'm'; + *p++ = 'p'; + *p++ = ';'; + break; + case '<': case '>': + *p++ = '&'; + *p++ = (*s == '<' ? 'l' : 'g'); + *p++ = 't'; + *p++ = ';'; + break; + case '\"': + *p++ = '&'; + *p++ = 'q'; + *p++ = 'u'; + *p++ = 'o'; + *p++ = 't'; + *p++ = ';'; + break; + case ' ': + *p++ = '&'; + *p++ = '#'; + *p++ = '3'; + *p++ = '2'; + *p++ = ';'; + break; + default: + *p++ = *s; + } + } + *p = '\0'; + return res; +} + +/* + * vim: et ts=2 sw=2 + */ diff --git a/src/convert.h b/src/convert.h new file mode 100644 index 0000000..531afbd --- /dev/null +++ b/src/convert.h @@ -0,0 +1,115 @@ +/* Declarations for convert.c + Copyright (C) 2003-2006, 2009-2011, 2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef CONVERT_H +#define CONVERT_H + +struct hash_table; /* forward decl */ +extern struct hash_table *dl_url_file_map; +extern struct hash_table *downloaded_html_set; +extern struct hash_table *downloaded_css_set; + +enum convert_options { + CO_NOCONVERT = 0, /* don't convert this URL */ + CO_CONVERT_TO_RELATIVE, /* convert to relative, e.g. to + "../../otherdir/foo.gif" */ + CO_CONVERT_BASENAME_ONLY, /* convert the file portion only (basename) + leaving the rest of the URL unchanged */ + CO_CONVERT_TO_COMPLETE, /* convert to absolute, e.g. to + "http://orighost/somedir/bar.jpg". */ + CO_NULLIFY_BASE /* change to empty string. */ +}; + +struct url; + +/* A structure that defines the whereabouts of a URL, i.e. its + position in an HTML document, etc. */ + +struct urlpos { + struct url *url; /* the URL of the link, after it has + been merged with the base */ + char *local_name; /* local file to which it was saved + (used by convert_links) */ + + /* reserved for special links such as <base href="..."> which are + used when converting links, but ignored when downloading. */ + unsigned int ignore_when_downloading :1; + + /* Information about the original link: */ + + unsigned int link_relative_p :1; /* the link was relative */ + unsigned int link_complete_p :1; /* the link was complete (had host name) */ + unsigned int link_base_p :1; /* the url came from <base href=...> */ + unsigned int link_inline_p :1; /* needed to render the page */ + unsigned int link_css_p :1; /* the url came from CSS */ + unsigned int link_noquote_html_p :1; /* from HTML, but doesn't need " */ + unsigned int link_expect_html :1; /* expected to contain HTML */ + unsigned int link_expect_css :1; /* expected to contain CSS */ + + unsigned int link_refresh_p :1; /* link was received from + <meta http-equiv=refresh content=...> */ + int refresh_timeout; /* for reconstructing the refresh. */ + + /* Conversion requirements: */ + enum convert_options convert; /* is conversion required? */ + + /* URL's position in the buffer. */ + int pos, size; + + struct urlpos *next; /* next list element */ +}; + +/* downloaded_file() takes a parameter of this type and returns this type. */ +typedef enum +{ + /* Return enumerators: */ + FILE_NOT_ALREADY_DOWNLOADED = 0, + + /* Return / parameter enumerators: */ + FILE_DOWNLOADED_NORMALLY, + FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, + + /* Parameter enumerators: */ + CHECK_FOR_FILE +} downloaded_file_t; + +downloaded_file_t downloaded_file (downloaded_file_t, const char *); + +void register_download (const char *, const char *); +void register_redirection (const char *, const char *); +void register_html (const char *); +void register_css (const char *); +void register_delete_file (const char *); +void convert_all_links (void); +void convert_cleanup (void); + +char *html_quote_string (const char *); + +#endif /* CONVERT_H */ diff --git a/src/cookies.c b/src/cookies.c new file mode 100644 index 0000000..6ddb931 --- /dev/null +++ b/src/cookies.c @@ -0,0 +1,1539 @@ +/* Support for cookies. + Copyright (C) 2001-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +/* Written by Hrvoje Niksic. Parts are loosely inspired by the + cookie patch submitted by Tomasz Wegrzanowski. + + This implements the client-side cookie support, as specified + (loosely) by Netscape's "preliminary specification", currently + available at: + + http://wp.netscape.com/newsref/std/cookie_spec.html + + rfc2109 is not supported because of its incompatibilities with the + above widely-used specification. rfc2965 is entirely ignored, + since popular client software doesn't implement it, and even the + sites that do send Set-Cookie2 also emit Set-Cookie for + compatibility. */ + +#include "wget.h" + +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <assert.h> +#include <errno.h> +#include <time.h> +#ifdef HAVE_LIBPSL +# include <libpsl.h> +#endif +#include "utils.h" +#include "hash.h" +#include "cookies.h" +#include "http.h" /* for http_atotm */ +#include "c-strcase.h" + + +/* Declarations of `struct cookie' and the most basic functions. */ + +/* Cookie jar serves as cookie storage and a means of retrieving + cookies efficiently. All cookies with the same domain are stored + in a linked list called "chain". A cookie chain can be reached by + looking up the domain in the cookie jar's chains_by_domain table. + + For example, to reach all the cookies under google.com, one must + execute hash_table_get(jar->chains_by_domain, "google.com"). Of + course, when sending a cookie to `www.google.com', one must search + for cookies that belong to either `www.google.com' or `google.com' + -- but the point is that the code doesn't need to go through *all* + the cookies. */ + +struct cookie_jar { + /* Cookie chains indexed by domain. */ + struct hash_table *chains; + + int cookie_count; /* number of cookies in the jar. */ +}; + +/* Value set by entry point functions, so that the low-level + routines don't need to call time() all the time. */ +static time_t cookies_now; + +struct cookie_jar * +cookie_jar_new (void) +{ + struct cookie_jar *jar = xnew (struct cookie_jar); + jar->chains = make_nocase_string_hash_table (0); + jar->cookie_count = 0; + return jar; +} + +struct cookie { + char *domain; /* domain of the cookie */ + int port; /* port number */ + char *path; /* path prefix of the cookie */ + + unsigned discard_requested :1;/* whether cookie was created to + request discarding another + cookie. */ + + unsigned secure :1; /* whether cookie should be + transmitted over non-https + connections. */ + unsigned domain_exact :1; /* whether DOMAIN must match as a + whole. */ + + unsigned permanent :1; /* whether the cookie should outlive + the session. */ + time_t expiry_time; /* time when the cookie expires, 0 + means undetermined. */ + + char *attr; /* cookie attribute name */ + char *value; /* cookie attribute value */ + + struct cookie *next; /* used for chaining of cookies in the + same domain. */ +}; + +#define PORT_ANY (-1) + +/* Allocate and return a new, empty cookie structure. */ + +static struct cookie * +cookie_new (void) +{ + struct cookie *cookie = xnew0 (struct cookie); + + /* Both cookie->permanent and cookie->expiry_time are now 0. This + means that the cookie doesn't expire, but is only valid for this + session (i.e. not written out to disk). */ + + cookie->port = PORT_ANY; + return cookie; +} + +/* Non-zero if the cookie has expired. Assumes cookies_now has been + set by one of the entry point functions. */ + +static bool +cookie_expired_p (const struct cookie *c) +{ + return c->expiry_time != 0 && c->expiry_time < cookies_now; +} + +/* Deallocate COOKIE and its components. */ + +static void +delete_cookie (struct cookie *cookie) +{ + xfree (cookie->domain); + xfree (cookie->path); + xfree (cookie->attr); + xfree (cookie->value); + xfree (cookie); +} + +/* Functions for storing cookies. + + All cookies can be reached beginning with jar->chains. The key in + that table is the domain name, and the value is a linked list of + all cookies from that domain. Every new cookie is placed on the + head of the list. */ + +/* Find and return a cookie in JAR whose domain, path, and attribute + name correspond to COOKIE. If found, PREVPTR will point to the + location of the cookie previous in chain, or NULL if the found + cookie is the head of a chain. + + If no matching cookie is found, return NULL. */ + +static struct cookie * +find_matching_cookie (struct cookie_jar *jar, struct cookie *cookie, + struct cookie **prevptr) +{ + struct cookie *chain, *prev; + + chain = hash_table_get (jar->chains, cookie->domain); + if (!chain) + goto nomatch; + + prev = NULL; + for (; chain; prev = chain, chain = chain->next) + if (0 == strcmp (cookie->path, chain->path) + && 0 == strcmp (cookie->attr, chain->attr) + && cookie->port == chain->port) + { + *prevptr = prev; + return chain; + } + + nomatch: + *prevptr = NULL; + return NULL; +} + +/* Store COOKIE to the jar. + + This is done by placing COOKIE at the head of its chain. However, + if COOKIE matches a cookie already in memory, as determined by + find_matching_cookie, the old cookie is unlinked and destroyed. + + The key of each chain's hash table entry is allocated only the + first time; next hash_table_put's reuse the same key. */ + +static void +store_cookie (struct cookie_jar *jar, struct cookie *cookie) +{ + struct cookie *chain_head; + char *chain_key; + + if (hash_table_get_pair (jar->chains, cookie->domain, + &chain_key, &chain_head)) + { + /* A chain of cookies in this domain already exists. Check for + duplicates -- if an extant cookie exactly matches our domain, + port, path, and name, replace it. */ + struct cookie *prev; + struct cookie *victim = find_matching_cookie (jar, cookie, &prev); + + if (victim) + { + /* Remove VICTIM from the chain. COOKIE will be placed at + the head. */ + if (prev) + { + prev->next = victim->next; + cookie->next = chain_head; + } + else + { + /* prev is NULL; apparently VICTIM was at the head of + the chain. This place will be taken by COOKIE, so + all we need to do is: */ + cookie->next = victim->next; + } + delete_cookie (victim); + --jar->cookie_count; + DEBUGP (("Deleted old cookie (to be replaced.)\n")); + } + else + cookie->next = chain_head; + } + else + { + /* We are now creating the chain. Use a copy of cookie->domain + as the key for the life-time of the chain. Using + cookie->domain would be unsafe because the life-time of the + chain may exceed the life-time of the cookie. (Cookies may + be deleted from the chain by this very function.) */ + cookie->next = NULL; + chain_key = xstrdup (cookie->domain); + } + + hash_table_put (jar->chains, chain_key, cookie); + ++jar->cookie_count; + + IF_DEBUG + { + time_t exptime = cookie->expiry_time; + DEBUGP (("\nStored cookie %s %d%s %s <%s> <%s> [expiry %s] %s %s\n", + cookie->domain, cookie->port, + cookie->port == PORT_ANY ? " (ANY)" : "", + cookie->path, + cookie->permanent ? "permanent" : "session", + cookie->secure ? "secure" : "insecure", + cookie->expiry_time ? datetime_str (exptime) : "none", + cookie->attr, cookie->value)); + } +} + +/* Discard a cookie matching COOKIE's domain, port, path, and + attribute name. This gets called when we encounter a cookie whose + expiry date is in the past, or whose max-age is set to 0. The + former corresponds to netscape cookie spec, while the latter is + specified by rfc2109. */ + +static void +discard_matching_cookie (struct cookie_jar *jar, struct cookie *cookie) +{ + struct cookie *prev, *victim; + + if (!hash_table_count (jar->chains)) + /* No elements == nothing to discard. */ + return; + + victim = find_matching_cookie (jar, cookie, &prev); + if (victim) + { + if (prev) + /* Simply unchain the victim. */ + prev->next = victim->next; + else + { + /* VICTIM was head of its chain. We need to place a new + cookie at the head. */ + char *chain_key = NULL; + int res; + + res = hash_table_get_pair (jar->chains, victim->domain, + &chain_key, NULL); + + if (res == 0) + { + logprintf (LOG_VERBOSE, _("Unable to get cookie for %s\n"), + victim->domain); + } + if (!victim->next) + { + /* VICTIM was the only cookie in the chain. Destroy the + chain and deallocate the chain key. */ + hash_table_remove (jar->chains, victim->domain); + xfree (chain_key); + } + else + hash_table_put (jar->chains, chain_key, victim->next); + } + delete_cookie (victim); + DEBUGP (("Discarded old cookie.\n")); + } +} + +/* Functions for parsing the `Set-Cookie' header, and creating new + cookies from the wire. */ + +#define TOKEN_IS(token, string_literal) \ + BOUNDED_EQUAL_NO_CASE (token.b, token.e, string_literal) + +#define TOKEN_NON_EMPTY(token) (token.b != NULL && token.b != token.e) + +/* Parse the contents of the `Set-Cookie' header. The header looks + like this: + + name1=value1; name2=value2; ... + + Trailing semicolon is optional; spaces are allowed between all + tokens. Additionally, values may be quoted. + + A new cookie is returned upon success, NULL otherwise. + + The first name-value pair will be used to set the cookie's + attribute name and value. Subsequent parameters will be checked + against field names such as `domain', `path', etc. Recognized + fields will be parsed and the corresponding members of COOKIE + filled. */ + +static struct cookie * +parse_set_cookie (const char *set_cookie, bool silent) +{ + const char *ptr = set_cookie; + struct cookie *cookie = cookie_new (); + param_token name, value; + + if (!extract_param (&ptr, &name, &value, ';', NULL)) + goto error; + if (!value.b) + goto error; + + /* If the value is quoted, do not modify it. */ + if (*(value.b - 1) == '"') + value.b--; + if (*value.e == '"') + value.e++; + + cookie->attr = strdupdelim (name.b, name.e); + cookie->value = strdupdelim (value.b, value.e); + + while (extract_param (&ptr, &name, &value, ';', NULL)) + { + if (TOKEN_IS (name, "domain")) + { + if (!TOKEN_NON_EMPTY (value)) + goto error; + xfree (cookie->domain); + /* Strictly speaking, we should set cookie->domain_exact if the + domain doesn't begin with a dot. But many sites set the + domain to "foo.com" and expect "subhost.foo.com" to get the + cookie, and it apparently works in browsers. */ + if (*value.b == '.') + ++value.b; + cookie->domain = strdupdelim (value.b, value.e); + } + else if (TOKEN_IS (name, "path")) + { + if (!TOKEN_NON_EMPTY (value)) + goto error; + xfree (cookie->path); + cookie->path = strdupdelim (value.b, value.e); + } + else if (TOKEN_IS (name, "expires")) + { + char value_copy[128]; + size_t value_len = value.e - value.b; + time_t expires; + + if (!TOKEN_NON_EMPTY (value) || value_len >= sizeof (value_copy)) + goto error; + + memcpy (value_copy, value.b, value_len); + value_copy[value_len] = 0; + + /* Check if expiration spec is valid. + If not, assume default (cookie doesn't expire, but valid only for + this session.) */ + expires = http_atotm (value_copy); + if (expires != (time_t) -1) + { + cookie->permanent = 1; + cookie->expiry_time = expires; + /* According to netscape's specification, expiry time in + the past means that discarding of a matching cookie + is requested. */ + if (cookie->expiry_time < cookies_now) + cookie->discard_requested = 1; + } + } + else if (TOKEN_IS (name, "max-age")) + { + double maxage = -1; + char value_copy[32]; + size_t value_len = value.e - value.b; + + if (!TOKEN_NON_EMPTY (value) || value_len >= sizeof (value_copy)) + goto error; + + memcpy (value_copy, value.b, value_len); + value_copy[value_len] = 0; + + sscanf (value_copy, "%lf", &maxage); + if (maxage == -1) + /* something went wrong. */ + goto error; + cookie->permanent = 1; + cookie->expiry_time = cookies_now + (time_t) maxage; + + /* According to rfc2109, a cookie with max-age of 0 means that + discarding of a matching cookie is requested. */ + if (maxage == 0) + cookie->discard_requested = 1; + } + else if (TOKEN_IS (name, "secure")) + { + /* ignore value completely */ + cookie->secure = 1; + } + /* else: Ignore unrecognized attribute. */ + } + if (*ptr) + /* extract_param has encountered a syntax error */ + goto error; + + /* The cookie has been successfully constructed; return it. */ + return cookie; + + error: + if (!silent) + logprintf (LOG_NOTQUIET, + _("Syntax error in Set-Cookie: %s at position %d.\n"), + quotearg_style (escape_quoting_style, set_cookie), + (int) (ptr - set_cookie)); + delete_cookie (cookie); + return NULL; +} + +#undef TOKEN_IS +#undef TOKEN_NON_EMPTY + +/* Sanity checks. These are important, otherwise it is possible for + mailcious attackers to destroy important cookie information and/or + violate your privacy. */ + + +#define REQUIRE_DIGITS(p) do { \ + if (!c_isdigit (*p)) \ + return false; \ + for (++p; c_isdigit (*p); p++) \ + ; \ +} while (0) + +#define REQUIRE_DOT(p) do { \ + if (*p++ != '.') \ + return false; \ +} while (0) + +/* Check whether ADDR matches <digits>.<digits>.<digits>.<digits>. + + We don't want to call network functions like inet_addr() because + all we need is a check, preferably one that is small, fast, and + well-defined. */ + +static bool +numeric_address_p (const char *addr) +{ + const char *p = addr; + + REQUIRE_DIGITS (p); /* A */ + REQUIRE_DOT (p); /* . */ + REQUIRE_DIGITS (p); /* B */ + REQUIRE_DOT (p); /* . */ + REQUIRE_DIGITS (p); /* C */ + REQUIRE_DOT (p); /* . */ + REQUIRE_DIGITS (p); /* D */ + + if (*p != '\0') + return false; + return true; +} + +/* Check whether COOKIE_DOMAIN is an appropriate domain for HOST. + Originally I tried to make the check compliant with rfc2109, but + the sites deviated too often, so I had to fall back to "tail + matching", as defined by the original Netscape's cookie spec. + + Wget now uses libpsl to check domain names against a public suffix + list to see if they are valid. However, since we don't provide a + psl on our own, if libpsl is compiled without a public suffix list, + fall back to using the original "tail matching" heuristic. Also if + libpsl is unable to convert the domain to lowercase, which means that + it doesn't have any runtime conversion support, we again fall back to + "tail matching" since libpsl states the results are unpredictable with + upper case strings. + */ + +#ifdef HAVE_LIBPSL +static psl_ctx_t *psl; +#endif + +static bool +check_domain_match (const char *cookie_domain, const char *host) +{ +#ifdef HAVE_LIBPSL + static int init_psl; + char *cookie_domain_lower = NULL; + char *host_lower = NULL; + int is_acceptable; + + DEBUGP (("cdm: 1\n")); + if (!init_psl) + { + init_psl = 1; + +#ifdef HAVE_PSL_LATEST + if ((psl = psl_latest (NULL))) + goto have_psl; + + DEBUGP (("\nPSL: Failed to load any PSL data. " + "Falling back to insecure heuristics.\n")); +#else + if ((psl = psl_builtin ()) && !psl_builtin_outdated ()) + goto have_psl; + + DEBUGP (("\nPSL: built-in data outdated. " + "Trying to load data from %s.\n", + quote (psl_builtin_filename ()))); + + if ((psl = psl_load_file (psl_builtin_filename ()))) + goto have_psl; + + DEBUGP (("\nPSL: %s not found or not readable. " + "Falling back to built-in data.\n", + quote (psl_builtin_filename ()))); + + if (!(psl = psl_builtin ())) + { + DEBUGP (("\nPSL: libpsl not built with a public suffix list. " + "Falling back to insecure heuristics.\n")); + goto no_psl; + } +#endif + } + else if (!psl) + goto no_psl; + +have_psl: + if (psl_str_to_utf8lower (cookie_domain, NULL, NULL, &cookie_domain_lower) == PSL_SUCCESS && + psl_str_to_utf8lower (host, NULL, NULL, &host_lower) == PSL_SUCCESS) + { + is_acceptable = psl_is_cookie_domain_acceptable (psl, host_lower, cookie_domain_lower); + } + else + { + DEBUGP (("libpsl unable to parse domain name. " + "Falling back to simple heuristics.\n")); + goto no_psl; + } + + xfree (cookie_domain_lower); + xfree (host_lower); + + return is_acceptable == 1; + +no_psl: + /* Cleanup the PSL pointers first */ + xfree (cookie_domain_lower); + xfree (host_lower); +#endif + + /* For efficiency make some elementary checks first */ + DEBUGP (("cdm: 2\n")); + + /* For the sake of efficiency, check for exact match first. */ + if (0 == strcasecmp (cookie_domain, host)) + return true; + + DEBUGP (("cdm: 3\n")); + + /* HOST must match the tail of cookie_domain. */ + if (!match_tail (host, cookie_domain, true)) + return false; + + /* We know that COOKIE_DOMAIN is a subset of HOST; however, we must + make sure that somebody is not trying to set the cookie for a + subdomain shared by many entities. For example, "company.co.uk" + must not be allowed to set a cookie for ".co.uk". On the other + hand, "sso.redhat.de" should be able to set a cookie for + ".redhat.de". + + The only marginally sane way to handle this I can think of is to + reject on the basis of the length of the second-level domain name + (but when the top-level domain is unknown), with the assumption + that those of three or less characters could be reserved. For + example: + + .co.org -> works because the TLD is known + .co.uk -> doesn't work because "co" is only two chars long + .com.au -> doesn't work because "com" is only 3 chars long + .cnn.uk -> doesn't work because "cnn" is also only 3 chars long (ugh) + .cnn.de -> doesn't work for the same reason (ugh!!) + .abcd.de -> works because "abcd" is 4 chars long + .img.cnn.de -> works because it's not trying to set the 2nd level domain + .cnn.co.uk -> works for the same reason + + That should prevent misuse, while allowing reasonable usage. If + someone knows of a better way to handle this, please let me + know. */ + { + const char *p = cookie_domain; + int dccount = 1; /* number of domain components */ + int ldcl = 0; /* last domain component length */ + int nldcl = 0; /* next to last domain component length */ + int out; + if (*p == '.') + /* Ignore leading period in this calculation. */ + ++p; + DEBUGP (("cdm: 4\n")); + for (out = 0; !out; p++) + switch (*p) + { + case '\0': + out = 1; + break; + case '.': + if (ldcl == 0) + /* Empty domain component found -- the domain is invalid. */ + return false; + if (*(p + 1) == '\0') + { + /* Tolerate trailing '.' by not treating the domain as + one ending with an empty domain component. */ + out = 1; + break; + } + nldcl = ldcl; + ldcl = 0; + ++dccount; + break; + default: + ++ldcl; + } + + DEBUGP (("cdm: 5\n")); + + if (dccount < 2) + return false; + + DEBUGP (("cdm: 6\n")); + + if (dccount == 2) + { + size_t i; + int known_toplevel = false; + static const char *known_toplevel_domains[] = { + ".com", ".edu", ".net", ".org", ".gov", ".mil", ".int" + }; + for (i = 0; i < countof (known_toplevel_domains); i++) + if (match_tail (cookie_domain, known_toplevel_domains[i], true)) + { + known_toplevel = true; + break; + } + if (!known_toplevel && nldcl <= 3) + return false; + } + } + + DEBUGP (("cdm: 7\n")); + + /* Don't allow the host "foobar.com" to set a cookie for domain + "bar.com". */ + if (*cookie_domain != '.') + { + int dlen = strlen (cookie_domain); + int hlen = strlen (host); + /* cookie host: hostname.foobar.com */ + /* desired domain: bar.com */ + /* '.' must be here in host-> ^ */ + if (hlen > dlen && host[hlen - dlen - 1] != '.') + return false; + } + + DEBUGP (("cdm: 8\n")); + + return true; +} + +static int path_matches (const char *, const char *); + +/* Check whether PATH begins with COOKIE_PATH. */ + +static bool +check_path_match (const char *cookie_path, const char *path) +{ + return path_matches (path, cookie_path) != 0; +} + +/* Process the HTTP `Set-Cookie' header. This results in storing the + cookie or discarding a matching one, or ignoring it completely, all + depending on the contents. */ + +void +cookie_handle_set_cookie (struct cookie_jar *jar, + const char *host, int port, + const char *path, const char *set_cookie) +{ + struct cookie *cookie; + cookies_now = time (NULL); + char buf[1024], *tmp; + size_t pathlen = strlen(path); + + /* Wget's paths don't begin with '/' (blame rfc1808), but cookie + usage assumes /-prefixed paths. Until the rest of Wget is fixed, + simply prepend slash to PATH. */ + if (pathlen < sizeof (buf) - 1) + tmp = buf; + else + tmp = xmalloc (pathlen + 2); + + *tmp = '/'; + memcpy (tmp + 1, path, pathlen + 1); + path = tmp; + + cookie = parse_set_cookie (set_cookie, false); + if (!cookie) + goto out; + + /* Sanitize parts of cookie. */ + + if (!cookie->domain) + { + cookie->domain = xstrdup (host); + cookie->domain_exact = 1; + /* Set the port, but only if it's non-default. */ + if (port != 80 && port != 443) + cookie->port = port; + } + else + { + if (!check_domain_match (cookie->domain, host)) + { + logprintf (LOG_NOTQUIET, + _("Cookie coming from %s attempted to set domain to "), + quotearg_style (escape_quoting_style, host)); + logprintf (LOG_NOTQUIET, + _("%s\n"), + quotearg_style (escape_quoting_style, cookie->domain)); + cookie->discard_requested = true; + } + } + + if (!cookie->path) + { + /* The cookie doesn't set path: set it to the URL path, sans the + file part ("/dir/file" truncated to "/dir/"). */ + char *trailing_slash = strrchr (path, '/'); + if (trailing_slash) + cookie->path = strdupdelim (path, trailing_slash + 1); + else + /* no slash in the string -- can this even happen? */ + cookie->path = xstrdup (path); + } + else + { + /* The cookie sets its own path; verify that it is legal. */ + if (!check_path_match (cookie->path, path)) + { + DEBUGP (("Attempt to fake the path: %s, %s\n", + cookie->path, path)); + goto out; + } + } + + /* Now store the cookie, or discard an existing cookie, if + discarding was requested. */ + + if (cookie->discard_requested) + { + discard_matching_cookie (jar, cookie); + goto out; + } + + store_cookie (jar, cookie); + if (tmp != buf) + xfree (tmp); + return; + + out: + if (cookie) + delete_cookie (cookie); + if (tmp != buf) + xfree (tmp); +} + +/* Support for sending out cookies in HTTP requests, based on + previously stored cookies. Entry point is + `build_cookies_request'. */ + +/* Return a count of how many times CHR occurs in STRING. */ + +static int +count_char (const char *string, char chr) +{ + const char *p; + int count = 0; + for (p = string; *p; p++) + if (*p == chr) + ++count; + return count; +} + +/* Find the cookie chains whose domains match HOST and store them to + DEST. + + A cookie chain is the head of a list of cookies that belong to a + host/domain. Given HOST "img.search.xemacs.org", this function + will return the chains for "img.search.xemacs.org", + "search.xemacs.org", and "xemacs.org" -- those of them that exist + (if any), that is. + + DEST should be large enough to accept (in the worst case) as many + elements as there are domain components of HOST. */ + +static int +find_chains_of_host (struct cookie_jar *jar, const char *host, + struct cookie *dest[]) +{ + int dest_count = 0; + int passes, passcnt; + + /* Bail out quickly if there are no cookies in the jar. */ + if (!hash_table_count (jar->chains)) + return 0; + + if (numeric_address_p (host)) + /* If host is an IP address, only check for the exact match. */ + passes = 1; + else + /* Otherwise, check all the subdomains except the top-level (last) + one. As a domain with N components has N-1 dots, the number of + passes equals the number of dots. */ + passes = count_char (host, '.'); + + passcnt = 0; + + /* Find chains that match HOST, starting with exact match and + progressing to less specific domains. For instance, given HOST + fly.srk.fer.hr, first look for fly.srk.fer.hr's chain, then + srk.fer.hr's, then fer.hr's. */ + while (1) + { + struct cookie *chain = hash_table_get (jar->chains, host); + if (chain) + dest[dest_count++] = chain; + if (++passcnt >= passes) + break; + host = strchr (host, '.') + 1; + } + + return dest_count; +} + +/* If FULL_PATH begins with PREFIX, return the length of PREFIX, zero + otherwise. */ + +static int +path_matches (const char *full_path, const char *prefix) +{ + int len = strlen (prefix); + + if (0 != strncmp (full_path, prefix, len)) + /* FULL_PATH doesn't begin with PREFIX. */ + return 0; + + /* Length of PREFIX determines the quality of the match. */ + return len + 1; +} + +/* Return true if COOKIE matches the provided parameters of the URL + being downloaded: HOST, PORT, PATH, and SECFLAG. + + If PATH_GOODNESS is non-NULL, store the "path goodness" value + there. That value is a measure of how closely COOKIE matches PATH, + used for ordering cookies. */ + +static bool +cookie_matches_url (const struct cookie *cookie, + const char *host, int port, const char *path, + bool secflag, int *path_goodness) +{ + int pg; + + if (cookie_expired_p (cookie)) + /* Ignore stale cookies. Don't bother unchaining the cookie at + this point -- Wget is a relatively short-lived application, and + stale cookies will not be saved by `save_cookies'. On the + other hand, this function should be as efficient as + possible. */ + return false; + + if (cookie->secure && !secflag) + /* Don't transmit secure cookies over insecure connections. */ + return false; + if (cookie->port != PORT_ANY && cookie->port != port) + return false; + + /* If exact domain match is required, verify that cookie's domain is + equal to HOST. If not, assume success on the grounds of the + cookie's chain having been found by find_chains_of_host. */ + if (cookie->domain_exact + && 0 != strcasecmp (host, cookie->domain)) + return false; + + pg = path_matches (path, cookie->path); + if (pg == 0) + return false; + + if (path_goodness) + /* If the caller requested path_goodness, we return it. This is + an optimization, so that the caller doesn't need to call + path_matches() again. */ + *path_goodness = pg; + return true; +} + +/* A structure that points to a cookie, along with the additional + information about the cookie's "goodness". This allows us to sort + the cookies when returning them to the server, as required by the + spec. */ + +struct weighed_cookie { + struct cookie *cookie; + int domain_goodness; + int path_goodness; +}; + +/* Comparator used for uniquifying the list. */ + +static int +equality_comparator (const void *p1, const void *p2) +{ + struct weighed_cookie *wc1 = (struct weighed_cookie *)p1; + struct weighed_cookie *wc2 = (struct weighed_cookie *)p2; + + int namecmp = strcmp (wc1->cookie->attr, wc2->cookie->attr); + int valuecmp = strcmp (wc1->cookie->value, wc2->cookie->value); + + /* We only really care whether both name and value are equal. We + return them in this order only for consistency... */ + return namecmp ? namecmp : valuecmp; +} + +/* Eliminate duplicate cookies. "Duplicate cookies" are any two + cookies with the same attr name and value. Whenever a duplicate + pair is found, one of the cookies is removed. */ + +static int +eliminate_dups (struct weighed_cookie *outgoing, int count) +{ + struct weighed_cookie *h; /* hare */ + struct weighed_cookie *t; /* tortoise */ + struct weighed_cookie *end = outgoing + count; + + /* We deploy a simple uniquify algorithm: first sort the array + according to our sort criteria, then copy it to itself, comparing + each cookie to its neighbor and ignoring the duplicates. */ + + qsort (outgoing, count, sizeof (struct weighed_cookie), equality_comparator); + + /* "Hare" runs through all the entries in the array, followed by + "tortoise". If a duplicate is found, the hare skips it. + Non-duplicate entries are copied to the tortoise ptr. */ + + for (h = t = outgoing; h < end; h++) + { + if (h != end - 1) + { + struct cookie *c0 = h[0].cookie; + struct cookie *c1 = h[1].cookie; + if (!strcmp (c0->attr, c1->attr) && !strcmp (c0->value, c1->value)) + continue; /* ignore the duplicate */ + } + + /* If the hare has advanced past the tortoise (because of + previous dups), make sure the values get copied. Otherwise, + no copying is necessary. */ + if (h != t) + *t++ = *h; + else + t++; + } + return t - outgoing; +} + +/* Comparator used for sorting by quality. */ + +static int +goodness_comparator (const void *p1, const void *p2) +{ + struct weighed_cookie *wc1 = (struct weighed_cookie *)p1; + struct weighed_cookie *wc2 = (struct weighed_cookie *)p2; + + /* Subtractions take `wc2' as the first argument becauase we want a + sort in *decreasing* order of goodness. */ + int dgdiff = wc2->domain_goodness - wc1->domain_goodness; + int pgdiff = wc2->path_goodness - wc1->path_goodness; + + /* Sort by domain goodness; if these are the same, sort by path + goodness. (The sorting order isn't really specified; maybe it + should be the other way around.) */ + return dgdiff ? dgdiff : pgdiff; +} + +/* Generate a `Cookie' header for a request that goes to HOST:PORT and + requests PATH from the server. The resulting string is allocated + with `malloc', and the caller is responsible for freeing it. If no + cookies pertain to this request, i.e. no cookie header should be + generated, NULL is returned. */ + +char * +cookie_header (struct cookie_jar *jar, const char *host, + int port, const char *path, bool secflag) +{ + struct cookie *chains[32]; + int chain_count; + + struct cookie *cookie; + struct weighed_cookie *outgoing; + size_t count, i, ocnt; + char *result = NULL; + int result_size, pos; + char pathbuf[1024]; + + /* First, find the cookie chains whose domains match HOST. */ + + /* Allocate room for find_chains_of_host to write to. The number of + chains can at most equal the number of subdomains, hence + 1+<number of dots>. We ignore cookies with more than 32 labels. */ + chain_count = 1 + count_char (host, '.'); + if (chain_count > (int) countof (chains)) + return NULL; + chain_count = find_chains_of_host (jar, host, chains); + + /* No cookies for this host. */ + if (chain_count <= 0) + return NULL; + + /* Wget's paths don't begin with '/' (blame rfc1808), but cookie + usage assumes /-prefixed paths. Until the rest of Wget is fixed, + simply prepend slash to PATH. */ + { + char *tmp; + size_t pathlen = strlen(path); + + if (pathlen < sizeof (pathbuf) - 1) + tmp = pathbuf; + else + tmp = xmalloc (pathlen + 2); + + *tmp = '/'; + memcpy (tmp + 1, path, pathlen + 1); + path = tmp; + } + + cookies_now = time (NULL); + + /* Now extract from the chains those cookies that match our host + (for domain_exact cookies), port (for cookies with port other + than PORT_ANY), etc. See matching_cookie for details. */ + + /* Count the number of matching cookies. */ + count = 0; + for (i = 0; i < (unsigned) chain_count; i++) + for (cookie = chains[i]; cookie; cookie = cookie->next) + if (cookie_matches_url (cookie, host, port, path, secflag, NULL)) + ++count; + if (!count) + goto out; /* no cookies matched */ + + /* Allocate the array. */ + if (count > SIZE_MAX / sizeof (struct weighed_cookie)) + goto out; /* unable to process so many cookies */ + outgoing = xmalloc (count * sizeof (struct weighed_cookie)); + + /* Fill the array with all the matching cookies from the chains that + match HOST. */ + ocnt = 0; + for (i = 0; i < (unsigned) chain_count; i++) + for (cookie = chains[i]; cookie; cookie = cookie->next) + { + int pg; + if (!cookie_matches_url (cookie, host, port, path, secflag, &pg)) + continue; + outgoing[ocnt].cookie = cookie; + outgoing[ocnt].domain_goodness = strlen (cookie->domain); + outgoing[ocnt].path_goodness = pg; + ++ocnt; + } + assert (ocnt == count); + + /* Eliminate duplicate cookies; that is, those whose name and value + are the same. */ + count = eliminate_dups (outgoing, count); + + /* Sort the array so that best-matching domains come first, and + that, within one domain, best-matching paths come first. */ + qsort (outgoing, count, sizeof (struct weighed_cookie), goodness_comparator); + + /* Count the space the name=value pairs will take. */ + result_size = 0; + for (i = 0; i < count; i++) + { + struct cookie *c = outgoing[i].cookie; + /* name=value */ + result_size += strlen (c->attr) + 1 + strlen (c->value); + } + + /* Allocate output buffer: + name=value pairs -- result_size + "; " separators -- (count - 1) * 2 + \0 terminator -- 1 */ + result_size = result_size + (count - 1) * 2 + 1; + result = xmalloc (result_size); + pos = 0; + for (i = 0; i < count; i++) + { + struct cookie *c = outgoing[i].cookie; + int namlen = strlen (c->attr); + int vallen = strlen (c->value); + + memcpy (result + pos, c->attr, namlen); + pos += namlen; + result[pos++] = '='; + memcpy (result + pos, c->value, vallen); + pos += vallen; + if (i < count - 1) + { + result[pos++] = ';'; + result[pos++] = ' '; + } + } + result[pos++] = '\0'; + xfree (outgoing); + assert (pos == result_size); + +out: + if (path != pathbuf) + xfree (path); + +return result; +} + +/* Support for loading and saving cookies. The format used for + loading and saving should be the format of the `cookies.txt' file + used by Netscape and Mozilla, at least the Unix versions. + (Apparently IE can export cookies in that format as well.) The + format goes like this: + + DOMAIN DOMAIN-FLAG PATH SECURE-FLAG TIMESTAMP ATTR-NAME ATTR-VALUE + + DOMAIN -- cookie domain, optionally followed by :PORT + DOMAIN-FLAG -- whether all hosts in the domain match + PATH -- cookie path + SECURE-FLAG -- whether cookie requires secure connection + TIMESTAMP -- expiry timestamp, number of seconds since epoch + ATTR-NAME -- name of the cookie attribute + ATTR-VALUE -- value of the cookie attribute (empty if absent) + + The fields are separated by TABs. All fields are mandatory, except + for ATTR-VALUE. The `-FLAG' fields are boolean, their legal values + being "TRUE" and "FALSE'. Empty lines, lines consisting of + whitespace only, and comment lines (beginning with # optionally + preceded by whitespace) are ignored. + + Example line from cookies.txt (split in two lines for readability): + + .google.com TRUE / FALSE 2147368447 \ + PREF ID=34bb47565bbcd47b:LD=en:NR=20:TM=985172580:LM=985739012 + +*/ + +/* If the region [B, E) ends with :<digits>, parse the number, return + it, and store new boundary (location of the `:') to DOMAIN_E_PTR. + If port is not specified, return 0. */ + +static int +domain_port (const char *domain_b, const char *domain_e, + const char **domain_e_ptr) +{ + int port = 0; + const char *p; + const char *colon = memchr (domain_b, ':', domain_e - domain_b); + if (!colon) + return 0; + for (p = colon + 1; p < domain_e && c_isdigit (*p); p++) + port = 10 * port + (*p - '0'); + if (p < domain_e) + /* Garbage following port number. */ + return 0; + *domain_e_ptr = colon; + return port; +} + +#define GET_WORD(p, b, e) do { \ + b = p; \ + while (*p && *p != '\t') \ + ++p; \ + e = p; \ + if (b == e || !*p) \ + goto next; \ + ++p; \ +} while (0) + +/* Load cookies from FILE. */ + +void +cookie_jar_load (struct cookie_jar *jar, const char *file) +{ + char *line = NULL; + size_t bufsize = 0; + + FILE *fp = fopen (file, "r"); + if (!fp) + { + logprintf (LOG_NOTQUIET, _("Cannot open cookies file %s: %s\n"), + quote (file), strerror (errno)); + return; + } + + cookies_now = time (NULL); + + while (getline (&line, &bufsize, fp) > 0) + { + struct cookie *cookie; + char *p = line; + + double expiry; + int port; + + char *domain_b = NULL, *domain_e = NULL; + char *domflag_b = NULL, *domflag_e = NULL; + char *path_b = NULL, *path_e = NULL; + char *secure_b = NULL, *secure_e = NULL; + char *expires_b = NULL, *expires_e = NULL; + char *name_b = NULL, *name_e = NULL; + char *value_b = NULL, *value_e = NULL; + + /* Skip leading white-space. */ + while (*p && c_isspace (*p)) + ++p; + /* Ignore empty lines. */ + if (!*p || *p == '#') + continue; + + GET_WORD (p, domain_b, domain_e); + GET_WORD (p, domflag_b, domflag_e); + GET_WORD (p, path_b, path_e); + GET_WORD (p, secure_b, secure_e); + GET_WORD (p, expires_b, expires_e); + GET_WORD (p, name_b, name_e); + + /* Don't use GET_WORD for value because it ends with newline, + not TAB. */ + value_b = p; + value_e = p + strlen (p); + if (value_e > value_b && value_e[-1] == '\n') + --value_e; + if (value_e > value_b && value_e[-1] == '\r') + --value_e; + /* Empty values are legal (I think), so don't bother checking. */ + + cookie = cookie_new (); + + cookie->attr = strdupdelim (name_b, name_e); + cookie->value = strdupdelim (value_b, value_e); + cookie->path = strdupdelim (path_b, path_e); + cookie->secure = BOUNDED_EQUAL (secure_b, secure_e, "TRUE"); + + /* Curl source says, quoting Andre Garcia: "flag: A TRUE/FALSE + value indicating if all machines within a given domain can + access the variable. This value is set automatically by the + browser, depending on the value set for the domain." */ + cookie->domain_exact = !BOUNDED_EQUAL (domflag_b, domflag_e, "TRUE"); + + /* DOMAIN needs special treatment because we might need to + extract the port. */ + port = domain_port (domain_b, domain_e, (const char **)&domain_e); + if (port) + cookie->port = port; + + if (*domain_b == '.') + ++domain_b; /* remove leading dot internally */ + cookie->domain = strdupdelim (domain_b, domain_e); + + /* safe default in case EXPIRES field is garbled. */ + expiry = (double)cookies_now - 1; + + /* I don't like changing the line, but it's safe here. (line is + malloced.) */ + *expires_e = '\0'; + sscanf (expires_b, "%lf", &expiry); + + if (expiry == 0) + { + /* EXPIRY can be 0 for session cookies saved because the + user specified `--keep-session-cookies' in the past. + They remain session cookies, and will be saved only if + the user has specified `keep-session-cookies' again. */ + } + else + { + if (expiry < cookies_now) + goto abort_cookie; /* ignore stale cookie. */ + cookie->expiry_time = (time_t) expiry; + cookie->permanent = 1; + } + + store_cookie (jar, cookie); + + next: + continue; + + abort_cookie: + delete_cookie (cookie); + } + + xfree(line); + fclose (fp); +} + +/* Save cookies, in format described above, to FILE. */ + +void +cookie_jar_save (struct cookie_jar *jar, const char *file) +{ + FILE *fp; + hash_table_iterator iter; + + DEBUGP (("Saving cookies to %s.\n", file)); + + cookies_now = time (NULL); + + fp = fopen (file, "w"); + if (!fp) + { + logprintf (LOG_NOTQUIET, _("Cannot open cookies file %s: %s\n"), + quote (file), strerror (errno)); + return; + } + + fputs ("# HTTP Cookie File\n", fp); + fprintf (fp, "# Generated by Wget on %s.\n", datetime_str (cookies_now)); + fputs ("# Edit at your own risk.\n\n", fp); + + for (hash_table_iterate (jar->chains, &iter); + hash_table_iter_next (&iter); + ) + { + const char *domain = iter.key; + struct cookie *cookie = iter.value; + for (; cookie; cookie = cookie->next) + { + if (!cookie->permanent && !opt.keep_session_cookies) + continue; + if (cookie_expired_p (cookie)) + continue; + if (!cookie->domain_exact) + fputc ('.', fp); + fputs (domain, fp); + if (cookie->port != PORT_ANY) + fprintf (fp, ":%d", cookie->port); + fprintf (fp, "\t%s\t%s\t%s\t%.0f\t%s\t%s\n", + cookie->domain_exact ? "FALSE" : "TRUE", + cookie->path, cookie->secure ? "TRUE" : "FALSE", + (double)cookie->expiry_time, + cookie->attr, cookie->value); + if (ferror (fp)) + goto out; + } + } + out: + if (ferror (fp)) + logprintf (LOG_NOTQUIET, _("Error writing to %s: %s\n"), + quote (file), strerror (errno)); + if (fclose (fp) < 0) + logprintf (LOG_NOTQUIET, _("Error closing %s: %s\n"), + quote (file), strerror (errno)); + + DEBUGP (("Done saving cookies.\n")); +} + +/* Clean up cookie-related data. */ + +void +cookie_jar_delete (struct cookie_jar *jar) +{ + /* Iterate over chains (indexed by domain) and free them. */ + hash_table_iterator iter; + for (hash_table_iterate (jar->chains, &iter); hash_table_iter_next (&iter); ) + { + struct cookie *chain = iter.value; + xfree (iter.key); + /* Then all cookies in this chain. */ + while (chain) + { + struct cookie *next = chain->next; + delete_cookie (chain); + chain = next; + } + } + hash_table_destroy (jar->chains); + xfree (jar); + +#ifdef HAVE_LIBPSL + psl_free (psl); + psl = NULL; +#endif +} + +/* Test cases. Currently this is only tests parse_set_cookies. To + use, recompile Wget with -DTEST_COOKIES and call test_cookies() + from main. */ + +#ifdef TEST_COOKIES +void +test_cookies (void) +{ + /* Tests expected to succeed: */ + static struct { + const char *data; + const char *results[10]; + } tests_succ[] = { + { "arg=value", {"arg", "value", NULL} }, + { "arg1=value1;arg2=value2", {"arg1", "value1", "arg2", "value2", NULL} }, + { "arg1=value1; arg2=value2", {"arg1", "value1", "arg2", "value2", NULL} }, + { "arg1=value1; arg2=value2;", {"arg1", "value1", "arg2", "value2", NULL} }, + { "arg1=value1; arg2=value2; ", {"arg1", "value1", "arg2", "value2", NULL} }, + { "arg1=\"value1\"; arg2=\"\"", {"arg1", "value1", "arg2", "", NULL} }, + { "arg=", {"arg", "", NULL} }, + { "arg1=; arg2=", {"arg1", "", "arg2", "", NULL} }, + { "arg1 = ; arg2= ", {"arg1", "", "arg2", "", NULL} }, + }; + + /* Tests expected to fail: */ + static char *tests_fail[] = { + ";", + "arg=\"unterminated", + "=empty-name", + "arg1=;=another-empty-name", + }; + int i; + + for (i = 0; i < countof (tests_succ); i++) + { + int ind; + const char *data = tests_succ[i].data; + const char **expected = tests_succ[i].results; + struct cookie *c; + + c = parse_set_cookie (data, true); + if (!c) + { + printf ("NULL cookie returned for valid data: %s\n", data); + continue; + } + + /* Test whether extract_param handles these cases correctly. */ + { + param_token name, value; + const char *ptr = data; + int j = 0; + while (extract_param (&ptr, &name, &value, ';', NULL)) + { + char *n = strdupdelim (name.b, name.e); + char *v = strdupdelim (value.b, value.e); + if (!expected[j]) + { + printf ("Too many parameters for '%s'\n", data); + break; + } + if (0 != strcmp (expected[j], n)) + printf ("Invalid name %d for '%s' (expected '%s', got '%s')\n", + j / 2 + 1, data, expected[j], n); + if (0 != strcmp (expected[j + 1], v)) + printf ("Invalid value %d for '%s' (expected '%s', got '%s')\n", + j / 2 + 1, data, expected[j + 1], v); + j += 2; + xfree (n); + xfree (v); + } + if (expected[j]) + printf ("Too few parameters for '%s'\n", data); + } + } + + for (i = 0; i < countof (tests_fail); i++) + { + struct cookie *c; + char *data = tests_fail[i]; + c = parse_set_cookie (data, true); + if (c) + printf ("Failed to report error on invalid data: %s\n", data); + } +} +#endif /* TEST_COOKIES */ diff --git a/src/cookies.h b/src/cookies.h new file mode 100644 index 0000000..903db32 --- /dev/null +++ b/src/cookies.h @@ -0,0 +1,47 @@ +/* Support for cookies. + Copyright (C) 2001-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef COOKIES_H +#define COOKIES_H + +struct cookie_jar; + +struct cookie_jar *cookie_jar_new (void); +void cookie_jar_delete (struct cookie_jar *); + +void cookie_handle_set_cookie (struct cookie_jar *, const char *, int, + const char *, const char *); +char *cookie_header (struct cookie_jar *, const char *, int, + const char *, bool); + +void cookie_jar_load (struct cookie_jar *, const char *); +void cookie_jar_save (struct cookie_jar *, const char *); + +#endif /* COOKIES_H */ diff --git a/src/css-tokens.h b/src/css-tokens.h new file mode 100644 index 0000000..40307ff --- /dev/null +++ b/src/css-tokens.h @@ -0,0 +1,65 @@ +/* Declarations for css.lex + Copyright (C) 2006, 2009-2011, 2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef CSS_TOKENS_H +#define CSS_TOKENS_H + +enum { + CSSEOF = 0, + S = 1, + CDO = 2, + CDC = 3, + INCLUDES = 4, + DASHMATCH = 5, + STRING = 6, + BAD_STRING = 7, + IDENT = 8, + HASH = 9, + IMPORT_SYM = 10, + PAGE_SYM = 11, + MEDIA_SYM = 12, + CHARSET_SYM = 13, + IMPORTANT_SYM = 14, + EMS = 15, + EXS = 16, + LENGTH = 17, + ANGLE = 18, + TIME = 19, + FREQ = 20, + DIMENSION = 21, + PERCENTAGE = 22, + NUMBER = 23, + URI = 24, + BAD_URI = 25, + FUNCTION = 26, + COMMENT = 27 +}; + +#endif /* CSS_TOKENS_H */ diff --git a/src/css-url.c b/src/css-url.c new file mode 100644 index 0000000..eacd926 --- /dev/null +++ b/src/css-url.c @@ -0,0 +1,236 @@ +/* Collect URLs from CSS source. + Copyright (C) 1998, 2000-2003, 2009-2011, 2014-2015, 2018-2020 Free + Software Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +/* + Note that this is not an actual CSS parser, but just a lexical + scanner with a tiny bit more smarts bolted on top. A full parser + is somewhat overkill for this job. The only things we're interested + in are @import rules and url() tokens, so it's easy enough to + grab those without truly understanding the input. The only downside + to this is that we might be coerced into downloading files that + a browser would ignore. That might merit some more investigation. + */ + +#include <wget.h> + +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <ctype.h> +#include <errno.h> + +#include "wget.h" +#include "utils.h" +#include "convert.h" +#include "html-url.h" +#include "css-tokens.h" +#include "css-url.h" +#include "xstrndup.h" + +/* from lex.yy.c */ +extern char *yytext; +extern int yyleng; +typedef struct yy_buffer_state *YY_BUFFER_STATE; +extern YY_BUFFER_STATE yy_scan_bytes (const char *bytes,int len ); +extern void yy_delete_buffer (YY_BUFFER_STATE b); +extern int yylex (void); +extern void yylex_destroy(void); + +/* + Given a detected URI token, get only the URI specified within. + Also adjust the starting position and length of the string. + + A URI can be specified with or without quotes, and the quotes + can be single or double quotes. In addition there can be + whitespace after the opening parenthesis and before the closing + parenthesis. +*/ +static char * +get_uri_string (const char *at, int *pos, int *length) +{ + if (*length < 4) + return NULL; + + if (0 != strncasecmp (at + *pos, "url(", 4)) + return NULL; + + *pos += 4; + *length -= 5; /* url() */ + + /* skip leading space */ + while (*length > 0 && isspace (at[*pos])) + { + (*pos)++; + if (--(*length) == 0) + return NULL; + } + + /* skip trailing space */ + while (*length > 0 && isspace (at[*pos + *length - 1])) + { + (*length)--; + } + + /* trim off quotes */ + if (*length >= 2 && (at[*pos] == '\'' || at[*pos] == '"')) + { + (*pos)++; + *length -= 2; + } + + if (*length <= 0) + return NULL; + + return xstrndup (at + *pos, *length); +} + +void +get_urls_css (struct map_context *ctx, int offset, int buf_length) +{ + int token; + /*char tmp[2048];*/ + int buffer_pos = 0; + int pos, length; + char *uri; + YY_BUFFER_STATE b; + + /* tell flex to scan from this buffer */ + b = yy_scan_bytes (ctx->text + offset, buf_length); + + while((token = yylex()) != CSSEOF) + { + /*DEBUGP (("%s ", token_names[token]));*/ + /* @import "foo.css" + or @import url(foo.css) + */ + if(token == IMPORT_SYM) + { + do { + buffer_pos += yyleng; + } while((token = yylex()) == S); + + /*DEBUGP (("%s ", token_names[token]));*/ + + if (token == STRING || token == URI) + { + /*DEBUGP (("Got URI "));*/ + pos = buffer_pos + offset; + length = yyleng; + + if (token == URI) + { + uri = get_uri_string (ctx->text, &pos, &length); + } + else if (length >= 2) + { + /* cut out quote characters */ + pos++; + length -= 2; + uri = xmalloc (length + 1); + memcpy (uri, yytext + 1, length); + uri[length] = '\0'; + } + else + uri = NULL; + + if (uri) + { + struct urlpos *up = append_url (uri, pos, length, ctx); + DEBUGP (("Found @import: [%s] at %d [%s]\n", yytext, buffer_pos, uri)); + + if (up) + { + up->link_inline_p = 1; + up->link_css_p = 1; + up->link_expect_css = 1; + } + + xfree(uri); + } + } + } + /* background-image: url(foo.png) + note that we don't care what + property this is actually on. + */ + else if(token == URI) + { + pos = buffer_pos + offset; + length = yyleng; + uri = get_uri_string (ctx->text, &pos, &length); + + if (uri) + { + struct urlpos *up = append_url (uri, pos, length, ctx); + DEBUGP (("Found URI: [%s] at %d [%s]\n", yytext, buffer_pos, uri)); + if (up) + { + up->link_inline_p = 1; + up->link_css_p = 1; + } + + xfree (uri); + } + } + buffer_pos += yyleng; + } + + yy_delete_buffer(b); + yylex_destroy(); + + DEBUGP (("\n")); +} + +struct urlpos * +get_urls_css_file (const char *file, const char *url) +{ + struct file_memory *fm; + struct map_context ctx; + + /* Load the file. */ + fm = wget_read_file (file); + if (!fm) + { + logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno)); + return NULL; + } + DEBUGP (("Loaded %s (size %s).\n", file, number_to_static_string (fm->length))); + + ctx.text = fm->content; + ctx.head = NULL; + ctx.base = NULL; + ctx.parent_base = url ? url : opt.base_href; + ctx.document_file = file; + ctx.nofollow = 0; + + get_urls_css (&ctx, 0, fm->length); + wget_read_file_free (fm); + return ctx.head; +} diff --git a/src/css-url.h b/src/css-url.h new file mode 100644 index 0000000..e8d7d1f --- /dev/null +++ b/src/css-url.h @@ -0,0 +1,37 @@ +/* Declarations for css-url.c. + Copyright (C) 2006, 2009-2011, 2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef CSS_URL_H +#define CSS_URL_H + +void get_urls_css (struct map_context *, int, int); +struct urlpos *get_urls_css_file (const char *, const char *); + +#endif /* CSS_URL_H */ diff --git a/src/css.c b/src/css.c new file mode 100644 index 0000000..7f51801 --- /dev/null +++ b/src/css.c @@ -0,0 +1,3932 @@ +#line 1 "css.c" +/* config.h must precede flex's inclusion of <stdio.h> + in order for its _GNU_SOURCE definition to take effect. */ +#include <config.h> + +#line 6 "css.c" + +#define YY_INT_ALIGNED short int + +/* A lexical scanner generated by flex */ + +#define FLEX_SCANNER +#define YY_FLEX_MAJOR_VERSION 2 +#define YY_FLEX_MINOR_VERSION 6 +#define YY_FLEX_SUBMINOR_VERSION 4 +#if YY_FLEX_SUBMINOR_VERSION > 0 +#define FLEX_BETA +#endif + +/* First, we deal with platform-specific or compiler-specific issues. */ + +/* begin standard C headers. */ +#include <stdio.h> +#include <string.h> +#include <errno.h> +#include <stdlib.h> + +/* end standard C headers. */ + +/* flex integer type definitions */ + +#ifndef FLEXINT_H +#define FLEXINT_H + +/* C99 systems have <inttypes.h>. Non-C99 systems may or may not. */ + +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + +/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h, + * if you want the limit (max/min) macros for int types. + */ +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS 1 +#endif + +#include <inttypes.h> +typedef int8_t flex_int8_t; +typedef uint8_t flex_uint8_t; +typedef int16_t flex_int16_t; +typedef uint16_t flex_uint16_t; +typedef int32_t flex_int32_t; +typedef uint32_t flex_uint32_t; +#else +typedef signed char flex_int8_t; +typedef short int flex_int16_t; +typedef int flex_int32_t; +typedef unsigned char flex_uint8_t; +typedef unsigned short int flex_uint16_t; +typedef unsigned int flex_uint32_t; + +/* Limits of integral types. */ +#ifndef INT8_MIN +#define INT8_MIN (-128) +#endif +#ifndef INT16_MIN +#define INT16_MIN (-32767-1) +#endif +#ifndef INT32_MIN +#define INT32_MIN (-2147483647-1) +#endif +#ifndef INT8_MAX +#define INT8_MAX (127) +#endif +#ifndef INT16_MAX +#define INT16_MAX (32767) +#endif +#ifndef INT32_MAX +#define INT32_MAX (2147483647) +#endif +#ifndef UINT8_MAX +#define UINT8_MAX (255U) +#endif +#ifndef UINT16_MAX +#define UINT16_MAX (65535U) +#endif +#ifndef UINT32_MAX +#define UINT32_MAX (4294967295U) +#endif + +#ifndef SIZE_MAX +#define SIZE_MAX (~(size_t)0) +#endif + +#endif /* ! C99 */ + +#endif /* ! FLEXINT_H */ + +/* begin standard C++ headers. */ + +/* TODO: this is always defined, so inline it */ +#define yyconst const + +#if defined(__GNUC__) && __GNUC__ >= 3 +#define yynoreturn __attribute__((__noreturn__)) +#else +#define yynoreturn +#endif + +/* Returned upon end-of-file. */ +#define YY_NULL 0 + +/* Promotes a possibly negative, possibly signed char to an + * integer in range [0..255] for use as an array index. + */ +#define YY_SC_TO_UI(c) ((YY_CHAR) (c)) + +/* Enter a start condition. This macro really ought to take a parameter, + * but we do it the disgusting crufty way forced on us by the ()-less + * definition of BEGIN. + */ +#define BEGIN (yy_start) = 1 + 2 * +/* Translate the current start state into a value that can be later handed + * to BEGIN to return to the state. The YYSTATE alias is for lex + * compatibility. + */ +#define YY_START (((yy_start) - 1) / 2) +#define YYSTATE YY_START +/* Action number for EOF rule of a given start state. */ +#define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) +/* Special action meaning "start processing a new file". */ +#define YY_NEW_FILE yyrestart( yyin ) +#define YY_END_OF_BUFFER_CHAR 0 + +/* Size of default input buffer. */ +#ifndef YY_BUF_SIZE +#ifdef __ia64__ +/* On IA-64, the buffer size is 16k, not 8k. + * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case. + * Ditto for the __ia64__ case accordingly. + */ +#define YY_BUF_SIZE 32768 +#else +#define YY_BUF_SIZE 16384 +#endif /* __ia64__ */ +#endif + +/* The state buf must be large enough to hold one state per character in the main buffer. + */ +#define YY_STATE_BUF_SIZE ((YY_BUF_SIZE + 2) * sizeof(yy_state_type)) + +#ifndef YY_TYPEDEF_YY_BUFFER_STATE +#define YY_TYPEDEF_YY_BUFFER_STATE +typedef struct yy_buffer_state *YY_BUFFER_STATE; +#endif + +#ifndef YY_TYPEDEF_YY_SIZE_T +#define YY_TYPEDEF_YY_SIZE_T +typedef size_t yy_size_t; +#endif + +extern int yyleng; + +extern FILE *yyin, *yyout; + +#define EOB_ACT_CONTINUE_SCAN 0 +#define EOB_ACT_END_OF_FILE 1 +#define EOB_ACT_LAST_MATCH 2 + + #define YY_LESS_LINENO(n) + #define YY_LINENO_REWIND_TO(ptr) + +/* Return all but the first "n" matched characters back to the input stream. */ +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + int yyless_macro_arg = (n); \ + YY_LESS_LINENO(yyless_macro_arg);\ + *yy_cp = (yy_hold_char); \ + YY_RESTORE_YY_MORE_OFFSET \ + (yy_c_buf_p) = yy_cp = yy_bp + yyless_macro_arg - YY_MORE_ADJ; \ + YY_DO_BEFORE_ACTION; /* set up yytext again */ \ + } \ + while ( 0 ) +#define unput(c) yyunput( c, (yytext_ptr) ) + +#ifndef YY_STRUCT_YY_BUFFER_STATE +#define YY_STRUCT_YY_BUFFER_STATE +struct yy_buffer_state + { + FILE *yy_input_file; + + char *yy_ch_buf; /* input buffer */ + char *yy_buf_pos; /* current position in input buffer */ + + /* Size of input buffer in bytes, not including room for EOB + * characters. + */ + int yy_buf_size; + + /* Number of characters read into yy_ch_buf, not including EOB + * characters. + */ + int yy_n_chars; + + /* Whether we "own" the buffer - i.e., we know we created it, + * and can realloc() it to grow it, and should free() it to + * delete it. + */ + int yy_is_our_buffer; + + /* Whether this is an "interactive" input source; if so, and + * if we're using stdio for input, then we want to use getc() + * instead of fread(), to make sure we stop fetching input after + * each newline. + */ + int yy_is_interactive; + + /* Whether we're considered to be at the beginning of a line. + * If so, '^' rules will be active on the next match, otherwise + * not. + */ + int yy_at_bol; + + int yy_bs_lineno; /**< The line count. */ + int yy_bs_column; /**< The column count. */ + + /* Whether to try to fill the input buffer when we reach the + * end of it. + */ + int yy_fill_buffer; + + int yy_buffer_status; + +#define YY_BUFFER_NEW 0 +#define YY_BUFFER_NORMAL 1 + /* When an EOF's been seen but there's still some text to process + * then we mark the buffer as YY_EOF_PENDING, to indicate that we + * shouldn't try reading from the input source any more. We might + * still have a bunch of tokens to match, though, because of + * possible backing-up. + * + * When we actually see the EOF, we change the status to "new" + * (via yyrestart()), so that the user can continue scanning by + * just pointing yyin at a new input file. + */ +#define YY_BUFFER_EOF_PENDING 2 + + }; +#endif /* !YY_STRUCT_YY_BUFFER_STATE */ + +/* Stack of input buffers. */ +static size_t yy_buffer_stack_top = 0; /**< index of top of stack. */ +static size_t yy_buffer_stack_max = 0; /**< capacity of stack. */ +static YY_BUFFER_STATE * yy_buffer_stack = NULL; /**< Stack as an array. */ + +/* We provide macros for accessing buffer states in case in the + * future we want to put the buffer states in a more general + * "scanner state". + * + * Returns the top of the stack, or NULL. + */ +#define YY_CURRENT_BUFFER ( (yy_buffer_stack) \ + ? (yy_buffer_stack)[(yy_buffer_stack_top)] \ + : NULL) +/* Same as previous macro, but useful when we know that the buffer stack is not + * NULL or when we need an lvalue. For internal use only. + */ +#define YY_CURRENT_BUFFER_LVALUE (yy_buffer_stack)[(yy_buffer_stack_top)] + +/* yy_hold_char holds the character lost when yytext is formed. */ +static char yy_hold_char; +static int yy_n_chars; /* number of characters read into yy_ch_buf */ +int yyleng; + +/* Points to current character in buffer. */ +static char *yy_c_buf_p = NULL; +static int yy_init = 0; /* whether we need to initialize */ +static int yy_start = 0; /* start state number */ + +/* Flag which is used to allow yywrap()'s to do buffer switches + * instead of setting up a fresh yyin. A bit of a hack ... + */ +static int yy_did_buffer_switch_on_eof; + +void yyrestart ( FILE *input_file ); +void yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer ); +YY_BUFFER_STATE yy_create_buffer ( FILE *file, int size ); +void yy_delete_buffer ( YY_BUFFER_STATE b ); +void yy_flush_buffer ( YY_BUFFER_STATE b ); +void yypush_buffer_state ( YY_BUFFER_STATE new_buffer ); +void yypop_buffer_state ( void ); + +static void yyensure_buffer_stack ( void ); +static void yy_load_buffer_state ( void ); +static void yy_init_buffer ( YY_BUFFER_STATE b, FILE *file ); +#define YY_FLUSH_BUFFER yy_flush_buffer( YY_CURRENT_BUFFER ) + +YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size ); +YY_BUFFER_STATE yy_scan_string ( const char *yy_str ); +YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, int len ); + +void *yyalloc ( yy_size_t ); +void *yyrealloc ( void *, yy_size_t ); +void yyfree ( void * ); + +#define yy_new_buffer yy_create_buffer +#define yy_set_interactive(is_interactive) \ + { \ + if ( ! YY_CURRENT_BUFFER ){ \ + yyensure_buffer_stack (); \ + YY_CURRENT_BUFFER_LVALUE = \ + yy_create_buffer( yyin, YY_BUF_SIZE ); \ + } \ + YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \ + } +#define yy_set_bol(at_bol) \ + { \ + if ( ! YY_CURRENT_BUFFER ){\ + yyensure_buffer_stack (); \ + YY_CURRENT_BUFFER_LVALUE = \ + yy_create_buffer( yyin, YY_BUF_SIZE ); \ + } \ + YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \ + } +#define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol) + +/* Begin user sect3 */ + +#define yywrap() (/*CONSTCOND*/1) +#define YY_SKIP_YYWRAP +typedef flex_uint8_t YY_CHAR; + +FILE *yyin = NULL, *yyout = NULL; + +typedef int yy_state_type; + +extern int yylineno; +int yylineno = 1; + +extern char *yytext; +#ifdef yytext_ptr +#undef yytext_ptr +#endif +#define yytext_ptr yytext + +static yy_state_type yy_get_previous_state ( void ); +static yy_state_type yy_try_NUL_trans ( yy_state_type current_state ); +static int yy_get_next_buffer ( void ); +static void yynoreturn yy_fatal_error ( const char* msg ); + +/* Done after the current pattern has been matched and before the + * corresponding action - sets up yytext. + */ +#define YY_DO_BEFORE_ACTION \ + (yytext_ptr) = yy_bp; \ + yyleng = (int) (yy_cp - yy_bp); \ + (yy_hold_char) = *yy_cp; \ + *yy_cp = '\0'; \ + (yy_c_buf_p) = yy_cp; +#define YY_NUM_RULES 41 +#define YY_END_OF_BUFFER 42 +/* This struct is not used in this scanner, + but its presence is necessary. */ +struct yy_trans_info + { + flex_int32_t yy_verify; + flex_int32_t yy_nxt; + }; +static const flex_int16_t yy_accept[1103] = + { 0, + 0, 0, 42, 40, 1, 1, 40, 10, 40, 10, + 40, 40, 40, 35, 40, 40, 11, 11, 40, 40, + 40, 1, 0, 0, 0, 0, 10, 9, 10, 12, + 0, 0, 10, 10, 0, 11, 0, 35, 4, 34, + 0, 0, 35, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 30, 0, 0, 0, 0, 0, + 0, 0, 39, 11, 0, 11, 11, 11, 8, 7, + 0, 0, 0, 0, 0, 0, 0, 10, 10, 10, + 0, 12, 12, 10, 10, 10, 6, 4, 4, 0, + 33, 0, 21, 0, 33, 0, 18, 19, 0, 33, + + 0, 31, 0, 23, 0, 33, 0, 22, 29, 0, + 25, 24, 20, 0, 33, 0, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 11, 11, 11, + 11, 11, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 10, 10, 10, 0, 0, 12, 12, 10, + 10, 10, 4, 2, 33, 33, 33, 33, 33, 21, + 26, 0, 33, 33, 33, 33, 33, 33, 33, 33, + 18, 19, 33, 0, 33, 33, 33, 33, 33, 33, + + 33, 31, 33, 33, 33, 23, 32, 0, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 22, 29, 33, + 33, 33, 33, 33, 24, 20, 27, 0, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 30, 33, + 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 11, 11, 38, 11, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 10, 0, 3, 12, 10, 4, 4, 33, + + 33, 33, 33, 33, 21, 21, 33, 33, 33, 26, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 18, + 19, 18, 28, 0, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 31, 31, 33, 33, 33, 23, + 23, 33, 33, 33, 32, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 22, 29, 22, 33, 33, 33, + 33, 33, 25, 24, 20, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 30, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 25, 33, 33, 33, 30, 30, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 11, 38, + 38, 38, 38, 37, 0, 11, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10, 0, 0, 12, 10, 33, 33, 33, 33, 21, + 21, 21, 21, 33, 33, 33, 26, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 18, 19, + + 18, 18, 18, 19, 19, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 31, 31, 31, 31, 33, 33, 33, 23, + 23, 23, 23, 33, 33, 33, 32, 32, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 22, + 29, 22, 22, 22, 29, 29, 33, 33, 33, 33, + 33, 25, 24, 20, 25, 25, 24, 24, 20, 20, + 33, 33, 33, 27, 33, 33, 33, 33, 33, 33, + 27, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 30, 33, 33, + + 33, 25, 33, 27, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 11, 38, 38, 38, + 38, 38, 38, 38, 38, 0, 38, 37, 38, 38, + 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 10, 12, 10, 33, 33, 33, + 33, 21, 21, 33, 33, 33, 26, 26, 26, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 18, 19, + 18, 33, 33, 33, 28, 33, 33, 33, 33, 33, + + 33, 28, 33, 33, 33, 33, 33, 28, 33, 33, + 33, 31, 31, 33, 33, 33, 23, 23, 33, 33, + 33, 32, 32, 32, 32, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 22, 29, 22, 33, 33, 33, + 33, 33, 25, 24, 20, 33, 33, 33, 27, 27, + 27, 33, 33, 33, 33, 27, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 33, 30, 33, 33, 33, 25, 33, 27, 0, 13, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 14, 0, 0, 0, 0, 11, 38, 36, + 38, 38, 38, 38, 38, 38, 38, 38, 38, 0, + 38, 38, 37, 38, 0, 11, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 10, 12, + 10, 33, 33, 33, 21, 21, 33, 33, 33, 26, + 33, 33, 33, 33, 33, 33, 33, 18, 19, 18, + 33, 33, 33, 28, 28, 28, 33, 33, 33, 33, + 33, 33, 33, 33, 28, 33, 33, 31, 31, 33, + 33, 23, 23, 33, 33, 33, 32, 32, 33, 33, + 33, 33, 33, 33, 33, 22, 29, 22, 33, 33, + + 33, 33, 25, 24, 20, 33, 33, 33, 27, 33, + 33, 33, 27, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 30, 33, 33, 33, 25, 33, + 27, 0, 0, 0, 0, 13, 0, 0, 0, 0, + 0, 0, 0, 15, 0, 0, 0, 0, 0, 14, + 14, 0, 11, 38, 38, 38, 38, 38, 38, 38, + 0, 0, 38, 38, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 33, 21, 21, 33, 33, + 26, 33, 18, 19, 18, 33, 33, 33, 28, 33, + 33, 33, 33, 33, 28, 31, 31, 23, 23, 33, + + 33, 32, 32, 33, 22, 29, 22, 25, 24, 20, + 33, 33, 27, 33, 27, 16, 0, 13, 0, 0, + 0, 0, 0, 15, 15, 0, 0, 38, 38, 38, + 0, 0, 0, 0, 38, 17, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 26, 33, 33, + 28, 33, 32, 32, 27, 0, 13, 13, 0, 0, + 38, 38, 38, 0, 0, 38, 0, 0, 0, 17, + 0, 0, 0, 0, 0, 0, 0, 28, 0, 38, + 38, 38, 0, 38, 0, 17, 0, 0, 0, 0, + 38, 38, 0, 38, 0, 17, 17, 0, 0, 0, + + 0, 0 + } ; + +static const YY_CHAR yy_ec[256] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, + 1, 4, 5, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 6, 7, 8, 9, 10, 11, 10, 12, 13, + 14, 15, 10, 10, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 10, 10, 29, + 30, 31, 10, 32, 33, 34, 35, 36, 37, 38, + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 42, 49, 50, 51, 52, 42, 42, 53, 42, 54, + 10, 55, 10, 10, 42, 10, 56, 57, 58, 59, + + 60, 61, 62, 63, 64, 42, 65, 66, 67, 68, + 69, 70, 42, 71, 72, 73, 74, 42, 42, 75, + 42, 76, 10, 77, 10, 78, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79 + } ; + +static const YY_CHAR yy_meta[80] = + { 0, + 1, 2, 3, 3, 3, 2, 2, 4, 2, 2, + 2, 4, 5, 2, 2, 6, 2, 7, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 2, 2, + 2, 2, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 2, 2, 9 + } ; + +static const flex_int16_t yy_base[1137] = + { 0, + 0, 0, 4195, 7110, 78, 83, 88, 87, 78, 85, + 82, 88, 4161, 142, 4151, 90, 86, 206, 259, 4118, + 4083, 98, 234, 4083, 72, 109, 116, 7110, 318, 100, + 4082, 361, 208, 420, 4017, 92, 463, 205, 4024, 7110, + 3977, 222, 0, 3974, 209, 89, 257, 202, 180, 245, + 248, 259, 355, 272, 3955, 524, 3987, 83, 280, 311, + 274, 585, 7110, 117, 637, 348, 210, 697, 7110, 7110, + 3963, 302, 382, 243, 3940, 3933, 371, 251, 356, 757, + 3945, 101, 817, 246, 357, 877, 7110, 3938, 252, 920, + 3894, 974, 3886, 1017, 397, 447, 3861, 3860, 661, 376, + + 904, 3850, 721, 3841, 448, 451, 1040, 3840, 3825, 1045, + 3813, 3796, 3787, 1063, 569, 600, 3784, 1137, 484, 1180, + 613, 1221, 748, 380, 566, 347, 591, 907, 654, 462, + 3767, 3789, 118, 559, 779, 633, 593, 588, 628, 782, + 654, 3776, 775, 3775, 802, 772, 682, 336, 1279, 345, + 447, 1322, 3754, 254, 695, 837, 658, 581, 632, 656, + 806, 1012, 456, 862, 1365, 3744, 256, 903, 1408, 278, + 964, 1451, 3730, 7110, 3689, 1511, 839, 751, 790, 3655, + 3633, 1086, 865, 3648, 3645, 868, 812, 3627, 850, 3599, + 3559, 3554, 895, 897, 344, 3587, 3580, 885, 900, 892, + + 922, 3526, 1001, 929, 943, 3484, 3483, 1165, 1002, 3500, + 3499, 1037, 1074, 945, 3485, 969, 3476, 3439, 3424, 1096, + 3450, 410, 3442, 438, 3403, 3397, 3396, 1109, 1104, 3416, + 3396, 745, 1554, 1126, 1597, 1199, 1638, 1237, 1696, 1591, + 1451, 1224, 1636, 1495, 1673, 1680, 1762, 1836, 1318, 1740, + 1703, 7110, 573, 1049, 1161, 1158, 970, 987, 1131, 1017, + 1192, 1245, 3385, 3371, 1046, 1269, 1252, 3369, 3358, 1342, + 1600, 1403, 1552, 1788, 1263, 1891, 1796, 1934, 3362, 1868, + 1238, 1307, 1248, 3350, 3304, 1055, 1352, 1779, 1931, 1391, + 1392, 1393, 1983, 3284, 7110, 2026, 2069, 3253, 347, 892, + + 2112, 1392, 1108, 1110, 1248, 1488, 1500, 3239, 3204, 3174, + 1541, 3194, 3177, 2112, 1624, 1119, 3145, 1176, 3142, 1861, + 1873, 1878, 3104, 1704, 1607, 3071, 2999, 1394, 2951, 2939, + 1979, 1503, 1180, 1209, 1896, 1901, 1669, 1259, 1273, 1936, + 1971, 1707, 1278, 1296, 2904, 1691, 2927, 2900, 2022, 1731, + 1336, 2867, 1403, 2856, 1994, 2027, 2063, 1753, 2851, 716, + 2848, 950, 2068, 2106, 2117, 1786, 2836, 2831, 1793, 2833, + 2795, 2150, 2191, 1807, 2234, 2138, 2275, 2156, 2333, 2228, + 2237, 2273, 2344, 2358, 2367, 2378, 2449, 2523, 2189, 2433, + 2413, 1498, 2121, 2020, 2269, 2451, 1769, 2278, 1797, 2271, + + 1426, 1684, 1625, 2322, 1465, 2311, 2342, 2464, 2148, 2457, + 2261, 1528, 2388, 2423, 2759, 1179, 1347, 2149, 2296, 2245, + 2683, 2676, 1875, 1831, 2535, 2548, 1910, 2480, 2177, 2645, + 2637, 2383, 2555, 7110, 2446, 2468, 2600, 2595, 2508, 2546, + 2570, 2561, 2456, 2552, 2533, 2540, 1981, 2561, 2614, 2651, + 2674, 741, 457, 7110, 2729, 2807, 2597, 661, 2134, 2596, + 2579, 1548, 1591, 2330, 2405, 2814, 2604, 1589, 2641, 97, + 2882, 2584, 771, 2942, 3003, 3046, 2605, 1637, 1675, 2688, + 2695, 2537, 1235, 2639, 2556, 2555, 2710, 2662, 2543, 2538, + 2819, 1925, 2402, 2684, 1849, 2508, 1979, 2507, 2715, 2720, + + 2844, 2463, 1448, 2462, 1598, 2832, 2475, 2463, 2837, 2420, + 2412, 2894, 2582, 2374, 2365, 3042, 2023, 2628, 2816, 2853, + 2025, 2109, 2889, 2919, 2312, 1614, 2857, 2229, 2270, 2929, + 3047, 2262, 1615, 2931, 2423, 2439, 3052, 3083, 3071, 2289, + 2269, 3106, 2300, 2407, 3094, 2461, 2233, 2530, 2232, 3119, + 3124, 3129, 2191, 1714, 2183, 1734, 3117, 2186, 1033, 2175, + 1110, 3142, 3147, 3152, 2135, 1803, 2132, 2181, 2103, 2338, + 3076, 2090, 2077, 3160, 3148, 2035, 2003, 3172, 3132, 3134, + 0, 3213, 1864, 3230, 3173, 3254, 3181, 3312, 3378, 3437, + 3511, 3581, 3656, 3723, 3785, 3859, 3933, 3219, 3988, 4050, + + 3236, 2813, 3269, 2888, 2606, 2901, 3277, 3160, 2608, 2666, + 3251, 3162, 3338, 3190, 2881, 3264, 7110, 3191, 3285, 1994, + 1985, 2987, 3302, 3350, 3326, 3303, 3358, 3340, 1965, 1964, + 3341, 3369, 3367, 3038, 3373, 3009, 4107, 3394, 1039, 3416, + 4166, 696, 4225, 3442, 3463, 3481, 3497, 3517, 4285, 4346, + 4407, 3059, 3408, 3193, 1957, 1942, 3295, 3430, 3539, 3544, + 3283, 2258, 3485, 311, 4467, 4510, 4553, 4596, 590, 2669, + 3080, 3327, 3553, 3451, 1920, 1919, 3559, 1876, 2372, 805, + 1897, 1896, 3572, 3260, 3261, 1801, 3311, 1794, 3565, 3614, + 3619, 3505, 1764, 1751, 3626, 3572, 1712, 1704, 3635, 3537, + + 3597, 0, 842, 1683, 1668, 3640, 3543, 0, 962, 3354, + 3355, 3645, 3661, 1143, 3420, 3436, 3672, 3681, 3673, 3477, + 3547, 3701, 3710, 1613, 2478, 1302, 1608, 1607, 3716, 3707, + 3553, 1605, 3554, 1537, 3732, 3737, 3748, 3732, 1529, 1796, + 1502, 1830, 3762, 3770, 3777, 3774, 1498, 1459, 3798, 1394, + 3135, 1339, 1384, 1374, 3811, 0, 4639, 3784, 1740, 3822, + 2122, 3822, 3869, 3895, 3916, 3941, 3970, 3978, 3995, 4699, + 4002, 3880, 4083, 4105, 3722, 0, 3634, 0, 1410, 7110, + 3904, 3810, 1349, 1342, 3782, 3868, 4066, 4635, 3883, 3654, + 3856, 1427, 3908, 1333, 1308, 3944, 4176, 3996, 3768, 3982, + + 1575, 4001, 4071, 4011, 3853, 4024, 1763, 4756, 4088, 7110, + 1487, 3423, 4816, 896, 3208, 4876, 4262, 4444, 4919, 4504, + 4547, 4591, 4677, 4979, 5040, 5101, 3896, 4040, 4023, 1274, + 1266, 3798, 4080, 4509, 4041, 4065, 4639, 1821, 4709, 5144, + 5187, 5230, 3632, 3726, 4208, 4213, 2076, 1260, 1207, 4231, + 1176, 1123, 4793, 3760, 1108, 3762, 1099, 4236, 4322, 4327, + 4093, 1043, 1038, 4332, 994, 3434, 2212, 1028, 1002, 4913, + 992, 951, 4552, 3833, 0, 3946, 3953, 4474, 4479, 4041, + 4059, 4645, 4716, 2284, 4091, 4106, 4721, 4733, 929, 840, + 4919, 4115, 838, 4116, 837, 4738, 4743, 4756, 811, 2240, + + 777, 3157, 4763, 4798, 4803, 2371, 745, 718, 4823, 658, + 646, 5138, 0, 4956, 5225, 5181, 5269, 5279, 5291, 5296, + 5303, 5315, 5386, 5143, 5350, 5369, 5375, 5394, 5408, 5448, + 5413, 642, 4130, 618, 583, 7110, 4253, 5467, 4134, 4243, + 5276, 2587, 4250, 4854, 4277, 4128, 4675, 2903, 4432, 7110, + 526, 2948, 4859, 1544, 4963, 5522, 1357, 3705, 5565, 5608, + 5472, 5669, 5477, 5730, 4466, 5171, 4581, 508, 493, 4435, + 5496, 4865, 4634, 5275, 3053, 5506, 5511, 5565, 478, 449, + 4864, 5767, 5602, 5607, 5772, 3203, 451, 445, 4924, 443, + 442, 5777, 5790, 5795, 5805, 5812, 5835, 5853, 5859, 4140, + + 4164, 4984, 5017, 5866, 5871, 5876, 5881, 5889, 5894, 5899, + 411, 381, 5022, 5907, 5913, 7110, 4866, 5027, 5214, 4678, + 5521, 3400, 4944, 7110, 370, 3468, 3483, 5950, 5993, 6036, + 5987, 6030, 6096, 0, 6139, 7110, 5502, 5131, 4161, 4218, + 4832, 5304, 6035, 5313, 4441, 5307, 3527, 5992, 344, 285, + 5490, 6133, 6073, 6138, 6176, 5382, 7110, 296, 3629, 3675, + 6213, 6256, 6299, 6199, 6342, 6385, 5403, 237, 230, 7110, + 5415, 6212, 6255, 5439, 3976, 5380, 3852, 6293, 3928, 6428, + 6471, 6514, 6557, 6600, 5441, 5531, 5500, 5100, 5886, 4057, + 6643, 6686, 6729, 5612, 5628, 7110, 133, 4058, 6772, 4181, + + 6336, 7110, 6833, 6837, 6846, 6850, 6855, 6864, 6873, 6882, + 6891, 6900, 112, 6904, 6913, 6922, 6931, 6940, 6949, 6958, + 6967, 6976, 6984, 6993, 7002, 7011, 7020, 7029, 7038, 7047, + 7056, 7065, 7074, 7083, 7092, 7100 + } ; + +static const flex_int16_t yy_def[1137] = + { 0, + 1102, 1, 1102, 1102, 1102, 1102, 1102, 1103, 1104, 1105, + 1106, 1102, 1102, 1102, 1102, 1102, 1107, 1107, 1108, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1103, 1102, 1109, 1104, + 1102, 1110, 1105, 1111, 1102, 1107, 1108, 14, 1112, 1102, + 1113, 1102, 14, 1114, 1114, 1114, 1114, 1114, 1114, 1114, + 1114, 1114, 1114, 1114, 1114, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1107, 1115, 1107, 1107, 1107, 1102, 1102, + 1116, 1102, 1102, 1102, 1102, 1102, 1102, 1103, 1103, 1103, + 1117, 1104, 1104, 1105, 1105, 1105, 1102, 1112, 1118, 56, + 1114, 1119, 1114, 1119, 1114, 94, 1114, 1114, 94, 1114, + + 94, 1114, 94, 1114, 94, 1114, 94, 1114, 1114, 94, + 1114, 1114, 1114, 94, 1114, 94, 1114, 1114, 118, 118, + 118, 118, 118, 1114, 1114, 1114, 1114, 1114, 1114, 1114, + 1114, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1107, 68, 1107, + 1107, 68, 1116, 1120, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1103, 1103, 80, 1117, 1121, 1104, 83, 1105, + 1105, 86, 1122, 1102, 1114, 118, 176, 176, 176, 1114, + 1114, 94, 176, 176, 176, 176, 176, 176, 176, 176, + 1114, 1114, 1114, 94, 176, 176, 176, 1114, 176, 176, + + 176, 1114, 176, 176, 176, 1114, 1114, 94, 176, 176, + 176, 1114, 176, 176, 176, 176, 176, 1114, 1114, 176, + 176, 176, 176, 176, 1114, 1114, 1114, 94, 176, 176, + 176, 1114, 118, 233, 233, 233, 233, 233, 233, 239, + 239, 239, 239, 239, 239, 239, 239, 233, 248, 248, + 239, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1107, 68, 1123, 68, 1124, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 80, 1125, 1102, 83, 86, 1122, 1126, 1114, + + 176, 301, 301, 301, 301, 301, 176, 176, 176, 1114, + 301, 301, 301, 301, 301, 301, 301, 301, 301, 301, + 301, 301, 1114, 94, 176, 176, 176, 301, 301, 301, + 301, 301, 301, 301, 301, 301, 301, 301, 301, 301, + 301, 176, 176, 176, 1114, 301, 301, 301, 301, 301, + 301, 301, 301, 301, 301, 301, 301, 301, 301, 301, + 301, 301, 301, 301, 301, 176, 176, 176, 301, 301, + 301, 301, 233, 373, 373, 373, 373, 373, 373, 379, + 379, 379, 379, 379, 379, 379, 379, 373, 388, 388, + 379, 1114, 1114, 1114, 1114, 373, 1114, 1114, 1114, 1114, + + 1114, 1114, 1114, 1114, 1114, 1114, 1114, 1114, 1114, 1114, + 373, 1114, 1114, 373, 1114, 1114, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 276, 1123, + 1123, 1127, 1128, 1102, 1102, 276, 1129, 1130, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1131, 1132, 1133, 1102, 86, 301, 476, 476, 476, 476, + 476, 1114, 1114, 301, 301, 301, 301, 476, 476, 476, + 476, 1114, 1114, 476, 476, 476, 476, 476, 476, 476, + + 476, 1114, 1114, 1114, 1114, 176, 176, 176, 301, 301, + 301, 301, 476, 476, 476, 476, 1114, 1114, 476, 476, + 476, 476, 476, 476, 1114, 1114, 476, 476, 476, 476, + 476, 1114, 1114, 301, 301, 301, 301, 301, 476, 476, + 476, 476, 1114, 1114, 476, 476, 476, 476, 476, 476, + 476, 476, 1114, 1114, 1114, 1114, 476, 476, 476, 476, + 476, 476, 476, 476, 1114, 1114, 1114, 1114, 1114, 1114, + 301, 301, 301, 301, 476, 476, 476, 476, 1114, 1114, + 476, 373, 582, 582, 582, 582, 582, 582, 582, 582, + 582, 582, 582, 582, 590, 590, 582, 582, 582, 590, + + 582, 582, 582, 582, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 276, 1102, 1127, 1102, + 1134, 1128, 1135, 1123, 1123, 1102, 1123, 1123, 1123, 1102, + 456, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1131, 474, 475, 476, 668, 668, + 668, 668, 668, 476, 476, 476, 476, 1114, 1114, 668, + 668, 668, 668, 668, 668, 668, 668, 668, 668, 668, + 668, 301, 301, 301, 301, 476, 476, 476, 476, 1114, + + 1114, 476, 668, 668, 668, 668, 668, 668, 668, 668, + 668, 668, 668, 668, 668, 668, 668, 668, 476, 476, + 476, 476, 476, 1114, 1114, 668, 668, 668, 668, 668, + 668, 668, 668, 668, 668, 668, 668, 668, 668, 668, + 668, 668, 668, 668, 668, 476, 476, 476, 476, 1114, + 1114, 668, 668, 668, 668, 668, 582, 757, 757, 757, + 757, 757, 757, 757, 757, 757, 757, 757, 757, 757, + 757, 757, 757, 757, 757, 757, 757, 757, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 276, 1102, 1102, + 1127, 1127, 1127, 1128, 1128, 1128, 1123, 1123, 649, 1136, + 1136, 1123, 1136, 649, 1102, 651, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1131, 474, + 475, 668, 842, 842, 842, 842, 668, 668, 668, 668, + 842, 842, 842, 842, 842, 842, 842, 842, 842, 842, + 476, 476, 476, 476, 1114, 1114, 668, 668, 668, 668, + 842, 842, 842, 842, 842, 842, 842, 842, 842, 842, + 842, 842, 842, 668, 668, 668, 668, 668, 842, 842, + 842, 842, 842, 842, 842, 842, 842, 842, 842, 842, + + 842, 842, 842, 842, 842, 668, 668, 668, 668, 842, + 842, 842, 842, 1114, 1114, 1114, 1114, 1114, 1114, 1114, + 1114, 1114, 1114, 1114, 1114, 1114, 1114, 1114, 1114, 1114, + 1114, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 826, 1127, 1127, 813, 1128, 1128, 816, 649, + 1136, 1102, 1136, 824, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1114, 1114, 1114, 842, 842, + 842, 1114, 1114, 1114, 1114, 668, 668, 668, 668, 842, + 842, 842, 1114, 1114, 1114, 1114, 1114, 1114, 1114, 842, + + 842, 842, 842, 1114, 1114, 1114, 1114, 1114, 1114, 1114, + 842, 842, 842, 1114, 1114, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 813, 816, 649, + 1136, 1136, 1136, 962, 824, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1114, 842, 842, + 842, 1114, 1114, 1114, 1114, 1102, 1102, 1102, 1102, 1102, + 813, 816, 649, 1136, 1033, 824, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1114, 1102, 813, + 816, 649, 1033, 824, 1102, 1102, 1102, 1102, 1102, 1102, + 813, 816, 1033, 1082, 1102, 1102, 1102, 1102, 1033, 1102, + + 1136, 0, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102 + } ; + +static const flex_int16_t yy_nxt[7190] = + { 0, + 4, 5, 6, 5, 5, 5, 7, 8, 9, 4, + 4, 10, 4, 4, 4, 11, 12, 13, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 15, 4, + 4, 16, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 18, 17, 17, 19, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 18, 17, 17, 20, 21, 17, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 23, + 23, 23, 23, 23, 28, 31, 28, 35, 63, 22, + + 22, 22, 22, 22, 63, 24, 38, 38, 38, 38, + 38, 38, 38, 38, 38, 38, 72, 1102, 1102, 75, + 44, 76, 133, 28, 58, 95, 73, 74, 25, 63, + 59, 75, 32, 76, 60, 1096, 37, 61, 72, 34, + 65, 29, 26, 96, 62, 133, 65, 58, 95, 77, + 253, 25, 40, 59, 32, 32, 60, 41, 42, 61, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 29, 65, 77, 253, 44, 44, 45, 46, 47, 44, + 48, 49, 50, 44, 51, 44, 52, 44, 44, 53, + 54, 55, 44, 44, 44, 44, 56, 44, 44, 45, + + 46, 47, 44, 48, 49, 50, 51, 44, 52, 44, + 44, 53, 54, 55, 44, 44, 44, 44, 63, 28, + 44, 1102, 63, 38, 38, 38, 38, 38, 38, 38, + 38, 38, 38, 102, 103, 23, 23, 23, 23, 23, + 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, + 100, 24, 1086, 93, 66, 102, 101, 28, 28, 1086, + 65, 161, 34, 94, 65, 75, 89, 76, 154, 174, + 167, 280, 100, 295, 25, 93, 66, 68, 68, 68, + 68, 68, 68, 68, 68, 68, 68, 106, 26, 28, + 104, 68, 68, 68, 68, 68, 68, 25, 1057, 105, + + 34, 97, 107, 108, 115, 29, 138, 1078, 109, 98, + 106, 99, 104, 110, 68, 68, 68, 68, 68, 68, + 27, 27, 79, 97, 134, 108, 116, 115, 139, 138, + 109, 98, 34, 158, 135, 159, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 134, 136, 63, 155, + 80, 80, 80, 80, 80, 80, 156, 277, 27, 33, + 63, 299, 328, 28, 174, 137, 1078, 329, 28, 330, + 136, 155, 1024, 80, 80, 80, 80, 80, 80, 83, + 83, 83, 83, 83, 83, 83, 83, 83, 83, 111, + 65, 150, 104, 83, 83, 83, 83, 83, 83, 65, + + 157, 105, 65, 1055, 158, 112, 159, 113, 193, 114, + 29, 34, 111, 150, 104, 72, 83, 83, 83, 83, + 83, 83, 33, 33, 85, 73, 160, 112, 100, 113, + 194, 193, 364, 1055, 101, 181, 365, 72, 86, 86, + 86, 86, 86, 86, 86, 86, 86, 86, 160, 67, + 100, 182, 86, 86, 86, 86, 86, 86, 181, 63, + 364, 1052, 1052, 28, 365, 183, 203, 1051, 640, 184, + 204, 185, 205, 1051, 1048, 86, 86, 86, 86, 86, + 86, 68, 68, 68, 68, 68, 68, 68, 68, 68, + 68, 175, 175, 206, 115, 68, 68, 68, 68, 68, + + 68, 65, 234, 1048, 207, 208, 234, 234, 234, 234, + 29, 643, 1043, 175, 175, 206, 116, 115, 68, 68, + 68, 68, 68, 68, 117, 117, 207, 1043, 950, 117, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 118, 119, 119, 119, 120, 121, 122, 123, + 119, 119, 117, 117, 117, 117, 119, 119, 119, 119, + 119, 119, 124, 125, 126, 117, 127, 117, 128, 117, + 117, 129, 130, 131, 117, 117, 117, 117, 117, 119, + 119, 119, 119, 119, 119, 124, 125, 126, 127, 117, + 128, 117, 117, 129, 130, 131, 117, 117, 117, 117, + + 117, 117, 117, 140, 227, 1018, 254, 141, 142, 143, + 144, 262, 843, 255, 844, 263, 288, 264, 229, 102, + 103, 417, 230, 228, 231, 145, 265, 227, 254, 146, + 106, 247, 147, 248, 249, 234, 234, 234, 234, 288, + 1018, 102, 266, 417, 175, 107, 267, 1016, 145, 265, + 268, 146, 269, 106, 147, 149, 149, 149, 149, 149, + 149, 149, 149, 149, 149, 1014, 175, 289, 260, 149, + 149, 149, 149, 149, 149, 458, 287, 1014, 280, 186, + 158, 271, 159, 187, 188, 189, 190, 261, 111, 272, + 289, 260, 149, 149, 149, 149, 149, 149, 67, 67, + + 67, 151, 67, 155, 112, 191, 113, 640, 114, 63, + 156, 111, 272, 192, 138, 152, 152, 152, 152, 152, + 152, 152, 152, 152, 152, 155, 112, 191, 113, 152, + 152, 152, 152, 152, 152, 192, 139, 138, 563, 199, + 1013, 281, 564, 176, 200, 176, 201, 117, 640, 282, + 643, 65, 152, 152, 152, 152, 152, 152, 163, 78, + 78, 164, 163, 281, 28, 175, 247, 1013, 248, 249, + 234, 234, 234, 234, 202, 165, 165, 165, 165, 165, + 165, 165, 165, 165, 165, 473, 305, 175, 295, 165, + 165, 165, 165, 165, 165, 641, 202, 256, 1008, 92, + + 270, 257, 271, 258, 141, 142, 143, 144, 136, 305, + 274, 29, 165, 165, 165, 165, 165, 165, 82, 82, + 82, 168, 82, 259, 290, 306, 137, 851, 75, 852, + 76, 136, 1008, 274, 1102, 169, 169, 169, 169, 169, + 169, 169, 169, 169, 169, 259, 134, 320, 306, 169, + 169, 169, 169, 169, 169, 283, 135, 302, 1006, 1006, + 284, 303, 285, 304, 78, 871, 1004, 872, 134, 28, + 320, 32, 169, 169, 169, 169, 169, 169, 170, 84, + 84, 171, 170, 311, 286, 322, 315, 312, 28, 313, + 316, 317, 318, 319, 175, 172, 172, 172, 172, 172, + + 172, 172, 172, 172, 172, 82, 286, 640, 322, 172, + 172, 172, 172, 172, 172, 325, 29, 193, 332, 326, + 1102, 327, 195, 333, 335, 334, 176, 196, 176, 197, + 323, 34, 172, 172, 172, 172, 172, 172, 119, 194, + 193, 175, 119, 119, 119, 119, 92, 335, 175, 324, + 643, 108, 198, 323, 336, 1004, 109, 32, 117, 117, + 117, 110, 117, 175, 117, 340, 84, 117, 117, 117, + 175, 993, 563, 108, 198, 28, 564, 336, 109, 341, + 355, 117, 117, 117, 117, 876, 117, 877, 340, 117, + 117, 117, 176, 176, 176, 176, 176, 176, 176, 176, + + 176, 176, 341, 355, 357, 425, 176, 176, 176, 176, + 176, 176, 993, 291, 291, 291, 292, 291, 34, 337, + 346, 992, 426, 338, 347, 339, 348, 357, 425, 176, + 176, 176, 176, 176, 176, 177, 176, 176, 176, 178, + 176, 179, 176, 176, 176, 426, 640, 992, 92, 176, + 176, 176, 176, 176, 176, 744, 72, 427, 209, 745, + 989, 180, 210, 213, 211, 989, 73, 214, 215, 216, + 217, 428, 176, 176, 176, 176, 176, 176, 72, 212, + 427, 220, 434, 180, 175, 221, 222, 223, 224, 218, + 207, 208, 350, 641, 219, 418, 351, 352, 353, 354, + + 435, 281, 212, 419, 307, 434, 175, 175, 308, 282, + 309, 218, 207, 225, 358, 226, 219, 418, 359, 360, + 361, 362, 369, 281, 310, 984, 370, 366, 371, 175, + 175, 367, 744, 368, 984, 225, 745, 226, 117, 117, + 117, 232, 117, 480, 374, 481, 982, 310, 374, 374, + 374, 374, 175, 175, 499, 233, 234, 234, 234, 235, + 236, 237, 238, 234, 234, 880, 480, 881, 481, 234, + 234, 234, 234, 234, 234, 175, 424, 499, 254, 420, + 257, 415, 258, 342, 421, 255, 422, 176, 343, 176, + 344, 92, 234, 234, 234, 234, 234, 234, 234, 982, + + 254, 239, 240, 241, 234, 242, 243, 244, 423, 175, + 429, 501, 523, 245, 430, 246, 431, 387, 345, 388, + 389, 374, 374, 374, 374, 399, 399, 399, 400, 399, + 423, 175, 981, 92, 501, 523, 245, 482, 246, 234, + 345, 524, 239, 240, 241, 234, 242, 243, 244, 482, + 482, 482, 483, 482, 250, 387, 251, 388, 389, 374, + 374, 374, 374, 432, 524, 148, 465, 263, 91, 264, + 440, 284, 100, 285, 268, 63, 269, 250, 101, 251, + 148, 148, 148, 275, 148, 981, 459, 436, 971, 92, + 91, 437, 460, 438, 100, 530, 971, 276, 276, 276, + + 276, 276, 276, 276, 276, 276, 276, 439, 459, 531, + 537, 276, 276, 276, 276, 276, 276, 65, 530, 415, + 415, 415, 416, 415, 889, 461, 890, 944, 538, 462, + 439, 463, 531, 537, 276, 276, 276, 276, 276, 276, + 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, + 374, 538, 944, 464, 278, 278, 278, 278, 278, 278, + 442, 910, 938, 911, 141, 142, 143, 144, 640, 938, + 467, 550, 92, 374, 158, 464, 159, 278, 278, 278, + 278, 278, 278, 293, 293, 293, 293, 293, 293, 293, + 293, 293, 293, 912, 550, 291, 605, 293, 293, 293, + + 293, 293, 293, 912, 445, 445, 445, 446, 445, 470, + 477, 643, 513, 75, 478, 76, 479, 514, 605, 515, + 293, 293, 293, 293, 293, 293, 296, 296, 296, 296, + 296, 296, 296, 296, 296, 296, 72, 72, 552, 136, + 296, 296, 296, 296, 296, 296, 73, 73, 92, 257, + 502, 258, 397, 397, 397, 398, 397, 137, 72, 72, + 932, 552, 136, 296, 296, 296, 296, 296, 296, 297, + 297, 297, 297, 297, 297, 297, 297, 297, 297, 102, + 103, 909, 932, 297, 297, 297, 297, 297, 297, 482, + 482, 482, 483, 482, 640, 97, 403, 403, 403, 404, + + 403, 102, 92, 98, 106, 99, 297, 297, 297, 297, + 297, 297, 175, 175, 175, 300, 175, 97, 484, 107, + 909, 520, 485, 903, 486, 98, 521, 106, 522, 301, + 301, 301, 301, 301, 301, 301, 301, 301, 301, 91, + 104, 641, 93, 301, 301, 301, 301, 301, 301, 105, + 903, 640, 94, 447, 447, 447, 448, 447, 897, 488, + 115, 91, 104, 489, 93, 490, 301, 301, 301, 301, + 301, 301, 373, 374, 374, 374, 375, 376, 377, 378, + 374, 374, 116, 115, 138, 659, 374, 374, 374, 374, + 374, 374, 394, 394, 394, 395, 394, 263, 641, 264, + + 504, 443, 443, 443, 444, 443, 139, 138, 659, 374, + 374, 374, 374, 374, 374, 374, 525, 532, 379, 380, + 381, 374, 382, 383, 384, 509, 897, 396, 660, 510, + 385, 511, 386, 891, 891, 91, 155, 401, 401, 401, + 402, 401, 494, 156, 134, 96, 495, 496, 497, 498, + 396, 660, 92, 385, 135, 386, 374, 91, 155, 379, + 380, 381, 374, 382, 383, 384, 134, 92, 92, 92, + 104, 390, 672, 391, 405, 405, 405, 406, 405, 105, + 91, 407, 407, 407, 408, 407, 401, 527, 873, 102, + 103, 528, 104, 529, 390, 672, 391, 392, 392, 392, + + 393, 392, 91, 873, 407, 407, 407, 408, 407, 539, + 673, 102, 106, 540, 374, 541, 553, 91, 374, 374, + 374, 374, 506, 870, 108, 534, 507, 107, 508, 109, + 535, 870, 536, 673, 110, 106, 555, 102, 103, 91, + 93, 405, 405, 405, 406, 405, 108, 108, 175, 545, + 94, 109, 109, 546, 547, 548, 549, 110, 923, 102, + 924, 925, 93, 409, 409, 409, 410, 409, 92, 108, + 175, 557, 374, 864, 109, 558, 559, 560, 561, 106, + 468, 468, 468, 469, 468, 268, 864, 269, 92, 445, + 445, 445, 446, 445, 107, 374, 411, 450, 450, 450, + + 450, 450, 106, 452, 571, 565, 91, 453, 572, 454, + 573, 575, 112, 97, 113, 576, 114, 577, 904, 411, + 859, 98, 905, 99, 136, 583, 155, 859, 91, 583, + 583, 583, 583, 156, 112, 97, 113, 412, 412, 412, + 413, 412, 137, 98, 284, 100, 285, 136, 155, 614, + 455, 101, 904, 257, 374, 258, 905, 92, 374, 374, + 374, 374, 502, 502, 502, 503, 502, 100, 414, 23, + 23, 23, 23, 23, 504, 504, 504, 505, 504, 502, + 502, 502, 503, 502, 689, 24, 757, 757, 757, 757, + 116, 414, 148, 148, 148, 275, 148, 525, 525, 525, + + 526, 525, 525, 525, 525, 526, 525, 689, 25, 449, + 449, 449, 449, 449, 449, 449, 449, 449, 449, 853, + 853, 418, 26, 449, 449, 449, 449, 449, 449, 419, + 92, 25, 468, 468, 468, 469, 468, 532, 532, 532, + 533, 532, 617, 418, 850, 850, 449, 449, 449, 449, + 449, 449, 456, 456, 456, 456, 456, 456, 456, 456, + 456, 456, 834, 181, 618, 617, 456, 456, 456, 456, + 456, 456, 532, 532, 532, 533, 532, 834, 155, 182, + 517, 517, 517, 518, 517, 156, 181, 803, 803, 456, + 456, 456, 456, 456, 456, 553, 553, 553, 554, 553, + + 155, 471, 471, 471, 471, 471, 471, 471, 471, 471, + 471, 519, 797, 138, 691, 471, 471, 471, 471, 471, + 471, 797, 755, 543, 543, 543, 544, 543, 555, 555, + 555, 556, 555, 194, 519, 139, 138, 691, 471, 471, + 471, 471, 471, 471, 474, 474, 474, 474, 474, 474, + 474, 474, 474, 474, 755, 193, 95, 712, 474, 474, + 474, 474, 474, 474, 553, 553, 553, 554, 553, 565, + 565, 565, 566, 565, 96, 207, 208, 194, 193, 95, + 712, 474, 474, 474, 474, 474, 474, 475, 475, 475, + 475, 475, 475, 475, 475, 475, 475, 207, 979, 749, + + 980, 475, 475, 475, 475, 475, 475, 567, 567, 567, + 568, 567, 749, 492, 492, 492, 493, 492, 569, 569, + 569, 570, 569, 392, 475, 475, 475, 475, 475, 475, + 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + 923, 713, 924, 925, 476, 476, 476, 476, 476, 476, + 181, 579, 579, 579, 580, 579, 596, 92, 597, 598, + 583, 583, 583, 583, 713, 93, 182, 476, 476, 476, + 476, 476, 476, 181, 596, 94, 597, 598, 583, 583, + 583, 583, 111, 567, 652, 581, 92, 93, 653, 92, + 415, 415, 415, 416, 415, 623, 743, 606, 112, 430, + + 113, 431, 114, 607, 228, 111, 652, 743, 581, 582, + 583, 583, 583, 584, 585, 586, 587, 583, 583, 606, + 112, 583, 113, 583, 583, 583, 583, 583, 583, 394, + 394, 394, 395, 394, 990, 92, 991, 92, 397, 397, + 397, 398, 397, 92, 583, 92, 583, 583, 583, 583, + 583, 583, 583, 736, 736, 588, 589, 590, 583, 591, + 592, 593, 1009, 612, 601, 717, 1010, 594, 421, 595, + 422, 394, 91, 399, 399, 399, 399, 400, 399, 583, + 397, 97, 96, 583, 583, 583, 583, 601, 717, 98, + 594, 99, 595, 583, 91, 729, 588, 589, 590, 583, + + 591, 592, 593, 97, 281, 95, 718, 1000, 599, 1001, + 600, 98, 282, 405, 608, 729, 92, 91, 609, 100, + 610, 100, 97, 96, 403, 101, 281, 101, 95, 718, + 98, 599, 99, 600, 392, 392, 392, 393, 392, 91, + 569, 100, 611, 100, 97, 401, 401, 401, 402, 401, + 106, 583, 98, 207, 208, 583, 583, 583, 583, 403, + 403, 403, 404, 403, 611, 107, 92, 104, 405, 405, + 405, 406, 405, 106, 678, 207, 105, 93, 459, 407, + 407, 407, 408, 407, 460, 706, 108, 94, 91, 104, + 412, 109, 92, 1011, 706, 1012, 110, 102, 103, 93, + + 459, 625, 91, 104, 492, 263, 106, 264, 108, 543, + 91, 91, 105, 109, 407, 407, 407, 408, 407, 102, + 115, 107, 108, 661, 91, 104, 92, 109, 284, 106, + 285, 699, 110, 91, 405, 405, 405, 406, 405, 699, + 181, 583, 116, 115, 108, 583, 583, 583, 583, 109, + 409, 409, 409, 410, 409, 722, 182, 108, 604, 409, + 207, 208, 109, 181, 628, 583, 407, 110, 629, 583, + 630, 723, 106, 583, 583, 583, 583, 228, 722, 108, + 724, 604, 207, 602, 109, 695, 631, 107, 583, 181, + 437, 111, 438, 91, 723, 106, 735, 695, 619, 112, + + 134, 113, 620, 114, 621, 182, 602, 112, 108, 113, + 135, 114, 181, 109, 111, 91, 92, 92, 110, 735, + 622, 112, 134, 113, 412, 412, 412, 413, 412, 112, + 108, 113, 92, 690, 690, 109, 615, 615, 615, 616, + 615, 583, 445, 622, 434, 583, 583, 583, 583, 615, + 615, 615, 616, 615, 443, 603, 626, 626, 626, 627, + 626, 683, 435, 447, 633, 737, 683, 434, 268, 136, + 269, 634, 634, 634, 635, 634, 136, 116, 603, 636, + 677, 677, 254, 141, 142, 143, 144, 137, 737, 255, + 260, 92, 136, 138, 137, 254, 134, 658, 473, 136, + + 703, 462, 255, 463, 254, 704, 135, 705, 265, 261, + 421, 458, 422, 260, 654, 139, 138, 254, 134, 655, + 632, 656, 664, 669, 266, 632, 158, 670, 159, 671, + 517, 265, 637, 637, 637, 637, 637, 637, 637, 637, + 637, 637, 779, 468, 657, 787, 637, 637, 637, 637, + 637, 637, 450, 450, 450, 450, 450, 674, 452, 624, + 193, 675, 453, 676, 454, 779, 657, 624, 787, 637, + 637, 637, 637, 637, 637, 638, 638, 638, 638, 638, + 680, 1102, 194, 193, 681, 1102, 682, 454, 155, 482, + 482, 482, 483, 482, 613, 156, 482, 482, 482, 483, + + 482, 613, 684, 788, 845, 455, 685, 686, 687, 688, + 155, 678, 678, 678, 679, 678, 502, 502, 502, 503, + 502, 504, 504, 504, 505, 504, 788, 845, 455, 644, + 645, 646, 646, 646, 645, 647, 644, 647, 647, 647, + 644, 644, 648, 647, 647, 647, 647, 649, 649, 649, + 649, 649, 649, 649, 649, 649, 649, 647, 647, 647, + 647, 649, 649, 649, 649, 649, 649, 647, 647, 647, + 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + 647, 647, 647, 650, 649, 649, 649, 649, 649, 649, + 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + + 647, 647, 647, 647, 647, 647, 647, 647, 67, 67, + 67, 151, 67, 92, 578, 662, 662, 662, 663, 662, + 492, 492, 492, 493, 492, 651, 651, 651, 651, 651, + 651, 651, 651, 651, 651, 757, 757, 757, 757, 651, + 651, 651, 651, 651, 651, 502, 502, 502, 503, 502, + 692, 708, 578, 574, 693, 696, 694, 181, 574, 697, + 281, 698, 651, 651, 651, 651, 651, 651, 282, 562, + 324, 709, 562, 182, 708, 714, 710, 551, 711, 715, + 181, 716, 281, 163, 78, 78, 164, 163, 551, 28, + 525, 525, 525, 526, 525, 700, 700, 700, 701, 700, + + 665, 665, 665, 665, 665, 665, 665, 665, 665, 665, + 757, 757, 757, 757, 665, 665, 665, 665, 665, 665, + 525, 525, 525, 526, 525, 430, 542, 431, 254, 702, + 532, 532, 532, 533, 532, 255, 29, 665, 665, 665, + 665, 665, 665, 82, 82, 82, 168, 82, 324, 719, + 254, 780, 702, 542, 720, 781, 721, 30, 92, 516, + 666, 666, 666, 666, 666, 666, 666, 666, 666, 666, + 437, 516, 438, 780, 666, 666, 666, 666, 666, 666, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 32, 666, 666, 666, + + 666, 666, 666, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 512, 617, + 30, 667, 667, 667, 667, 667, 667, 667, 667, 667, + 667, 141, 142, 143, 144, 667, 667, 667, 667, 667, + 667, 618, 617, 517, 517, 517, 518, 517, 532, 532, + 532, 533, 532, 724, 724, 724, 725, 724, 667, 667, + 667, 667, 667, 667, 668, 668, 668, 668, 668, 668, + 668, 668, 668, 668, 707, 462, 265, 463, 668, 668, + 668, 668, 668, 668, 724, 724, 724, 725, 724, 726, + 512, 827, 266, 727, 746, 728, 194, 707, 747, 265, + + 748, 668, 668, 668, 668, 668, 668, 543, 543, 543, + 544, 543, 730, 828, 827, 846, 731, 732, 733, 734, + 553, 553, 553, 554, 553, 555, 555, 555, 556, 555, + 553, 553, 553, 554, 553, 738, 579, 750, 846, 739, + 740, 741, 742, 565, 565, 565, 566, 565, 567, 567, + 567, 568, 567, 569, 569, 569, 570, 569, 92, 207, + 208, 750, 750, 750, 751, 750, 752, 227, 500, 227, + 753, 500, 754, 579, 579, 579, 580, 579, 786, 1009, + 789, 207, 609, 1010, 610, 421, 228, 422, 228, 92, + 227, 770, 227, 771, 772, 757, 757, 757, 757, 770, + + 491, 771, 772, 757, 757, 757, 757, 756, 792, 793, + 642, 833, 257, 794, 258, 795, 655, 491, 656, 640, + 415, 415, 415, 416, 415, 1049, 228, 1050, 92, 487, + 756, 757, 757, 757, 757, 758, 759, 760, 761, 757, + 757, 757, 757, 757, 757, 757, 757, 757, 757, 757, + 757, 762, 763, 764, 757, 765, 766, 767, 757, 757, + 757, 757, 643, 768, 487, 769, 615, 299, 757, 757, + 757, 757, 757, 757, 181, 762, 763, 764, 757, 765, + 766, 767, 854, 855, 856, 857, 768, 773, 769, 774, + 182, 757, 757, 757, 757, 782, 858, 181, 473, 606, + + 783, 838, 784, 796, 778, 607, 284, 620, 285, 621, + 773, 254, 774, 392, 392, 392, 393, 392, 255, 858, + 798, 606, 466, 228, 430, 785, 431, 778, 482, 482, + 482, 483, 482, 254, 757, 757, 757, 757, 260, 790, + 790, 790, 791, 790, 801, 652, 860, 785, 263, 653, + 264, 799, 799, 799, 800, 799, 93, 261, 802, 804, + 626, 260, 629, 437, 630, 438, 94, 652, 466, 860, + 805, 805, 805, 806, 805, 634, 458, 441, 93, 394, + 394, 394, 395, 394, 418, 807, 878, 879, 441, 268, + 427, 269, 419, 260, 433, 638, 638, 638, 638, 638, + + 757, 757, 757, 757, 428, 434, 418, 454, 433, 878, + 879, 265, 261, 427, 775, 372, 260, 809, 809, 809, + 809, 809, 609, 435, 610, 639, 829, 266, 434, 810, + 640, 830, 96, 831, 265, 372, 865, 775, 397, 397, + 397, 398, 397, 638, 638, 638, 638, 638, 835, 1102, + 92, 92, 462, 1102, 463, 454, 882, 92, 832, 757, + 757, 757, 757, 363, 638, 638, 638, 638, 638, 847, + 1102, 363, 883, 848, 1102, 849, 454, 641, 92, 882, + 832, 97, 646, 646, 646, 646, 646, 662, 92, 98, + 620, 99, 621, 92, 454, 883, 455, 356, 638, 638, + + 638, 638, 638, 97, 1102, 629, 356, 630, 1102, 887, + 454, 98, 399, 399, 399, 400, 399, 455, 638, 638, + 638, 638, 638, 861, 1102, 349, 349, 862, 1102, 863, + 454, 281, 887, 757, 757, 757, 757, 92, 92, 282, + 836, 836, 836, 837, 836, 836, 836, 836, 837, 836, + 655, 455, 656, 281, 482, 482, 482, 483, 482, 100, + 678, 678, 678, 679, 678, 101, 502, 502, 502, 503, + 502, 455, 323, 492, 492, 492, 493, 492, 875, 888, + 92, 100, 401, 401, 401, 402, 401, 459, 896, 898, + 867, 324, 459, 460, 868, 323, 869, 324, 460, 700, + + 331, 875, 888, 757, 757, 757, 757, 331, 92, 459, + 181, 896, 898, 92, 459, 504, 504, 504, 505, 504, + 502, 502, 502, 503, 502, 321, 182, 865, 865, 865, + 866, 865, 323, 181, 102, 103, 700, 700, 700, 701, + 700, 517, 517, 517, 518, 517, 525, 525, 525, 526, + 525, 324, 783, 321, 784, 323, 102, 403, 403, 403, + 404, 403, 525, 525, 525, 526, 525, 977, 314, 931, + 708, 314, 874, 532, 532, 532, 533, 532, 757, 757, + 757, 757, 532, 532, 532, 533, 532, 92, 228, 324, + 977, 884, 931, 708, 194, 874, 885, 794, 886, 795, + + 418, 104, 724, 724, 724, 725, 724, 814, 419, 92, + 105, 724, 724, 724, 725, 724, 640, 543, 543, 543, + 544, 543, 418, 104, 405, 405, 405, 406, 405, 892, + 893, 894, 895, 553, 553, 553, 554, 553, 555, 555, + 555, 556, 555, 92, 299, 757, 757, 757, 757, 553, + 553, 553, 554, 553, 899, 900, 901, 902, 167, 643, + 181, 978, 106, 565, 565, 565, 566, 565, 154, 207, + 208, 567, 567, 567, 568, 567, 182, 107, 569, 569, + 569, 570, 569, 181, 978, 106, 407, 407, 407, 408, + 407, 207, 906, 273, 273, 983, 907, 985, 908, 750, + + 750, 750, 751, 750, 252, 915, 916, 917, 427, 918, + 919, 920, 579, 579, 579, 580, 579, 921, 983, 922, + 985, 92, 428, 392, 392, 392, 393, 392, 937, 108, + 827, 427, 780, 783, 109, 784, 781, 91, 92, 110, + 921, 92, 922, 915, 916, 917, 913, 918, 919, 920, + 92, 108, 828, 827, 780, 926, 109, 927, 790, 91, + 409, 409, 409, 410, 409, 228, 93, 92, 995, 913, + 394, 394, 394, 395, 394, 830, 94, 831, 926, 92, + 927, 415, 415, 415, 416, 415, 939, 324, 93, 434, + 609, 995, 610, 776, 92, 92, 397, 397, 397, 398, + + 397, 942, 418, 91, 92, 928, 421, 435, 422, 112, + 419, 113, 434, 114, 92, 92, 776, 399, 399, 399, + 400, 399, 933, 96, 418, 91, 943, 934, 928, 935, + 794, 112, 795, 113, 412, 412, 412, 413, 412, 97, + 92, 965, 401, 401, 401, 402, 401, 98, 92, 99, + 966, 934, 89, 935, 936, 757, 757, 757, 757, 167, + 162, 97, 945, 965, 100, 777, 620, 162, 621, 98, + 101, 403, 403, 403, 404, 403, 936, 154, 996, 405, + 405, 405, 406, 405, 799, 997, 100, 116, 777, 405, + 405, 405, 406, 405, 102, 103, 407, 407, 407, 408, + + 407, 996, 132, 412, 412, 412, 413, 412, 997, 92, + 757, 757, 757, 757, 948, 104, 102, 106, 430, 949, + 431, 965, 427, 629, 105, 630, 805, 106, 92, 952, + 966, 90, 107, 437, 930, 438, 428, 104, 89, 108, + 106, 970, 107, 965, 109, 427, 830, 87, 831, 110, + 106, 407, 407, 407, 408, 407, 116, 930, 967, 975, + 434, 108, 968, 462, 969, 463, 109, 940, 940, 940, + 941, 940, 950, 950, 950, 951, 950, 998, 435, 968, + 1039, 969, 1040, 434, 405, 405, 405, 406, 405, 809, + 809, 809, 809, 809, 108, 999, 81, 71, 972, 109, + + 998, 810, 91, 655, 110, 656, 407, 407, 407, 408, + 407, 986, 70, 459, 606, 987, 108, 988, 999, 460, + 607, 109, 106, 1002, 91, 808, 808, 808, 808, 808, + 808, 808, 808, 808, 808, 459, 606, 107, 1003, 808, + 808, 808, 808, 808, 808, 106, 1002, 69, 1017, 108, + 1005, 1007, 1022, 934, 109, 935, 609, 57, 610, 110, + 617, 1003, 808, 808, 808, 808, 808, 808, 639, 639, + 812, 108, 1053, 1005, 1007, 39, 109, 946, 946, 946, + 947, 946, 618, 617, 813, 813, 813, 813, 813, 813, + 813, 813, 813, 813, 1102, 1053, 1054, 1072, 813, 813, + + 813, 813, 813, 813, 1068, 1102, 1069, 1102, 617, 482, + 482, 482, 483, 482, 482, 482, 482, 483, 482, 1054, + 1072, 813, 813, 813, 813, 813, 813, 642, 642, 815, + 618, 617, 678, 678, 678, 679, 678, 502, 502, 502, + 503, 502, 1102, 816, 816, 816, 816, 816, 816, 816, + 816, 816, 816, 1102, 1073, 1102, 1102, 816, 816, 816, + 816, 816, 816, 638, 638, 638, 638, 638, 1023, 1102, + 1102, 1019, 794, 1102, 795, 454, 783, 1073, 784, 1102, + 816, 816, 816, 816, 816, 816, 817, 817, 817, 818, + 817, 606, 1102, 1102, 1102, 1026, 1102, 607, 454, 620, + + 1102, 621, 1102, 819, 819, 819, 819, 819, 819, 819, + 819, 819, 819, 606, 1102, 1102, 455, 819, 819, 819, + 819, 819, 819, 504, 504, 504, 505, 504, 502, 502, + 502, 503, 502, 865, 865, 865, 866, 865, 1102, 455, + 819, 819, 819, 819, 819, 819, 820, 821, 638, 638, + 638, 821, 822, 820, 822, 822, 822, 820, 820, 823, + 822, 822, 822, 822, 824, 824, 824, 824, 824, 824, + 824, 824, 824, 824, 822, 822, 822, 822, 824, 824, + 824, 824, 824, 824, 822, 822, 822, 822, 822, 822, + 822, 822, 822, 822, 822, 822, 822, 822, 822, 822, + + 825, 824, 824, 824, 824, 824, 824, 822, 822, 822, + 822, 822, 822, 822, 822, 822, 822, 822, 822, 822, + 822, 822, 822, 822, 822, 826, 826, 826, 826, 826, + 826, 826, 826, 826, 826, 1102, 1102, 1102, 1102, 826, + 826, 826, 826, 826, 826, 638, 817, 638, 638, 638, + 1027, 1102, 1102, 1044, 629, 1102, 630, 454, 830, 1102, + 831, 1102, 826, 826, 826, 826, 826, 826, 163, 78, + 78, 164, 163, 827, 28, 525, 525, 525, 526, 525, + 525, 525, 525, 526, 525, 839, 839, 839, 839, 839, + 839, 839, 839, 839, 839, 828, 827, 1102, 455, 839, + + 839, 839, 839, 839, 839, 646, 646, 646, 646, 646, + 973, 973, 973, 974, 973, 1102, 1036, 454, 1102, 1102, + 1037, 29, 839, 839, 839, 839, 839, 839, 840, 840, + 840, 840, 840, 840, 840, 840, 840, 840, 1036, 1102, + 1102, 1102, 840, 840, 840, 840, 840, 840, 638, 638, + 638, 638, 638, 517, 517, 517, 518, 517, 962, 652, + 454, 1102, 1102, 653, 1102, 840, 840, 840, 840, 840, + 840, 841, 841, 841, 841, 841, 841, 841, 841, 841, + 841, 652, 1102, 1102, 994, 841, 841, 841, 841, 841, + 841, 1102, 638, 638, 638, 638, 638, 1102, 1102, 1042, + + 1102, 962, 1102, 968, 454, 969, 194, 994, 841, 841, + 841, 841, 841, 841, 842, 842, 842, 842, 842, 842, + 842, 842, 842, 842, 1102, 1102, 1102, 1102, 842, 842, + 842, 842, 842, 842, 1102, 1102, 940, 940, 940, 941, + 940, 836, 1102, 1102, 1102, 455, 532, 532, 532, 533, + 532, 842, 842, 842, 842, 842, 842, 914, 914, 914, + 914, 914, 914, 914, 914, 914, 914, 1102, 1102, 1102, + 1102, 914, 914, 914, 914, 914, 914, 946, 646, 646, + 646, 646, 646, 606, 652, 1102, 1102, 459, 653, 607, + 454, 1102, 1102, 460, 914, 914, 914, 914, 914, 914, + + 409, 409, 409, 410, 409, 606, 652, 617, 1102, 459, + 163, 78, 78, 164, 163, 1102, 28, 532, 532, 532, + 533, 532, 724, 724, 724, 725, 724, 1102, 780, 618, + 617, 962, 781, 929, 724, 724, 724, 725, 724, 553, + 553, 553, 554, 553, 555, 555, 555, 556, 555, 112, + 780, 113, 1102, 114, 1102, 1102, 929, 553, 553, 553, + 554, 553, 1102, 29, 565, 565, 565, 566, 565, 1102, + 1102, 112, 1102, 113, 953, 953, 953, 953, 953, 953, + 953, 953, 953, 953, 1102, 1102, 1102, 1102, 953, 953, + 953, 953, 953, 953, 492, 492, 492, 493, 492, 567, + + 567, 567, 568, 567, 569, 569, 569, 570, 569, 1102, + 1102, 953, 953, 953, 953, 953, 953, 954, 811, 811, + 955, 954, 1102, 640, 750, 750, 750, 751, 750, 1102, + 1102, 181, 1102, 1102, 956, 956, 956, 956, 956, 956, + 956, 956, 956, 956, 1102, 1102, 1102, 182, 956, 956, + 956, 956, 956, 956, 181, 1024, 1024, 1024, 1025, 1024, + 148, 148, 148, 275, 148, 678, 678, 678, 679, 678, + 641, 956, 956, 956, 956, 956, 956, 957, 814, 814, + 958, 957, 1036, 1047, 1056, 1102, 1037, 640, 655, 934, + 656, 935, 1102, 1102, 959, 959, 959, 959, 959, 959, + + 959, 959, 959, 959, 1036, 1102, 1102, 1102, 959, 959, + 959, 959, 959, 959, 700, 700, 700, 701, 700, 1102, + 543, 543, 543, 544, 543, 865, 865, 865, 866, 865, + 643, 959, 959, 959, 959, 959, 959, 960, 960, 960, + 960, 960, 960, 960, 960, 960, 960, 1102, 875, 1102, + 1102, 960, 960, 960, 960, 960, 960, 117, 117, 117, + 232, 117, 1060, 1102, 1102, 811, 794, 324, 795, 1102, + 640, 875, 207, 208, 960, 960, 960, 960, 960, 960, + 821, 821, 821, 963, 821, 724, 724, 724, 725, 724, + 1102, 1102, 1102, 1102, 207, 1102, 1102, 964, 964, 964, + + 964, 964, 964, 964, 964, 964, 964, 1102, 1102, 1102, + 92, 964, 964, 964, 964, 964, 964, 641, 724, 724, + 724, 725, 724, 750, 750, 750, 751, 750, 1057, 1057, + 1057, 1058, 1057, 1102, 964, 964, 964, 964, 964, 964, + 644, 645, 646, 646, 646, 645, 647, 644, 647, 647, + 647, 644, 644, 648, 647, 647, 647, 647, 649, 649, + 649, 649, 649, 649, 649, 649, 649, 649, 647, 647, + 647, 647, 649, 649, 649, 649, 649, 649, 647, 647, + 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + 647, 647, 647, 647, 650, 649, 649, 649, 649, 649, + + 649, 647, 647, 647, 647, 647, 647, 647, 647, 647, + 647, 647, 647, 647, 647, 647, 647, 647, 647, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 1102, + 1102, 1102, 1102, 64, 64, 64, 64, 64, 64, 579, + 579, 579, 580, 579, 412, 412, 412, 413, 412, 1071, + 1036, 1102, 1102, 1039, 1037, 1040, 64, 64, 64, 64, + 64, 64, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 1036, 1015, 1102, 115, 30, 30, 30, 30, + 30, 30, 394, 394, 394, 395, 394, 1102, 1102, 1038, + 1102, 1102, 228, 1039, 1102, 1040, 1015, 116, 115, 30, + + 30, 30, 30, 30, 30, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 1102, 1041, 95, 1102, 33, + 33, 33, 33, 33, 33, 1102, 392, 392, 392, 393, + 392, 1102, 1059, 1102, 1102, 96, 1102, 783, 1041, 784, + 95, 1102, 33, 33, 33, 33, 33, 33, 976, 976, + 976, 976, 976, 976, 976, 976, 976, 976, 1102, 1102, + 1102, 1102, 976, 976, 976, 976, 976, 976, 1102, 93, + 397, 397, 397, 398, 397, 1102, 1102, 973, 940, 94, + 399, 399, 399, 400, 399, 976, 976, 976, 976, 976, + 976, 93, 401, 401, 401, 402, 401, 403, 403, 403, + + 404, 403, 1102, 1102, 405, 405, 405, 406, 405, 1045, + 1102, 1102, 1102, 97, 1102, 1102, 407, 407, 407, 408, + 407, 98, 1074, 99, 606, 652, 968, 100, 969, 653, + 607, 1077, 1102, 101, 1102, 97, 830, 1102, 831, 827, + 1102, 104, 106, 98, 102, 103, 606, 652, 1102, 100, + 105, 415, 415, 415, 416, 415, 1102, 107, 1102, 108, + 1102, 828, 827, 104, 109, 106, 102, 1102, 1102, 110, + 405, 405, 405, 406, 405, 1102, 407, 407, 407, 408, + 407, 108, 1075, 1102, 1102, 1102, 109, 409, 409, 409, + 410, 409, 1102, 1102, 1102, 117, 117, 117, 232, 117, + + 1079, 1102, 1102, 1102, 92, 934, 1102, 935, 106, 117, + 117, 117, 232, 117, 117, 117, 117, 232, 117, 108, + 111, 1085, 1102, 107, 109, 965, 1068, 1102, 1069, 110, + 1102, 106, 181, 1087, 966, 1102, 112, 1039, 113, 1040, + 114, 108, 1102, 111, 1102, 1102, 109, 965, 182, 117, + 117, 117, 232, 117, 1102, 181, 1102, 1090, 112, 1095, + 113, 968, 92, 969, 1068, 1102, 1069, 92, 1020, 1020, + 1020, 1021, 1020, 646, 646, 646, 646, 646, 638, 821, + 638, 638, 638, 227, 1102, 454, 1102, 1102, 1102, 1102, + 454, 865, 865, 865, 866, 865, 1102, 1045, 1045, 1045, + + 1046, 1045, 228, 1102, 1102, 1102, 227, 175, 175, 175, + 300, 175, 482, 482, 482, 483, 482, 780, 1098, 1102, + 1067, 781, 1039, 1020, 1040, 1068, 962, 1069, 827, 1102, + 1102, 962, 1096, 1096, 1096, 1097, 1096, 1102, 1102, 780, + 1028, 1028, 1028, 1028, 1028, 1028, 1028, 1028, 1028, 1028, + 828, 827, 1070, 1102, 1028, 1028, 1028, 1028, 1028, 1028, + 92, 1102, 1102, 1102, 1102, 92, 482, 482, 482, 483, + 482, 780, 1102, 1102, 1070, 781, 1102, 1028, 1028, 1028, + 1028, 1028, 1028, 1029, 1029, 1029, 1029, 1029, 1029, 1029, + 1029, 1029, 1029, 780, 1102, 1102, 1102, 1029, 1029, 1029, + + 1029, 1029, 1029, 502, 502, 502, 503, 502, 504, 504, + 504, 505, 504, 821, 821, 821, 963, 821, 1102, 92, + 1029, 1029, 1029, 1029, 1029, 1029, 1030, 1030, 1030, 1030, + 1030, 1030, 1030, 1030, 1030, 1030, 1102, 1102, 1102, 1102, + 1030, 1030, 1030, 1030, 1030, 1030, 1100, 1102, 1102, 1102, + 1102, 1068, 1102, 1069, 1102, 1102, 92, 1102, 1102, 1102, + 1102, 92, 1102, 1030, 1030, 1030, 1030, 1030, 1030, 820, + 1031, 646, 646, 646, 1031, 1032, 820, 1032, 1032, 1032, + 820, 820, 823, 1032, 1032, 1032, 1032, 1033, 1033, 1033, + 1033, 1033, 1033, 1033, 1033, 1033, 1033, 1032, 1032, 1032, + + 1032, 1033, 1033, 1033, 1033, 1033, 1033, 1032, 1032, 1032, + 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, + 1032, 1032, 1032, 1034, 1033, 1033, 1033, 1033, 1033, 1033, + 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, + 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1035, 1035, + 1035, 1035, 1035, 1035, 1035, 1035, 1035, 1035, 1102, 1102, + 1102, 1102, 1035, 1035, 1035, 1035, 1035, 1035, 492, 492, + 492, 493, 492, 502, 502, 502, 503, 502, 700, 700, + 700, 701, 700, 1102, 1102, 1035, 1035, 1035, 1035, 1035, + 1035, 517, 517, 517, 518, 517, 175, 175, 175, 300, + + 175, 1102, 1102, 1102, 1102, 181, 175, 175, 175, 300, + 175, 1102, 995, 525, 525, 525, 526, 525, 1102, 1102, + 1102, 182, 193, 1102, 1102, 1102, 92, 1102, 181, 1102, + 323, 324, 1102, 1102, 1102, 995, 525, 525, 525, 526, + 525, 1102, 1102, 1102, 194, 193, 1102, 1102, 1102, 324, + 1102, 1102, 1102, 323, 532, 532, 532, 533, 532, 92, + 532, 532, 532, 533, 532, 1102, 92, 543, 543, 543, + 544, 543, 553, 553, 553, 554, 553, 555, 555, 555, + 556, 555, 553, 553, 553, 554, 553, 1102, 1088, 92, + 565, 565, 565, 566, 565, 567, 567, 567, 568, 567, + + 569, 569, 569, 570, 569, 1102, 1102, 92, 579, 579, + 579, 580, 579, 92, 175, 175, 175, 300, 175, 207, + 208, 1102, 1102, 1102, 1102, 92, 1102, 1102, 1102, 1102, + 92, 1102, 1102, 1102, 1102, 92, 1036, 1102, 1102, 1102, + 1037, 207, 227, 92, 1102, 1102, 1102, 1102, 92, 1102, + 1102, 1102, 1102, 92, 1102, 1102, 1102, 1102, 1036, 1102, + 1102, 228, 1102, 1102, 1102, 227, 1102, 92, 1061, 1061, + 1061, 1061, 1061, 1061, 1061, 1061, 1061, 1061, 1102, 1102, + 1102, 1102, 1061, 1061, 1061, 1061, 1061, 1061, 646, 646, + 646, 646, 646, 678, 678, 678, 679, 678, 1102, 1102, + + 454, 1102, 1102, 1102, 1102, 1061, 1061, 1061, 1061, 1061, + 1061, 1062, 1062, 1062, 1062, 1062, 1062, 1062, 1062, 1062, + 1062, 1102, 1102, 1102, 1102, 1062, 1062, 1062, 1062, 1062, + 1062, 646, 646, 646, 646, 646, 1075, 1075, 1075, 1076, + 1075, 962, 1102, 454, 1102, 1102, 92, 1102, 1062, 1062, + 1062, 1062, 1062, 1062, 1063, 1063, 1063, 1063, 1063, 1063, + 1063, 1063, 1063, 1063, 1102, 1102, 1102, 1102, 1063, 1063, + 1063, 1063, 1063, 1063, 724, 724, 724, 725, 724, 1102, + 965, 1102, 1102, 1102, 962, 1102, 1102, 1102, 1102, 966, + 1102, 1063, 1063, 1063, 1063, 1063, 1063, 1031, 1031, 1031, + + 1064, 1031, 965, 1102, 1102, 1102, 1102, 1102, 1102, 454, + 1102, 1102, 1102, 1102, 1065, 1065, 1065, 1065, 1065, 1065, + 1065, 1065, 1065, 1065, 1102, 1102, 1102, 92, 1065, 1065, + 1065, 1065, 1065, 1065, 700, 700, 700, 701, 700, 724, + 724, 724, 725, 724, 1102, 1102, 1102, 1102, 1102, 1102, + 962, 1065, 1065, 1065, 1065, 1065, 1065, 1066, 1066, 1066, + 1066, 1066, 1066, 1066, 1066, 1066, 1066, 1102, 323, 1102, + 1102, 1066, 1066, 1066, 1066, 1066, 1066, 750, 750, 750, + 751, 750, 1102, 1102, 1102, 1102, 1102, 324, 1102, 1102, + 1102, 323, 92, 1102, 1066, 1066, 1066, 1066, 1066, 1066, + + 646, 1031, 646, 646, 646, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 454, 1088, 1088, 1088, 1089, 1088, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 92, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, + 1080, 1102, 1102, 1102, 1102, 1080, 1080, 1080, 1080, 1080, + 1080, 1102, 1102, 962, 1102, 1102, 1088, 1088, 1088, 1089, + 1088, 1102, 1036, 1102, 1102, 1102, 1037, 1102, 1080, 1080, + 1080, 1080, 1080, 1080, 1081, 1081, 1081, 1081, 1081, 1081, + 1081, 1081, 1081, 1081, 1036, 1102, 1102, 1102, 1081, 1081, + 1081, 1081, 1081, 1081, 865, 865, 865, 866, 865, 1102, + + 1102, 1102, 1102, 1102, 1102, 1036, 1102, 1102, 1102, 1037, + 1102, 1081, 1081, 1081, 1081, 1081, 1081, 1082, 1082, 1082, + 1082, 1082, 1082, 1082, 1082, 1082, 1082, 1036, 1102, 1102, + 1102, 1082, 1082, 1082, 1082, 1082, 1082, 1031, 1031, 1031, + 1064, 1031, 1102, 1102, 1102, 1102, 1102, 92, 1102, 454, + 1102, 1102, 1102, 1102, 1082, 1082, 1082, 1082, 1082, 1082, + 1083, 1083, 1083, 1083, 1083, 1083, 1083, 1083, 1083, 1083, + 1102, 1102, 1102, 1102, 1083, 1083, 1083, 1083, 1083, 1083, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 962, 1102, 1102, 1102, 1102, 1102, 1102, 1083, 1083, 1083, + + 1083, 1083, 1083, 1084, 1084, 1084, 1084, 1084, 1084, 1084, + 1084, 1084, 1084, 1102, 1102, 1102, 1102, 1084, 1084, 1084, + 1084, 1084, 1084, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1084, 1084, 1084, 1084, 1084, 1084, 1091, 1091, 1091, 1091, + 1091, 1091, 1091, 1091, 1091, 1091, 1102, 1102, 1102, 1102, + 1091, 1091, 1091, 1091, 1091, 1091, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1091, 1091, 1091, 1091, 1091, 1091, 1092, + 1092, 1092, 1092, 1092, 1092, 1092, 1092, 1092, 1092, 1102, + + 1102, 1102, 1102, 1092, 1092, 1092, 1092, 1092, 1092, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1092, 1092, 1092, 1092, + 1092, 1092, 451, 451, 451, 451, 451, 451, 451, 451, + 451, 451, 1102, 1102, 1102, 1102, 451, 451, 451, 451, + 451, 451, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 451, + 451, 451, 451, 451, 451, 1093, 1093, 1093, 1093, 1093, + 1093, 1093, 1093, 1093, 1093, 1102, 1102, 1102, 1102, 1093, + 1093, 1093, 1093, 1093, 1093, 1102, 1102, 1102, 1102, 1102, + + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1093, 1093, 1093, 1093, 1093, 1093, 1094, 1094, + 1094, 1094, 1094, 1094, 1094, 1094, 1094, 1094, 1102, 1102, + 1102, 1102, 1094, 1094, 1094, 1094, 1094, 1094, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1094, 1094, 1094, 1094, 1094, + 1094, 639, 639, 639, 639, 639, 639, 639, 639, 639, + 639, 1102, 1102, 1102, 1102, 639, 639, 639, 639, 639, + 639, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 639, 639, + + 639, 639, 639, 639, 642, 642, 642, 642, 642, 642, + 642, 642, 642, 642, 1102, 1102, 1102, 1102, 642, 642, + 642, 642, 642, 642, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 642, 642, 642, 642, 642, 642, 1099, 1099, 1099, + 1099, 1099, 1099, 1099, 1099, 1099, 1099, 1102, 1102, 1102, + 1102, 1099, 1099, 1099, 1099, 1099, 1099, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1099, 1099, 1099, 1099, 1099, 1099, + 1101, 1101, 1101, 1101, 1101, 1101, 1101, 1101, 1101, 1101, + + 1102, 1102, 1102, 1102, 1101, 1101, 1101, 1101, 1101, 1101, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1101, 1101, 1101, + 1101, 1101, 1101, 27, 27, 1102, 27, 27, 27, 27, + 27, 27, 30, 30, 30, 30, 33, 33, 1102, 33, + 33, 33, 33, 33, 33, 36, 1102, 1102, 36, 64, + 64, 1102, 64, 64, 67, 67, 1102, 67, 67, 67, + 67, 67, 67, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 82, 82, 1102, 82, 82, 82, 82, 82, + 82, 84, 84, 84, 84, 84, 84, 84, 84, 84, + + 88, 88, 88, 88, 88, 88, 88, 88, 88, 91, + 1102, 91, 91, 148, 148, 1102, 148, 148, 148, 148, + 148, 148, 153, 153, 153, 153, 153, 153, 153, 153, + 153, 166, 166, 166, 166, 166, 166, 166, 166, 166, + 173, 173, 173, 173, 173, 173, 173, 173, 173, 175, + 175, 1102, 175, 175, 175, 175, 175, 175, 279, 279, + 279, 279, 279, 279, 279, 279, 279, 294, 294, 294, + 294, 294, 294, 294, 294, 294, 298, 298, 298, 298, + 298, 298, 298, 298, 298, 451, 451, 451, 1102, 451, + 451, 451, 451, 457, 457, 457, 457, 457, 457, 457, + + 457, 457, 472, 472, 472, 472, 472, 472, 472, 472, + 472, 173, 173, 173, 173, 173, 173, 173, 173, 173, + 639, 639, 1102, 639, 639, 639, 639, 639, 639, 642, + 642, 1102, 642, 642, 642, 642, 642, 642, 457, 457, + 457, 457, 457, 457, 457, 457, 457, 279, 279, 279, + 279, 279, 279, 279, 279, 279, 27, 27, 27, 27, + 27, 27, 27, 27, 27, 472, 472, 472, 472, 472, + 472, 472, 472, 472, 294, 294, 294, 294, 294, 294, + 294, 294, 294, 811, 811, 811, 811, 811, 811, 811, + 811, 811, 814, 814, 814, 814, 814, 814, 814, 814, + + 814, 961, 961, 1102, 1102, 961, 961, 961, 961, 3, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102 + } ; + +static const flex_int16_t yy_chk[7190] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, + 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, + 7, 7, 7, 7, 8, 9, 10, 11, 17, 22, + + 22, 22, 22, 22, 36, 7, 12, 12, 12, 12, + 12, 12, 12, 12, 12, 12, 25, 30, 82, 470, + 1113, 470, 58, 27, 16, 46, 25, 26, 7, 64, + 16, 26, 9, 26, 16, 1097, 11, 16, 25, 10, + 17, 8, 7, 46, 16, 58, 36, 16, 46, 26, + 133, 7, 14, 16, 30, 82, 16, 14, 14, 16, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 27, 64, 26, 133, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 18, 33, + 14, 38, 67, 38, 38, 38, 38, 38, 38, 38, + 38, 38, 38, 49, 49, 23, 23, 23, 23, 23, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 48, 23, 1069, 45, 18, 49, 48, 84, 78, 1068, + 18, 74, 33, 45, 67, 74, 89, 74, 154, 89, + 167, 154, 48, 167, 23, 45, 18, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 51, 23, 170, + 50, 19, 19, 19, 19, 19, 19, 23, 1058, 50, + + 84, 47, 51, 52, 54, 78, 61, 1050, 52, 47, + 51, 47, 50, 52, 19, 19, 19, 19, 19, 19, + 29, 29, 29, 47, 59, 52, 54, 54, 61, 61, + 52, 47, 170, 664, 59, 664, 29, 29, 29, 29, + 29, 29, 29, 29, 29, 29, 59, 60, 148, 72, + 29, 29, 29, 29, 29, 29, 72, 150, 79, 85, + 66, 299, 195, 79, 299, 60, 1049, 195, 85, 195, + 60, 72, 1025, 29, 29, 29, 29, 29, 29, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 53, + 148, 66, 126, 32, 32, 32, 32, 32, 32, 150, + + 73, 126, 66, 1012, 73, 53, 73, 53, 100, 53, + 79, 85, 53, 66, 126, 77, 32, 32, 32, 32, + 32, 32, 34, 34, 34, 77, 73, 53, 124, 53, + 100, 100, 222, 1011, 124, 95, 222, 77, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 73, 151, + 124, 95, 34, 34, 34, 34, 34, 34, 95, 151, + 224, 991, 990, 163, 224, 96, 105, 988, 453, 96, + 105, 96, 105, 987, 980, 34, 34, 34, 34, 34, + 34, 37, 37, 37, 37, 37, 37, 37, 37, 37, + 37, 96, 105, 105, 130, 37, 37, 37, 37, 37, + + 37, 151, 119, 979, 106, 106, 119, 119, 119, 119, + 163, 453, 969, 96, 105, 105, 130, 130, 37, 37, + 37, 37, 37, 37, 56, 56, 106, 968, 951, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + + 56, 56, 56, 62, 115, 935, 134, 62, 62, 62, + 62, 137, 669, 134, 669, 137, 158, 137, 116, 125, + 125, 253, 116, 115, 116, 62, 138, 115, 134, 62, + 127, 121, 62, 121, 121, 121, 121, 121, 121, 158, + 934, 125, 138, 253, 116, 127, 139, 932, 62, 138, + 139, 62, 139, 127, 62, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 911, 116, 159, 136, 65, + 65, 65, 65, 65, 65, 458, 157, 910, 458, 99, + 157, 141, 157, 99, 99, 99, 99, 136, 129, 141, + 159, 136, 65, 65, 65, 65, 65, 65, 68, 68, + + 68, 68, 68, 160, 129, 99, 129, 642, 129, 68, + 160, 129, 141, 99, 147, 68, 68, 68, 68, 68, + 68, 68, 68, 68, 68, 160, 129, 99, 129, 68, + 68, 68, 68, 68, 68, 99, 147, 147, 360, 103, + 908, 155, 360, 103, 103, 103, 103, 232, 452, 155, + 642, 68, 68, 68, 68, 68, 68, 68, 80, 80, + 80, 80, 80, 155, 80, 103, 123, 907, 123, 123, + 123, 123, 123, 123, 103, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 473, 178, 103, 473, 80, + 80, 80, 80, 80, 80, 452, 103, 135, 901, 232, + + 140, 135, 143, 135, 140, 140, 140, 140, 146, 178, + 143, 80, 80, 80, 80, 80, 80, 80, 83, 83, + 83, 83, 83, 135, 161, 179, 146, 680, 161, 680, + 161, 146, 899, 143, 83, 83, 83, 83, 83, 83, + 83, 83, 83, 83, 83, 135, 145, 187, 179, 83, + 83, 83, 83, 83, 83, 156, 145, 177, 895, 893, + 156, 177, 156, 177, 164, 703, 890, 703, 145, 164, + 187, 83, 83, 83, 83, 83, 83, 83, 86, 86, + 86, 86, 86, 183, 156, 189, 186, 183, 86, 183, + 186, 186, 186, 186, 300, 86, 86, 86, 86, 86, + + 86, 86, 86, 86, 86, 168, 156, 814, 189, 86, + 86, 86, 86, 86, 86, 194, 164, 198, 199, 194, + 168, 194, 101, 199, 200, 199, 101, 101, 101, 101, + 193, 86, 86, 86, 86, 86, 86, 86, 90, 198, + 198, 194, 90, 90, 90, 90, 300, 200, 101, 193, + 814, 128, 101, 193, 201, 889, 128, 168, 90, 90, + 90, 128, 90, 194, 90, 204, 171, 90, 90, 90, + 101, 872, 362, 128, 101, 171, 362, 201, 128, 205, + 214, 90, 90, 90, 90, 709, 90, 709, 204, 90, + 90, 90, 92, 92, 92, 92, 92, 92, 92, 92, + + 92, 92, 205, 214, 216, 257, 92, 92, 92, 92, + 92, 92, 871, 162, 162, 162, 162, 162, 171, 203, + 209, 869, 258, 203, 209, 203, 209, 216, 257, 92, + 92, 92, 92, 92, 92, 94, 94, 94, 94, 94, + 94, 94, 94, 94, 94, 258, 639, 868, 865, 94, + 94, 94, 94, 94, 94, 559, 162, 260, 107, 559, + 863, 94, 107, 110, 107, 862, 162, 110, 110, 110, + 110, 260, 94, 94, 94, 94, 94, 94, 162, 107, + 260, 114, 265, 94, 107, 114, 114, 114, 114, 110, + 212, 212, 213, 639, 110, 254, 213, 213, 213, 213, + + 265, 286, 107, 254, 182, 265, 107, 114, 182, 286, + 182, 110, 212, 114, 220, 114, 110, 254, 220, 220, + 220, 220, 229, 286, 182, 857, 229, 228, 229, 114, + 182, 228, 561, 228, 855, 114, 561, 114, 118, 118, + 118, 118, 118, 303, 234, 304, 852, 182, 234, 234, + 234, 234, 182, 228, 316, 118, 118, 118, 118, 118, + 118, 118, 118, 118, 118, 714, 303, 714, 304, 118, + 118, 118, 118, 118, 118, 228, 256, 316, 259, 255, + 256, 416, 256, 208, 255, 259, 255, 208, 208, 208, + 208, 118, 118, 118, 118, 118, 118, 118, 120, 851, + + 259, 120, 120, 120, 120, 120, 120, 120, 255, 208, + 261, 318, 333, 120, 261, 120, 261, 236, 208, 236, + 236, 236, 236, 236, 236, 242, 242, 242, 242, 242, + 255, 208, 849, 416, 318, 333, 120, 483, 120, 122, + 208, 334, 122, 122, 122, 122, 122, 122, 122, 305, + 305, 305, 305, 305, 122, 238, 122, 238, 238, 238, + 238, 238, 238, 262, 334, 275, 283, 262, 242, 262, + 267, 283, 242, 283, 267, 275, 267, 122, 242, 122, + 149, 149, 149, 149, 149, 848, 281, 266, 831, 483, + 242, 266, 281, 266, 242, 338, 830, 149, 149, 149, + + 149, 149, 149, 149, 149, 149, 149, 266, 281, 339, + 343, 149, 149, 149, 149, 149, 149, 275, 338, 249, + 249, 249, 249, 249, 726, 282, 726, 795, 344, 282, + 266, 282, 339, 343, 149, 149, 149, 149, 149, 149, + 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, + 249, 344, 794, 282, 152, 152, 152, 152, 152, 152, + 270, 752, 784, 752, 270, 270, 270, 270, 957, 783, + 287, 351, 249, 249, 287, 282, 287, 152, 152, 152, + 152, 152, 152, 165, 165, 165, 165, 165, 165, 165, + 165, 165, 165, 754, 351, 292, 417, 165, 165, 165, + + 165, 165, 165, 753, 272, 272, 272, 272, 272, 290, + 302, 957, 328, 290, 302, 290, 302, 328, 417, 328, + 165, 165, 165, 165, 165, 165, 169, 169, 169, 169, + 169, 169, 169, 169, 169, 169, 291, 292, 353, 272, + 169, 169, 169, 169, 169, 169, 291, 292, 750, 792, + 503, 792, 241, 241, 241, 241, 241, 272, 291, 292, + 779, 353, 272, 169, 169, 169, 169, 169, 169, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 401, + 401, 748, 779, 172, 172, 172, 172, 172, 172, 306, + 306, 306, 306, 306, 811, 241, 244, 244, 244, 244, + + 244, 401, 503, 241, 405, 241, 172, 172, 172, 172, + 172, 172, 176, 176, 176, 176, 176, 241, 307, 405, + 747, 332, 307, 741, 307, 241, 332, 405, 332, 176, + 176, 176, 176, 176, 176, 176, 176, 176, 176, 244, + 244, 811, 392, 176, 176, 176, 176, 176, 176, 244, + 739, 954, 392, 273, 273, 273, 273, 273, 734, 311, + 412, 244, 244, 311, 392, 311, 176, 176, 176, 176, + 176, 176, 233, 233, 233, 233, 233, 233, 233, 233, + 233, 233, 412, 412, 273, 462, 233, 233, 233, 233, + 233, 233, 240, 240, 240, 240, 240, 801, 954, 801, + + 505, 271, 271, 271, 271, 271, 273, 273, 462, 233, + 233, 233, 233, 233, 233, 235, 526, 533, 235, 235, + 235, 235, 235, 235, 235, 325, 732, 240, 463, 325, + 235, 325, 235, 728, 727, 240, 468, 243, 243, 243, + 243, 243, 315, 468, 271, 240, 315, 315, 315, 315, + 240, 463, 505, 235, 271, 235, 237, 240, 468, 237, + 237, 237, 237, 237, 237, 237, 271, 724, 526, 533, + 403, 237, 478, 237, 245, 245, 245, 245, 245, 403, + 243, 246, 246, 246, 246, 246, 402, 337, 705, 243, + 243, 337, 403, 337, 237, 478, 237, 239, 239, 239, + + 239, 239, 243, 704, 251, 251, 251, 251, 251, 346, + 479, 243, 245, 346, 239, 346, 554, 245, 239, 239, + 239, 239, 324, 698, 246, 342, 324, 245, 324, 246, + 342, 697, 342, 479, 246, 245, 556, 402, 402, 245, + 239, 250, 250, 250, 250, 250, 246, 251, 324, 350, + 239, 246, 251, 350, 350, 350, 350, 251, 759, 402, + 759, 759, 239, 247, 247, 247, 247, 247, 554, 251, + 324, 358, 250, 694, 251, 358, 358, 358, 358, 250, + 288, 288, 288, 288, 288, 807, 693, 807, 556, 274, + 274, 274, 274, 274, 250, 250, 247, 277, 277, 277, + + 277, 277, 250, 277, 366, 566, 247, 277, 366, 277, + 366, 369, 247, 397, 247, 369, 247, 369, 740, 247, + 688, 397, 740, 397, 274, 374, 288, 686, 247, 374, + 374, 374, 374, 288, 247, 397, 247, 248, 248, 248, + 248, 248, 274, 397, 838, 399, 838, 274, 288, 424, + 277, 399, 742, 424, 248, 424, 742, 566, 248, 248, + 248, 248, 320, 320, 320, 320, 320, 399, 248, 280, + 280, 280, 280, 280, 321, 321, 321, 321, 321, 322, + 322, 322, 322, 322, 495, 280, 583, 583, 583, 583, + 248, 248, 276, 276, 276, 276, 276, 335, 335, 335, + + 335, 335, 336, 336, 336, 336, 336, 495, 280, 276, + 276, 276, 276, 276, 276, 276, 276, 276, 276, 682, + 681, 423, 280, 276, 276, 276, 276, 276, 276, 423, + 678, 280, 289, 289, 289, 289, 289, 340, 340, 340, + 340, 340, 427, 423, 676, 675, 276, 276, 276, 276, + 276, 276, 278, 278, 278, 278, 278, 278, 278, 278, + 278, 278, 656, 492, 427, 427, 278, 278, 278, 278, + 278, 278, 341, 341, 341, 341, 341, 655, 289, 492, + 331, 331, 331, 331, 331, 289, 492, 630, 629, 278, + 278, 278, 278, 278, 278, 355, 355, 355, 355, 355, + + 289, 293, 293, 293, 293, 293, 293, 293, 293, 293, + 293, 331, 621, 447, 497, 293, 293, 293, 293, 293, + 293, 620, 577, 349, 349, 349, 349, 349, 356, 356, + 356, 356, 356, 331, 331, 447, 447, 497, 293, 293, + 293, 293, 293, 293, 296, 296, 296, 296, 296, 296, + 296, 296, 296, 296, 576, 517, 394, 521, 296, 296, + 296, 296, 296, 296, 357, 357, 357, 357, 357, 363, + 363, 363, 363, 363, 394, 349, 349, 517, 517, 394, + 521, 296, 296, 296, 296, 296, 296, 297, 297, 297, + 297, 297, 297, 297, 297, 297, 297, 349, 847, 573, + + 847, 297, 297, 297, 297, 297, 297, 364, 364, 364, + 364, 364, 572, 314, 314, 314, 314, 314, 365, 365, + 365, 365, 365, 393, 297, 297, 297, 297, 297, 297, + 301, 301, 301, 301, 301, 301, 301, 301, 301, 301, + 761, 522, 761, 761, 301, 301, 301, 301, 301, 301, + 314, 372, 372, 372, 372, 372, 376, 569, 376, 376, + 376, 376, 376, 376, 522, 393, 314, 301, 301, 301, + 301, 301, 301, 314, 378, 393, 378, 378, 378, 378, + 378, 378, 409, 568, 459, 372, 567, 393, 459, 565, + 389, 389, 389, 389, 389, 429, 560, 418, 409, 429, + + 409, 429, 409, 418, 372, 409, 459, 558, 372, 373, + 373, 373, 373, 373, 373, 373, 373, 373, 373, 418, + 409, 389, 409, 373, 373, 373, 373, 373, 373, 380, + 380, 380, 380, 380, 867, 568, 867, 555, 381, 381, + 381, 381, 381, 389, 389, 553, 373, 373, 373, 373, + 373, 373, 375, 549, 547, 375, 375, 375, 375, 375, + 375, 375, 900, 420, 380, 528, 900, 375, 420, 375, + 420, 395, 380, 400, 382, 382, 382, 382, 382, 411, + 398, 381, 380, 411, 411, 411, 411, 380, 528, 381, + 375, 381, 375, 377, 380, 541, 377, 377, 377, 377, + + 377, 377, 377, 381, 662, 395, 529, 884, 377, 884, + 377, 381, 662, 406, 419, 540, 532, 382, 419, 400, + 419, 382, 398, 395, 404, 400, 662, 382, 395, 529, + 398, 377, 398, 377, 379, 379, 379, 379, 379, 382, + 570, 400, 419, 382, 398, 383, 383, 383, 383, 383, + 406, 379, 398, 543, 543, 379, 379, 379, 379, 384, + 384, 384, 384, 384, 419, 406, 525, 404, 385, 385, + 385, 385, 385, 406, 679, 543, 404, 379, 464, 386, + 386, 386, 386, 386, 464, 515, 407, 379, 383, 404, + 413, 407, 570, 906, 514, 906, 407, 383, 383, 379, + + 464, 432, 384, 384, 493, 432, 385, 432, 407, 544, + 383, 385, 384, 407, 391, 391, 391, 391, 391, 383, + 413, 385, 386, 465, 384, 384, 679, 386, 465, 385, + 465, 511, 386, 385, 390, 390, 390, 390, 390, 510, + 493, 414, 413, 413, 386, 414, 414, 414, 414, 386, + 387, 387, 387, 387, 387, 535, 493, 391, 414, 410, + 544, 544, 391, 493, 435, 390, 408, 391, 435, 396, + 435, 536, 390, 396, 396, 396, 396, 414, 535, 391, + 725, 414, 544, 387, 391, 508, 436, 390, 390, 396, + 436, 410, 436, 387, 536, 390, 546, 507, 428, 387, + + 443, 387, 428, 387, 428, 396, 387, 410, 408, 410, + 443, 410, 396, 408, 410, 387, 504, 502, 408, 546, + 428, 387, 443, 387, 388, 388, 388, 388, 388, 410, + 408, 410, 725, 498, 496, 408, 425, 425, 425, 425, + 425, 388, 446, 428, 439, 388, 388, 388, 388, 426, + 426, 426, 426, 426, 444, 388, 433, 433, 433, 433, + 433, 490, 439, 448, 440, 548, 489, 439, 440, 445, + 440, 441, 441, 441, 441, 441, 446, 388, 388, 442, + 486, 485, 425, 442, 442, 442, 442, 445, 548, 425, + 433, 482, 445, 448, 446, 426, 444, 461, 472, 446, + + 513, 461, 426, 461, 425, 513, 444, 513, 441, 433, + 942, 457, 942, 433, 460, 448, 448, 426, 444, 460, + 438, 460, 467, 477, 441, 437, 467, 477, 467, 477, + 518, 441, 449, 449, 449, 449, 449, 449, 449, 449, + 449, 449, 605, 469, 460, 609, 449, 449, 449, 449, + 449, 449, 450, 450, 450, 450, 450, 484, 450, 431, + 518, 484, 450, 484, 450, 605, 460, 430, 609, 449, + 449, 449, 449, 449, 449, 451, 451, 451, 451, 451, + 488, 451, 518, 518, 488, 451, 488, 451, 469, 480, + 480, 480, 480, 480, 422, 469, 481, 481, 481, 481, + + 481, 421, 494, 610, 670, 450, 494, 494, 494, 494, + 469, 487, 487, 487, 487, 487, 499, 499, 499, 499, + 499, 500, 500, 500, 500, 500, 610, 670, 451, 455, + 455, 455, 455, 455, 455, 455, 455, 455, 455, 455, + 455, 455, 455, 455, 455, 455, 455, 455, 455, 455, + 455, 455, 455, 455, 455, 455, 455, 455, 455, 455, + 455, 455, 455, 455, 455, 455, 455, 455, 455, 455, + 455, 455, 455, 455, 455, 455, 455, 455, 455, 455, + 455, 455, 455, 455, 455, 455, 455, 455, 455, 455, + 455, 455, 455, 455, 455, 455, 455, 455, 455, 455, + + 455, 455, 455, 455, 455, 455, 455, 455, 456, 456, + 456, 456, 456, 415, 371, 466, 466, 466, 466, 466, + 491, 491, 491, 491, 491, 456, 456, 456, 456, 456, + 456, 456, 456, 456, 456, 602, 602, 602, 602, 456, + 456, 456, 456, 456, 456, 501, 501, 501, 501, 501, + 506, 519, 370, 368, 506, 509, 506, 491, 367, 509, + 466, 509, 456, 456, 456, 456, 456, 456, 466, 361, + 519, 520, 359, 491, 519, 527, 520, 354, 520, 527, + 491, 527, 466, 471, 471, 471, 471, 471, 352, 471, + 523, 523, 523, 523, 523, 512, 512, 512, 512, 512, + + 471, 471, 471, 471, 471, 471, 471, 471, 471, 471, + 604, 604, 604, 604, 471, 471, 471, 471, 471, 471, + 524, 524, 524, 524, 524, 948, 348, 948, 615, 512, + 530, 530, 530, 530, 530, 615, 471, 471, 471, 471, + 471, 471, 471, 474, 474, 474, 474, 474, 512, 534, + 615, 606, 512, 347, 534, 606, 534, 474, 345, 330, + 474, 474, 474, 474, 474, 474, 474, 474, 474, 474, + 952, 329, 952, 606, 474, 474, 474, 474, 474, 474, + 474, 474, 474, 474, 474, 474, 474, 474, 474, 474, + 474, 474, 474, 474, 474, 474, 474, 474, 474, 474, + + 474, 474, 474, 474, 474, 474, 474, 474, 474, 474, + 474, 474, 474, 474, 474, 474, 474, 474, 327, 622, + 474, 475, 475, 475, 475, 475, 475, 475, 475, 475, + 475, 636, 636, 636, 636, 475, 475, 475, 475, 475, + 475, 622, 622, 516, 516, 516, 516, 516, 531, 531, + 531, 531, 531, 537, 537, 537, 537, 537, 475, 475, + 475, 475, 475, 475, 476, 476, 476, 476, 476, 476, + 476, 476, 476, 476, 516, 975, 634, 975, 476, 476, + 476, 476, 476, 476, 538, 538, 538, 538, 538, 539, + 326, 652, 634, 539, 571, 539, 516, 516, 571, 634, + + 571, 476, 476, 476, 476, 476, 476, 542, 542, 542, + 542, 542, 545, 652, 652, 671, 545, 545, 545, 545, + 550, 550, 550, 550, 550, 551, 551, 551, 551, 551, + 552, 552, 552, 552, 552, 557, 580, 751, 671, 557, + 557, 557, 557, 562, 562, 562, 562, 562, 563, 563, + 563, 563, 563, 564, 564, 564, 564, 564, 323, 542, + 542, 574, 574, 574, 574, 574, 575, 579, 319, 580, + 575, 317, 575, 578, 578, 578, 578, 578, 608, 902, + 612, 542, 608, 902, 608, 612, 579, 612, 580, 751, + 579, 585, 580, 585, 585, 585, 585, 585, 585, 587, + + 313, 587, 587, 587, 587, 587, 587, 578, 614, 618, + 815, 654, 614, 618, 614, 618, 654, 312, 654, 815, + 598, 598, 598, 598, 598, 986, 578, 986, 310, 309, + 578, 582, 582, 582, 582, 582, 582, 582, 582, 582, + 582, 598, 598, 598, 598, 582, 582, 582, 582, 582, + 582, 584, 584, 584, 584, 584, 584, 584, 601, 601, + 601, 601, 815, 584, 308, 584, 616, 298, 582, 582, + 582, 582, 582, 582, 601, 586, 586, 586, 586, 586, + 586, 586, 684, 684, 684, 684, 584, 586, 584, 586, + 601, 603, 603, 603, 603, 607, 685, 601, 294, 611, + + 607, 661, 607, 619, 603, 611, 661, 619, 661, 619, + 586, 616, 586, 588, 588, 588, 588, 588, 616, 685, + 623, 611, 285, 603, 623, 607, 623, 603, 672, 672, + 672, 672, 672, 616, 588, 588, 588, 588, 626, 613, + 613, 613, 613, 613, 625, 657, 687, 607, 625, 657, + 625, 624, 624, 624, 624, 624, 588, 626, 628, 631, + 627, 626, 628, 631, 628, 631, 588, 657, 284, 687, + 632, 632, 632, 632, 632, 635, 279, 269, 588, 589, + 589, 589, 589, 589, 613, 633, 710, 711, 268, 633, + 624, 633, 613, 627, 264, 638, 638, 638, 638, 638, + + 589, 589, 589, 589, 624, 632, 613, 638, 263, 710, + 711, 635, 627, 624, 589, 231, 627, 640, 640, 640, + 640, 640, 1022, 632, 1022, 812, 653, 635, 632, 640, + 812, 653, 589, 653, 635, 230, 866, 589, 590, 590, + 590, 590, 590, 644, 644, 644, 644, 644, 658, 644, + 227, 226, 658, 644, 658, 644, 715, 225, 653, 590, + 590, 590, 590, 223, 645, 645, 645, 645, 645, 674, + 645, 221, 716, 674, 645, 674, 645, 812, 219, 715, + 653, 590, 646, 646, 646, 646, 646, 663, 866, 590, + 1026, 590, 1026, 218, 646, 716, 644, 217, 647, 647, + + 647, 647, 647, 590, 647, 1027, 215, 1027, 647, 720, + 647, 590, 591, 591, 591, 591, 591, 645, 648, 648, + 648, 648, 648, 692, 648, 211, 210, 692, 648, 692, + 648, 663, 720, 591, 591, 591, 591, 207, 206, 663, + 659, 659, 659, 659, 659, 660, 660, 660, 660, 660, + 1047, 647, 1047, 663, 673, 673, 673, 673, 673, 591, + 677, 677, 677, 677, 677, 591, 689, 689, 689, 689, + 689, 648, 700, 683, 683, 683, 683, 683, 707, 721, + 202, 591, 592, 592, 592, 592, 592, 659, 731, 733, + 696, 700, 660, 659, 696, 700, 696, 707, 660, 701, + + 197, 707, 721, 592, 592, 592, 592, 196, 192, 659, + 683, 731, 733, 191, 660, 690, 690, 690, 690, 690, + 691, 691, 691, 691, 691, 190, 683, 695, 695, 695, + 695, 695, 701, 683, 592, 592, 699, 699, 699, 699, + 699, 706, 706, 706, 706, 706, 712, 712, 712, 712, + 712, 701, 1059, 188, 1059, 701, 592, 593, 593, 593, + 593, 593, 713, 713, 713, 713, 713, 843, 185, 777, + 699, 184, 706, 717, 717, 717, 717, 717, 593, 593, + 593, 593, 718, 718, 718, 718, 718, 181, 777, 699, + 843, 719, 777, 699, 706, 706, 719, 1060, 719, 1060, + + 790, 593, 722, 722, 722, 722, 722, 958, 790, 180, + 593, 723, 723, 723, 723, 723, 958, 729, 729, 729, + 729, 729, 790, 593, 594, 594, 594, 594, 594, 730, + 730, 730, 730, 735, 735, 735, 735, 735, 736, 736, + 736, 736, 736, 175, 173, 594, 594, 594, 594, 737, + 737, 737, 737, 737, 738, 738, 738, 738, 166, 958, + 775, 844, 594, 743, 743, 743, 743, 743, 153, 729, + 729, 744, 744, 744, 744, 744, 775, 594, 745, 745, + 745, 745, 745, 775, 844, 594, 595, 595, 595, 595, + 595, 729, 746, 144, 142, 854, 746, 856, 746, 749, + + 749, 749, 749, 749, 132, 758, 758, 758, 799, 758, + 758, 758, 755, 755, 755, 755, 755, 758, 854, 758, + 856, 131, 799, 762, 762, 762, 762, 762, 782, 595, + 832, 799, 785, 782, 595, 782, 785, 595, 117, 595, + 758, 113, 758, 760, 760, 760, 755, 760, 760, 760, + 112, 595, 832, 832, 785, 760, 595, 760, 791, 595, + 596, 596, 596, 596, 596, 755, 762, 111, 874, 755, + 763, 763, 763, 763, 763, 1077, 762, 1077, 760, 109, + 760, 772, 772, 772, 772, 772, 786, 874, 762, 805, + 786, 874, 786, 596, 108, 104, 764, 764, 764, 764, + + 764, 789, 791, 596, 102, 763, 789, 805, 789, 596, + 791, 596, 805, 596, 98, 97, 596, 765, 765, 765, + 765, 765, 781, 763, 791, 596, 793, 781, 763, 781, + 793, 596, 793, 596, 597, 597, 597, 597, 597, 764, + 93, 827, 766, 766, 766, 766, 766, 764, 91, 764, + 827, 1079, 88, 1079, 781, 597, 597, 597, 597, 81, + 76, 764, 796, 827, 765, 597, 796, 75, 796, 764, + 765, 767, 767, 767, 767, 767, 781, 71, 876, 768, + 768, 768, 768, 768, 800, 877, 765, 597, 597, 599, + 599, 599, 599, 599, 766, 766, 769, 769, 769, 769, + + 769, 876, 57, 771, 771, 771, 771, 771, 877, 55, + 599, 599, 599, 599, 798, 767, 766, 768, 798, 802, + 798, 1075, 800, 802, 767, 802, 806, 599, 44, 804, + 1075, 41, 768, 804, 771, 804, 800, 767, 39, 769, + 768, 829, 599, 1075, 769, 800, 829, 35, 829, 769, + 599, 600, 600, 600, 600, 600, 771, 771, 828, 835, + 806, 769, 828, 835, 828, 835, 769, 787, 787, 787, + 787, 787, 803, 803, 803, 803, 803, 880, 806, 1090, + 1098, 1090, 1098, 806, 773, 773, 773, 773, 773, 809, + 809, 809, 809, 809, 600, 881, 31, 24, 833, 600, + + 880, 809, 600, 833, 600, 833, 774, 774, 774, 774, + 774, 861, 21, 836, 787, 861, 600, 861, 881, 836, + 787, 600, 773, 885, 600, 637, 637, 637, 637, 637, + 637, 637, 637, 637, 637, 836, 787, 773, 886, 637, + 637, 637, 637, 637, 637, 773, 885, 20, 933, 774, + 892, 894, 939, 933, 774, 933, 939, 15, 939, 774, + 946, 886, 637, 637, 637, 637, 637, 637, 641, 641, + 641, 774, 1000, 892, 894, 13, 774, 797, 797, 797, + 797, 797, 946, 946, 641, 641, 641, 641, 641, 641, + 641, 641, 641, 641, 3, 1000, 1001, 1039, 641, 641, + + 641, 641, 641, 641, 1100, 0, 1100, 0, 797, 845, + 845, 845, 845, 845, 846, 846, 846, 846, 846, 1001, + 1039, 641, 641, 641, 641, 641, 641, 643, 643, 643, + 797, 797, 850, 850, 850, 850, 850, 858, 858, 858, + 858, 858, 0, 643, 643, 643, 643, 643, 643, 643, + 643, 643, 643, 0, 1040, 0, 0, 643, 643, 643, + 643, 643, 643, 817, 817, 817, 817, 817, 943, 817, + 0, 937, 943, 817, 943, 817, 937, 1040, 937, 0, + 643, 643, 643, 643, 643, 643, 649, 649, 649, 649, + 649, 940, 649, 0, 0, 945, 649, 940, 649, 945, + + 0, 945, 0, 649, 649, 649, 649, 649, 649, 649, + 649, 649, 649, 940, 0, 0, 817, 649, 649, 649, + 649, 649, 649, 859, 859, 859, 859, 859, 860, 860, + 860, 860, 860, 864, 864, 864, 864, 864, 0, 649, + 649, 649, 649, 649, 649, 649, 650, 650, 650, 650, + 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, + 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, + 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, + 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, + 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, + + 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, + 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, + 650, 650, 650, 650, 650, 651, 651, 651, 651, 651, + 651, 651, 651, 651, 651, 0, 0, 0, 0, 651, + 651, 651, 651, 651, 651, 818, 818, 818, 818, 818, + 949, 818, 0, 970, 949, 818, 949, 818, 970, 0, + 970, 0, 651, 651, 651, 651, 651, 651, 665, 665, + 665, 665, 665, 1045, 665, 878, 878, 878, 878, 878, + 879, 879, 879, 879, 879, 665, 665, 665, 665, 665, + 665, 665, 665, 665, 665, 1045, 1045, 0, 818, 665, + + 665, 665, 665, 665, 665, 820, 820, 820, 820, 820, + 834, 834, 834, 834, 834, 0, 965, 820, 0, 0, + 965, 665, 665, 665, 665, 665, 665, 665, 666, 666, + 666, 666, 666, 666, 666, 666, 666, 666, 965, 0, + 0, 0, 666, 666, 666, 666, 666, 666, 821, 821, + 821, 821, 821, 873, 873, 873, 873, 873, 820, 834, + 821, 0, 0, 834, 0, 666, 666, 666, 666, 666, + 666, 667, 667, 667, 667, 667, 667, 667, 667, 667, + 667, 834, 0, 0, 873, 667, 667, 667, 667, 667, + 667, 0, 822, 822, 822, 822, 822, 0, 822, 967, + + 0, 821, 822, 967, 822, 967, 873, 873, 667, 667, + 667, 667, 667, 667, 668, 668, 668, 668, 668, 668, + 668, 668, 668, 668, 0, 0, 0, 0, 668, 668, + 668, 668, 668, 668, 0, 0, 788, 788, 788, 788, + 788, 837, 0, 0, 0, 822, 882, 882, 882, 882, + 882, 668, 668, 668, 668, 668, 668, 757, 757, 757, + 757, 757, 757, 757, 757, 757, 757, 0, 0, 0, + 0, 757, 757, 757, 757, 757, 757, 947, 823, 823, + 823, 823, 823, 788, 973, 0, 0, 837, 973, 788, + 823, 0, 0, 837, 757, 757, 757, 757, 757, 757, + + 770, 770, 770, 770, 770, 788, 973, 947, 0, 837, + 839, 839, 839, 839, 839, 0, 839, 883, 883, 883, + 883, 883, 887, 887, 887, 887, 887, 0, 1020, 947, + 947, 823, 1020, 770, 888, 888, 888, 888, 888, 896, + 896, 896, 896, 896, 897, 897, 897, 897, 897, 770, + 1020, 770, 0, 770, 0, 0, 770, 898, 898, 898, + 898, 898, 0, 839, 903, 903, 903, 903, 903, 0, + 0, 770, 0, 770, 808, 808, 808, 808, 808, 808, + 808, 808, 808, 808, 0, 0, 0, 0, 808, 808, + 808, 808, 808, 808, 853, 853, 853, 853, 853, 904, + + 904, 904, 904, 904, 905, 905, 905, 905, 905, 0, + 0, 808, 808, 808, 808, 808, 808, 813, 813, 813, + 813, 813, 0, 813, 909, 909, 909, 909, 909, 0, + 0, 853, 0, 0, 813, 813, 813, 813, 813, 813, + 813, 813, 813, 813, 0, 0, 0, 853, 813, 813, + 813, 813, 813, 813, 853, 944, 944, 944, 944, 944, + 953, 953, 953, 953, 953, 981, 981, 981, 981, 981, + 813, 813, 813, 813, 813, 813, 813, 816, 816, 816, + 816, 816, 1041, 972, 1017, 0, 1041, 816, 972, 1017, + 972, 1017, 0, 0, 816, 816, 816, 816, 816, 816, + + 816, 816, 816, 816, 1041, 0, 0, 0, 816, 816, + 816, 816, 816, 816, 870, 870, 870, 870, 870, 0, + 891, 891, 891, 891, 891, 989, 989, 989, 989, 989, + 816, 816, 816, 816, 816, 816, 816, 819, 819, 819, + 819, 819, 819, 819, 819, 819, 819, 0, 870, 0, + 0, 819, 819, 819, 819, 819, 819, 914, 914, 914, + 914, 914, 1023, 0, 0, 955, 1023, 870, 1023, 0, + 955, 870, 891, 891, 819, 819, 819, 819, 819, 819, + 824, 824, 824, 824, 824, 1002, 1002, 1002, 1002, 1002, + 0, 0, 0, 0, 891, 0, 0, 824, 824, 824, + + 824, 824, 824, 824, 824, 824, 824, 0, 0, 0, + 914, 824, 824, 824, 824, 824, 824, 955, 1003, 1003, + 1003, 1003, 1003, 1013, 1013, 1013, 1013, 1013, 1018, 1018, + 1018, 1018, 1018, 0, 824, 824, 824, 824, 824, 824, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, + + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 826, + 826, 826, 826, 826, 826, 826, 826, 826, 826, 0, + 0, 0, 0, 826, 826, 826, 826, 826, 826, 912, + 912, 912, 912, 912, 924, 924, 924, 924, 924, 1038, + 1088, 0, 0, 1038, 1088, 1038, 826, 826, 826, 826, + 826, 826, 840, 840, 840, 840, 840, 840, 840, 840, + 840, 840, 1088, 912, 0, 924, 840, 840, 840, 840, + 840, 840, 916, 916, 916, 916, 916, 0, 0, 966, + 0, 0, 912, 966, 0, 966, 912, 924, 924, 840, + + 840, 840, 840, 840, 840, 841, 841, 841, 841, 841, + 841, 841, 841, 841, 841, 0, 966, 916, 0, 841, + 841, 841, 841, 841, 841, 0, 915, 915, 915, 915, + 915, 0, 1019, 0, 0, 916, 0, 1019, 966, 1019, + 916, 0, 841, 841, 841, 841, 841, 841, 842, 842, + 842, 842, 842, 842, 842, 842, 842, 842, 0, 0, + 0, 0, 842, 842, 842, 842, 842, 842, 0, 915, + 917, 917, 917, 917, 917, 0, 0, 974, 941, 915, + 918, 918, 918, 918, 918, 842, 842, 842, 842, 842, + 842, 915, 919, 919, 919, 919, 919, 920, 920, 920, + + 920, 920, 0, 0, 921, 921, 921, 921, 921, 1046, + 0, 0, 0, 917, 0, 0, 922, 922, 922, 922, + 922, 917, 1042, 917, 941, 974, 1042, 918, 1042, 974, + 941, 1044, 0, 918, 0, 917, 1044, 0, 1044, 1046, + 0, 920, 921, 917, 919, 919, 941, 974, 0, 918, + 920, 925, 925, 925, 925, 925, 0, 921, 0, 922, + 0, 1046, 1046, 920, 922, 921, 919, 0, 0, 922, + 926, 926, 926, 926, 926, 0, 927, 927, 927, 927, + 927, 922, 1076, 0, 0, 0, 922, 923, 923, 923, + 923, 923, 0, 0, 0, 928, 928, 928, 928, 928, + + 1056, 0, 0, 0, 925, 1056, 0, 1056, 926, 929, + 929, 929, 929, 929, 931, 931, 931, 931, 931, 927, + 923, 1067, 0, 926, 927, 1076, 1067, 0, 1067, 927, + 0, 926, 928, 1071, 1076, 0, 923, 1071, 923, 1071, + 923, 927, 0, 923, 0, 0, 927, 1076, 928, 930, + 930, 930, 930, 930, 0, 928, 0, 1074, 923, 1085, + 923, 1074, 929, 1074, 1085, 0, 1085, 931, 938, 938, + 938, 938, 938, 961, 961, 961, 961, 961, 963, 963, + 963, 963, 963, 930, 0, 961, 0, 0, 0, 0, + 963, 1051, 1051, 1051, 1051, 1051, 0, 971, 971, 971, + + 971, 971, 930, 0, 0, 0, 930, 976, 976, 976, + 976, 976, 977, 977, 977, 977, 977, 938, 1087, 0, + 1037, 938, 1087, 1021, 1087, 1037, 961, 1037, 971, 0, + 0, 963, 1086, 1086, 1086, 1086, 1086, 0, 0, 938, + 956, 956, 956, 956, 956, 956, 956, 956, 956, 956, + 971, 971, 1037, 0, 956, 956, 956, 956, 956, 956, + 976, 0, 0, 0, 0, 977, 978, 978, 978, 978, + 978, 1021, 0, 0, 1037, 1021, 0, 956, 956, 956, + 956, 956, 956, 959, 959, 959, 959, 959, 959, 959, + 959, 959, 959, 1021, 0, 0, 0, 959, 959, 959, + + 959, 959, 959, 983, 983, 983, 983, 983, 984, 984, + 984, 984, 984, 1094, 1094, 1094, 1094, 1094, 0, 978, + 959, 959, 959, 959, 959, 959, 960, 960, 960, 960, + 960, 960, 960, 960, 960, 960, 0, 0, 0, 0, + 960, 960, 960, 960, 960, 960, 1095, 0, 0, 0, + 0, 1095, 0, 1095, 0, 0, 983, 0, 0, 0, + 0, 984, 0, 960, 960, 960, 960, 960, 960, 962, + 962, 962, 962, 962, 962, 962, 962, 962, 962, 962, + 962, 962, 962, 962, 962, 962, 962, 962, 962, 962, + 962, 962, 962, 962, 962, 962, 962, 962, 962, 962, + + 962, 962, 962, 962, 962, 962, 962, 962, 962, 962, + 962, 962, 962, 962, 962, 962, 962, 962, 962, 962, + 962, 962, 962, 962, 962, 962, 962, 962, 962, 962, + 962, 962, 962, 962, 962, 962, 962, 962, 962, 962, + 962, 962, 962, 962, 962, 962, 962, 962, 964, 964, + 964, 964, 964, 964, 964, 964, 964, 964, 0, 0, + 0, 0, 964, 964, 964, 964, 964, 964, 982, 982, + 982, 982, 982, 985, 985, 985, 985, 985, 992, 992, + 992, 992, 992, 0, 0, 964, 964, 964, 964, 964, + 964, 993, 993, 993, 993, 993, 994, 994, 994, 994, + + 994, 0, 0, 0, 0, 982, 995, 995, 995, 995, + 995, 0, 992, 996, 996, 996, 996, 996, 0, 0, + 0, 982, 993, 0, 0, 0, 985, 0, 982, 0, + 994, 992, 0, 0, 0, 992, 997, 997, 997, 997, + 997, 0, 0, 0, 993, 993, 0, 0, 0, 994, + 0, 0, 0, 994, 998, 998, 998, 998, 998, 995, + 999, 999, 999, 999, 999, 0, 996, 1004, 1004, 1004, + 1004, 1004, 1005, 1005, 1005, 1005, 1005, 1006, 1006, 1006, + 1006, 1006, 1007, 1007, 1007, 1007, 1007, 0, 1089, 997, + 1008, 1008, 1008, 1008, 1008, 1009, 1009, 1009, 1009, 1009, + + 1010, 1010, 1010, 1010, 1010, 0, 0, 998, 1014, 1014, + 1014, 1014, 1014, 999, 1015, 1015, 1015, 1015, 1015, 1004, + 1004, 0, 0, 0, 0, 1005, 0, 0, 0, 0, + 1006, 0, 0, 0, 0, 1007, 1089, 0, 0, 0, + 1089, 1004, 1014, 1008, 0, 0, 0, 0, 1009, 0, + 0, 0, 0, 1010, 0, 0, 0, 0, 1089, 0, + 0, 1014, 0, 0, 0, 1014, 0, 1015, 1028, 1028, + 1028, 1028, 1028, 1028, 1028, 1028, 1028, 1028, 0, 0, + 0, 0, 1028, 1028, 1028, 1028, 1028, 1028, 1031, 1031, + 1031, 1031, 1031, 1048, 1048, 1048, 1048, 1048, 0, 0, + + 1031, 0, 0, 0, 0, 1028, 1028, 1028, 1028, 1028, + 1028, 1029, 1029, 1029, 1029, 1029, 1029, 1029, 1029, 1029, + 1029, 0, 0, 0, 0, 1029, 1029, 1029, 1029, 1029, + 1029, 1032, 1032, 1032, 1032, 1032, 1043, 1043, 1043, 1043, + 1043, 1031, 0, 1032, 0, 0, 1048, 0, 1029, 1029, + 1029, 1029, 1029, 1029, 1030, 1030, 1030, 1030, 1030, 1030, + 1030, 1030, 1030, 1030, 0, 0, 0, 0, 1030, 1030, + 1030, 1030, 1030, 1030, 1053, 1053, 1053, 1053, 1053, 0, + 1043, 0, 0, 0, 1032, 0, 0, 0, 0, 1043, + 0, 1030, 1030, 1030, 1030, 1030, 1030, 1033, 1033, 1033, + + 1033, 1033, 1043, 0, 0, 0, 0, 0, 0, 1033, + 0, 0, 0, 0, 1033, 1033, 1033, 1033, 1033, 1033, + 1033, 1033, 1033, 1033, 0, 0, 0, 1053, 1033, 1033, + 1033, 1033, 1033, 1033, 1052, 1052, 1052, 1052, 1052, 1054, + 1054, 1054, 1054, 1054, 0, 0, 0, 0, 0, 0, + 1033, 1033, 1033, 1033, 1033, 1033, 1033, 1035, 1035, 1035, + 1035, 1035, 1035, 1035, 1035, 1035, 1035, 0, 1052, 0, + 0, 1035, 1035, 1035, 1035, 1035, 1035, 1055, 1055, 1055, + 1055, 1055, 0, 0, 0, 0, 0, 1052, 0, 0, + 0, 1052, 1054, 0, 1035, 1035, 1035, 1035, 1035, 1035, + + 1064, 1064, 1064, 1064, 1064, 0, 0, 0, 0, 0, + 0, 0, 1064, 1072, 1072, 1072, 1072, 1072, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1055, 1061, 1061, 1061, 1061, 1061, 1061, 1061, 1061, 1061, + 1061, 0, 0, 0, 0, 1061, 1061, 1061, 1061, 1061, + 1061, 0, 0, 1064, 0, 0, 1073, 1073, 1073, 1073, + 1073, 0, 1072, 0, 0, 0, 1072, 0, 1061, 1061, + 1061, 1061, 1061, 1061, 1062, 1062, 1062, 1062, 1062, 1062, + 1062, 1062, 1062, 1062, 1072, 0, 0, 0, 1062, 1062, + 1062, 1062, 1062, 1062, 1078, 1078, 1078, 1078, 1078, 0, + + 0, 0, 0, 0, 0, 1073, 0, 0, 0, 1073, + 0, 1062, 1062, 1062, 1062, 1062, 1062, 1063, 1063, 1063, + 1063, 1063, 1063, 1063, 1063, 1063, 1063, 1073, 0, 0, + 0, 1063, 1063, 1063, 1063, 1063, 1063, 1101, 1101, 1101, + 1101, 1101, 0, 0, 0, 0, 0, 1078, 0, 1101, + 0, 0, 0, 0, 1063, 1063, 1063, 1063, 1063, 1063, + 1065, 1065, 1065, 1065, 1065, 1065, 1065, 1065, 1065, 1065, + 0, 0, 0, 0, 1065, 1065, 1065, 1065, 1065, 1065, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1101, 0, 0, 0, 0, 0, 0, 1065, 1065, 1065, + + 1065, 1065, 1065, 1066, 1066, 1066, 1066, 1066, 1066, 1066, + 1066, 1066, 1066, 0, 0, 0, 0, 1066, 1066, 1066, + 1066, 1066, 1066, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1066, 1066, 1066, 1066, 1066, 1066, 1080, 1080, 1080, 1080, + 1080, 1080, 1080, 1080, 1080, 1080, 0, 0, 0, 0, + 1080, 1080, 1080, 1080, 1080, 1080, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1080, 1080, 1080, 1080, 1080, 1080, 1081, + 1081, 1081, 1081, 1081, 1081, 1081, 1081, 1081, 1081, 0, + + 0, 0, 0, 1081, 1081, 1081, 1081, 1081, 1081, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1081, 1081, 1081, 1081, + 1081, 1081, 1082, 1082, 1082, 1082, 1082, 1082, 1082, 1082, + 1082, 1082, 0, 0, 0, 0, 1082, 1082, 1082, 1082, + 1082, 1082, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, + 1082, 1082, 1082, 1082, 1082, 1083, 1083, 1083, 1083, 1083, + 1083, 1083, 1083, 1083, 1083, 0, 0, 0, 0, 1083, + 1083, 1083, 1083, 1083, 1083, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1083, 1083, 1083, 1083, 1083, 1083, 1084, 1084, + 1084, 1084, 1084, 1084, 1084, 1084, 1084, 1084, 0, 0, + 0, 0, 1084, 1084, 1084, 1084, 1084, 1084, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1084, 1084, 1084, 1084, 1084, + 1084, 1091, 1091, 1091, 1091, 1091, 1091, 1091, 1091, 1091, + 1091, 0, 0, 0, 0, 1091, 1091, 1091, 1091, 1091, + 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1091, 1091, + + 1091, 1091, 1091, 1091, 1092, 1092, 1092, 1092, 1092, 1092, + 1092, 1092, 1092, 1092, 0, 0, 0, 0, 1092, 1092, + 1092, 1092, 1092, 1092, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1092, 1092, 1092, 1092, 1092, 1092, 1093, 1093, 1093, + 1093, 1093, 1093, 1093, 1093, 1093, 1093, 0, 0, 0, + 0, 1093, 1093, 1093, 1093, 1093, 1093, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1093, 1093, 1093, 1093, 1093, 1093, + 1099, 1099, 1099, 1099, 1099, 1099, 1099, 1099, 1099, 1099, + + 0, 0, 0, 0, 1099, 1099, 1099, 1099, 1099, 1099, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1099, 1099, 1099, + 1099, 1099, 1099, 1103, 1103, 0, 1103, 1103, 1103, 1103, + 1103, 1103, 1104, 1104, 1104, 1104, 1105, 1105, 0, 1105, + 1105, 1105, 1105, 1105, 1105, 1106, 0, 0, 1106, 1107, + 1107, 0, 1107, 1107, 1108, 1108, 0, 1108, 1108, 1108, + 1108, 1108, 1108, 1109, 1109, 1109, 1109, 1109, 1109, 1109, + 1109, 1109, 1110, 1110, 0, 1110, 1110, 1110, 1110, 1110, + 1110, 1111, 1111, 1111, 1111, 1111, 1111, 1111, 1111, 1111, + + 1112, 1112, 1112, 1112, 1112, 1112, 1112, 1112, 1112, 1114, + 0, 1114, 1114, 1115, 1115, 0, 1115, 1115, 1115, 1115, + 1115, 1115, 1116, 1116, 1116, 1116, 1116, 1116, 1116, 1116, + 1116, 1117, 1117, 1117, 1117, 1117, 1117, 1117, 1117, 1117, + 1118, 1118, 1118, 1118, 1118, 1118, 1118, 1118, 1118, 1119, + 1119, 0, 1119, 1119, 1119, 1119, 1119, 1119, 1120, 1120, + 1120, 1120, 1120, 1120, 1120, 1120, 1120, 1121, 1121, 1121, + 1121, 1121, 1121, 1121, 1121, 1121, 1122, 1122, 1122, 1122, + 1122, 1122, 1122, 1122, 1122, 1123, 1123, 1123, 0, 1123, + 1123, 1123, 1123, 1124, 1124, 1124, 1124, 1124, 1124, 1124, + + 1124, 1124, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, + 1125, 1126, 1126, 1126, 1126, 1126, 1126, 1126, 1126, 1126, + 1127, 1127, 0, 1127, 1127, 1127, 1127, 1127, 1127, 1128, + 1128, 0, 1128, 1128, 1128, 1128, 1128, 1128, 1129, 1129, + 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1130, 1130, 1130, + 1130, 1130, 1130, 1130, 1130, 1130, 1131, 1131, 1131, 1131, + 1131, 1131, 1131, 1131, 1131, 1132, 1132, 1132, 1132, 1132, + 1132, 1132, 1132, 1132, 1133, 1133, 1133, 1133, 1133, 1133, + 1133, 1133, 1133, 1134, 1134, 1134, 1134, 1134, 1134, 1134, + 1134, 1134, 1135, 1135, 1135, 1135, 1135, 1135, 1135, 1135, + + 1135, 1136, 1136, 0, 0, 1136, 1136, 1136, 1136, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102 + } ; + +static yy_state_type yy_last_accepting_state; +static char *yy_last_accepting_cpos; + +extern int yy_flex_debug; +int yy_flex_debug = 0; + +/* The intent behind this definition is that it'll catch + * any uses of REJECT which flex missed. + */ +#define REJECT reject_used_but_not_detected +#define yymore() yymore_used_but_not_detected +#define YY_MORE_ADJ 0 +#define YY_RESTORE_YY_MORE_OFFSET +char *yytext; +#line 1 "css.l" + +#line 13 "css.l" +/* Lex source for CSS tokenizing. + Taken from http://www.w3.org/TR/CSS21/grammar.html#q2 + Copyright (C) 2006, 2009-2011, 2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#define YY_NO_INPUT + +#include "css-tokens.h" + +#if defined __clang__ || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5) + #pragma GCC diagnostic ignored "-Wunknown-pragmas" // clang mourns about the next one + #pragma GCC diagnostic ignored "-Wunused-function" + #pragma GCC diagnostic ignored "-Wunused-macros" + #pragma GCC diagnostic ignored "-Wunused-parameter" + #pragma GCC diagnostic ignored "-Wsign-compare" + #pragma GCC diagnostic ignored "-Wswitch-default" + #pragma GCC diagnostic ignored "-Wunreachable-code" // clang + #pragma clang diagnostic ignored "-Wshorten-64-to-32" + #ifndef __clang__ + #pragma GCC diagnostic ignored "-Wsuggest-attribute=pure" + #endif +#endif + +#line 2452 "css.c" +#line 2453 "css.c" + +#define INITIAL 0 + +#ifndef YY_NO_UNISTD_H +/* Special case for "unistd.h", since it is non-ANSI. We include it way + * down here because we want the user's section 1 to have been scanned first. + * The user has a chance to override it with an option. + */ +#include <unistd.h> +#endif + +#ifndef YY_EXTRA_TYPE +#define YY_EXTRA_TYPE void * +#endif + +static int yy_init_globals ( void ); + +/* Accessor methods to globals. + These are made visible to non-reentrant scanners for convenience. */ + +int yylex_destroy ( void ); + +int yyget_debug ( void ); + +void yyset_debug ( int debug_flag ); + +YY_EXTRA_TYPE yyget_extra ( void ); + +void yyset_extra ( YY_EXTRA_TYPE user_defined ); + +FILE *yyget_in ( void ); + +void yyset_in ( FILE * _in_str ); + +FILE *yyget_out ( void ); + +void yyset_out ( FILE * _out_str ); + + int yyget_leng ( void ); + +char *yyget_text ( void ); + +int yyget_lineno ( void ); + +void yyset_lineno ( int _line_number ); + +/* Macros after this point can all be overridden by user definitions in + * section 1. + */ + +#ifndef YY_SKIP_YYWRAP +#ifdef __cplusplus +extern "C" int yywrap ( void ); +#else +extern int yywrap ( void ); +#endif +#endif + +#ifndef YY_NO_UNPUT + +#endif + +#ifndef yytext_ptr +static void yy_flex_strncpy ( char *, const char *, int ); +#endif + +#ifdef YY_NEED_STRLEN +static int yy_flex_strlen ( const char * ); +#endif + +#ifndef YY_NO_INPUT +#ifdef __cplusplus +static int yyinput ( void ); +#else +static int input ( void ); +#endif + +#endif + +/* Amount of stuff to slurp up with each read. */ +#ifndef YY_READ_BUF_SIZE +#ifdef __ia64__ +/* On IA-64, the buffer size is 16k, not 8k */ +#define YY_READ_BUF_SIZE 16384 +#else +#define YY_READ_BUF_SIZE 8192 +#endif /* __ia64__ */ +#endif + +/* Copy whatever the last rule matched to the standard output. */ +#ifndef ECHO +/* This used to be an fputs(), but since the string might contain NUL's, + * we now use fwrite(). + */ +#define ECHO do { if (fwrite( yytext, (size_t) yyleng, 1, yyout )) {} } while (0) +#endif + +/* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, + * is returned in "result". + */ +#ifndef YY_INPUT +#define YY_INPUT(buf,result,max_size) \ + if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \ + { \ + int c = '*'; \ + int n; \ + for ( n = 0; n < max_size && \ + (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ + buf[n] = (char) c; \ + if ( c == '\n' ) \ + buf[n++] = (char) c; \ + if ( c == EOF && ferror( yyin ) ) \ + YY_FATAL_ERROR( "input in flex scanner failed" ); \ + result = n; \ + } \ + else \ + { \ + errno=0; \ + while ( (result = (int) fread(buf, 1, (yy_size_t) max_size, yyin)) == 0 && ferror(yyin)) \ + { \ + if( errno != EINTR) \ + { \ + YY_FATAL_ERROR( "input in flex scanner failed" ); \ + break; \ + } \ + errno=0; \ + clearerr(yyin); \ + } \ + }\ +\ + +#endif + +/* No semi-colon after return; correct usage is to write "yyterminate();" - + * we don't want an extra ';' after the "return" because that will cause + * some compilers to complain about unreachable statements. + */ +#ifndef yyterminate +#define yyterminate() return YY_NULL +#endif + +/* Number of entries by which start-condition stack grows. */ +#ifndef YY_START_STACK_INCR +#define YY_START_STACK_INCR 25 +#endif + +/* Report a fatal error. */ +#ifndef YY_FATAL_ERROR +#define YY_FATAL_ERROR(msg) yy_fatal_error( msg ) +#endif + +/* end tables serialization structures and prototypes */ + +/* Default declaration of generated scanner - a define so the user can + * easily add parameters. + */ +#ifndef YY_DECL +#define YY_DECL_IS_OURS 1 + +extern int yylex (void); + +#define YY_DECL int yylex (void) +#endif /* !YY_DECL */ + +/* Code executed at the beginning of each rule, after yytext and yyleng + * have been set up. + */ +#ifndef YY_USER_ACTION +#define YY_USER_ACTION +#endif + +/* Code executed at the end of each rule. */ +#ifndef YY_BREAK +#define YY_BREAK /*LINTED*/break; +#endif + +#define YY_RULE_SETUP \ + YY_USER_ACTION + +/** The main scanner function which does all the work. + */ +YY_DECL +{ + yy_state_type yy_current_state; + char *yy_cp, *yy_bp; + int yy_act; + + if ( !(yy_init) ) + { + (yy_init) = 1; + +#ifdef YY_USER_INIT + YY_USER_INIT; +#endif + + if ( ! (yy_start) ) + (yy_start) = 1; /* first start state */ + + if ( ! yyin ) + yyin = stdin; + + if ( ! yyout ) + yyout = stdout; + + if ( ! YY_CURRENT_BUFFER ) { + yyensure_buffer_stack (); + YY_CURRENT_BUFFER_LVALUE = + yy_create_buffer( yyin, YY_BUF_SIZE ); + } + + yy_load_buffer_state( ); + } + + { +#line 112 "css.l" + + +#line 2671 "css.c" + + while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ + { + yy_cp = (yy_c_buf_p); + + /* Support of yytext. */ + *yy_cp = (yy_hold_char); + + /* yy_bp points to the position in yy_ch_buf of the start of + * the current run. + */ + yy_bp = yy_cp; + + yy_current_state = (yy_start); +yy_match: + do + { + YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)] ; + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 1103 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + ++yy_cp; + } + while ( yy_current_state != 1102 ); + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + +yy_find_action: + yy_act = yy_accept[yy_current_state]; + + YY_DO_BEFORE_ACTION; + +do_action: /* This label is used only to access EOF actions. */ + + switch ( yy_act ) + { /* beginning of action switch */ + case 0: /* must back up */ + /* undo the effects of YY_DO_BEFORE_ACTION */ + *yy_cp = (yy_hold_char); + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + goto yy_find_action; + +case 1: +/* rule 1 can match eol */ +YY_RULE_SETUP +#line 114 "css.l" +{return S;} + YY_BREAK +case 2: +/* rule 2 can match eol */ +YY_RULE_SETUP +#line 116 "css.l" +{return COMMENT;} + YY_BREAK +case 3: +/* rule 3 can match eol */ +YY_RULE_SETUP +#line 117 "css.l" +/* ignore comments */ + YY_BREAK +case 4: +/* rule 4 can match eol */ +YY_RULE_SETUP +#line 118 "css.l" +/* unclosed comment at EOF */ + YY_BREAK +case 5: +YY_RULE_SETUP +#line 120 "css.l" +{return CDO;} + YY_BREAK +case 6: +YY_RULE_SETUP +#line 121 "css.l" +{return CDC;} + YY_BREAK +case 7: +YY_RULE_SETUP +#line 122 "css.l" +{return INCLUDES;} + YY_BREAK +case 8: +YY_RULE_SETUP +#line 123 "css.l" +{return DASHMATCH;} + YY_BREAK +case 9: +/* rule 9 can match eol */ +YY_RULE_SETUP +#line 125 "css.l" +{return STRING;} + YY_BREAK +case 10: +/* rule 10 can match eol */ +YY_RULE_SETUP +#line 126 "css.l" +{return BAD_STRING;} + YY_BREAK +case 11: +/* rule 11 can match eol */ +YY_RULE_SETUP +#line 128 "css.l" +{return IDENT;} + YY_BREAK +case 12: +/* rule 12 can match eol */ +YY_RULE_SETUP +#line 130 "css.l" +{return HASH;} + YY_BREAK +case 13: +/* rule 13 can match eol */ +YY_RULE_SETUP +#line 132 "css.l" +{return IMPORT_SYM;} + YY_BREAK +case 14: +/* rule 14 can match eol */ +YY_RULE_SETUP +#line 133 "css.l" +{return PAGE_SYM;} + YY_BREAK +case 15: +/* rule 15 can match eol */ +YY_RULE_SETUP +#line 134 "css.l" +{return MEDIA_SYM;} + YY_BREAK +case 16: +YY_RULE_SETUP +#line 135 "css.l" +{return CHARSET_SYM;} + YY_BREAK +case 17: +/* rule 17 can match eol */ +YY_RULE_SETUP +#line 137 "css.l" +{return IMPORTANT_SYM;} + YY_BREAK +case 18: +/* rule 18 can match eol */ +YY_RULE_SETUP +#line 139 "css.l" +{return EMS;} + YY_BREAK +case 19: +/* rule 19 can match eol */ +YY_RULE_SETUP +#line 140 "css.l" +{return EXS;} + YY_BREAK +case 20: +/* rule 20 can match eol */ +YY_RULE_SETUP +#line 141 "css.l" +{return LENGTH;} + YY_BREAK +case 21: +/* rule 21 can match eol */ +YY_RULE_SETUP +#line 142 "css.l" +{return LENGTH;} + YY_BREAK +case 22: +/* rule 22 can match eol */ +YY_RULE_SETUP +#line 143 "css.l" +{return LENGTH;} + YY_BREAK +case 23: +/* rule 23 can match eol */ +YY_RULE_SETUP +#line 144 "css.l" +{return LENGTH;} + YY_BREAK +case 24: +/* rule 24 can match eol */ +YY_RULE_SETUP +#line 145 "css.l" +{return LENGTH;} + YY_BREAK +case 25: +/* rule 25 can match eol */ +YY_RULE_SETUP +#line 146 "css.l" +{return LENGTH;} + YY_BREAK +case 26: +/* rule 26 can match eol */ +YY_RULE_SETUP +#line 147 "css.l" +{return ANGLE;} + YY_BREAK +case 27: +/* rule 27 can match eol */ +YY_RULE_SETUP +#line 148 "css.l" +{return ANGLE;} + YY_BREAK +case 28: +/* rule 28 can match eol */ +YY_RULE_SETUP +#line 149 "css.l" +{return ANGLE;} + YY_BREAK +case 29: +/* rule 29 can match eol */ +YY_RULE_SETUP +#line 150 "css.l" +{return TIME;} + YY_BREAK +case 30: +/* rule 30 can match eol */ +YY_RULE_SETUP +#line 151 "css.l" +{return TIME;} + YY_BREAK +case 31: +/* rule 31 can match eol */ +YY_RULE_SETUP +#line 152 "css.l" +{return FREQ;} + YY_BREAK +case 32: +/* rule 32 can match eol */ +YY_RULE_SETUP +#line 153 "css.l" +{return FREQ;} + YY_BREAK +case 33: +/* rule 33 can match eol */ +YY_RULE_SETUP +#line 154 "css.l" +{return DIMENSION;} + YY_BREAK +case 34: +YY_RULE_SETUP +#line 156 "css.l" +{return PERCENTAGE;} + YY_BREAK +case 35: +YY_RULE_SETUP +#line 157 "css.l" +{return NUMBER;} + YY_BREAK +case 36: +/* rule 36 can match eol */ +YY_RULE_SETUP +#line 159 "css.l" +{return URI;} + YY_BREAK +case 37: +/* rule 37 can match eol */ +YY_RULE_SETUP +#line 160 "css.l" +{return URI;} + YY_BREAK +case 38: +/* rule 38 can match eol */ +YY_RULE_SETUP +#line 161 "css.l" +{return BAD_URI;} + YY_BREAK +case 39: +/* rule 39 can match eol */ +YY_RULE_SETUP +#line 163 "css.l" +{return FUNCTION;} + YY_BREAK +case 40: +YY_RULE_SETUP +#line 165 "css.l" +{return *yytext;} + YY_BREAK +case 41: +YY_RULE_SETUP +#line 167 "css.l" +ECHO; + YY_BREAK +#line 2961 "css.c" +case YY_STATE_EOF(INITIAL): + yyterminate(); + + case YY_END_OF_BUFFER: + { + /* Amount of text matched not including the EOB char. */ + int yy_amount_of_matched_text = (int) (yy_cp - (yytext_ptr)) - 1; + + /* Undo the effects of YY_DO_BEFORE_ACTION. */ + *yy_cp = (yy_hold_char); + YY_RESTORE_YY_MORE_OFFSET + + if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_NEW ) + { + /* We're scanning a new file or input source. It's + * possible that this happened because the user + * just pointed yyin at a new source and called + * yylex(). If so, then we have to assure + * consistency between YY_CURRENT_BUFFER and our + * globals. Here is the right place to do so, because + * this is the first action (other than possibly a + * back-up) that will match for the new input source. + */ + (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; + YY_CURRENT_BUFFER_LVALUE->yy_input_file = yyin; + YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = YY_BUFFER_NORMAL; + } + + /* Note that here we test for yy_c_buf_p "<=" to the position + * of the first EOB in the buffer, since yy_c_buf_p will + * already have been incremented past the NUL character + * (since all states make transitions on EOB to the + * end-of-buffer state). Contrast this with the test + * in input(). + */ + if ( (yy_c_buf_p) <= &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] ) + { /* This was really a NUL. */ + yy_state_type yy_next_state; + + (yy_c_buf_p) = (yytext_ptr) + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state( ); + + /* Okay, we're now positioned to make the NUL + * transition. We couldn't have + * yy_get_previous_state() go ahead and do it + * for us because it doesn't know how to deal + * with the possibility of jamming (and we don't + * want to build jamming into it because then it + * will run more slowly). + */ + + yy_next_state = yy_try_NUL_trans( yy_current_state ); + + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + + if ( yy_next_state ) + { + /* Consume the NUL. */ + yy_cp = ++(yy_c_buf_p); + yy_current_state = yy_next_state; + goto yy_match; + } + + else + { + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + goto yy_find_action; + } + } + + else switch ( yy_get_next_buffer( ) ) + { + case EOB_ACT_END_OF_FILE: + { + (yy_did_buffer_switch_on_eof) = 0; + + if ( yywrap( ) ) + { + /* Note: because we've taken care in + * yy_get_next_buffer() to have set up + * yytext, we can now set up + * yy_c_buf_p so that if some total + * hoser (like flex itself) wants to + * call the scanner after we return the + * YY_NULL, it'll still work - another + * YY_NULL will get returned. + */ + (yy_c_buf_p) = (yytext_ptr) + YY_MORE_ADJ; + + yy_act = YY_STATE_EOF(YY_START); + goto do_action; + } + + else + { + if ( ! (yy_did_buffer_switch_on_eof) ) + YY_NEW_FILE; + } + break; + } + + case EOB_ACT_CONTINUE_SCAN: + (yy_c_buf_p) = + (yytext_ptr) + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state( ); + + yy_cp = (yy_c_buf_p); + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + goto yy_match; + + case EOB_ACT_LAST_MATCH: + (yy_c_buf_p) = + &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)]; + + yy_current_state = yy_get_previous_state( ); + + yy_cp = (yy_c_buf_p); + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + goto yy_find_action; + } + break; + } + + default: + YY_FATAL_ERROR( + "fatal flex scanner internal error--no action found" ); + } /* end of action switch */ + } /* end of scanning one token */ + } /* end of user's declarations */ +} /* end of yylex */ + +/* yy_get_next_buffer - try to read in a new buffer + * + * Returns a code representing an action: + * EOB_ACT_LAST_MATCH - + * EOB_ACT_CONTINUE_SCAN - continue scanning from current position + * EOB_ACT_END_OF_FILE - end of file + */ +static int yy_get_next_buffer (void) +{ + char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; + char *source = (yytext_ptr); + int number_to_move, i; + int ret_val; + + if ( (yy_c_buf_p) > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] ) + YY_FATAL_ERROR( + "fatal flex scanner internal error--end of buffer missed" ); + + if ( YY_CURRENT_BUFFER_LVALUE->yy_fill_buffer == 0 ) + { /* Don't try to fill the buffer, so this is an EOF. */ + if ( (yy_c_buf_p) - (yytext_ptr) - YY_MORE_ADJ == 1 ) + { + /* We matched a single character, the EOB, so + * treat this as a final EOF. + */ + return EOB_ACT_END_OF_FILE; + } + + else + { + /* We matched some text prior to the EOB, first + * process it. + */ + return EOB_ACT_LAST_MATCH; + } + } + + /* Try to read more data. */ + + /* First move last chars to start of buffer. */ + number_to_move = (int) ((yy_c_buf_p) - (yytext_ptr) - 1); + + for ( i = 0; i < number_to_move; ++i ) + *(dest++) = *(source++); + + if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_EOF_PENDING ) + /* don't do the read, it's not guaranteed to return an EOF, + * just force an EOF + */ + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars) = 0; + + else + { + int num_to_read = + YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1; + + while ( num_to_read <= 0 ) + { /* Not enough room in the buffer - grow it. */ + + /* just a shorter name for the current buffer */ + YY_BUFFER_STATE b = YY_CURRENT_BUFFER_LVALUE; + + int yy_c_buf_p_offset = + (int) ((yy_c_buf_p) - b->yy_ch_buf); + + if ( b->yy_is_our_buffer ) + { + int new_size = b->yy_buf_size * 2; + + if ( new_size <= 0 ) + b->yy_buf_size += b->yy_buf_size / 8; + else + b->yy_buf_size *= 2; + + b->yy_ch_buf = (char *) + /* Include room in for 2 EOB chars. */ + yyrealloc( (void *) b->yy_ch_buf, + (yy_size_t) (b->yy_buf_size + 2) ); + } + else + /* Can't grow it, we don't own it. */ + b->yy_ch_buf = NULL; + + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( + "fatal error - scanner input buffer overflow" ); + + (yy_c_buf_p) = &b->yy_ch_buf[yy_c_buf_p_offset]; + + num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size - + number_to_move - 1; + + } + + if ( num_to_read > YY_READ_BUF_SIZE ) + num_to_read = YY_READ_BUF_SIZE; + + /* Read in more data. */ + YY_INPUT( (&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]), + (yy_n_chars), num_to_read ); + + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + if ( (yy_n_chars) == 0 ) + { + if ( number_to_move == YY_MORE_ADJ ) + { + ret_val = EOB_ACT_END_OF_FILE; + yyrestart( yyin ); + } + + else + { + ret_val = EOB_ACT_LAST_MATCH; + YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = + YY_BUFFER_EOF_PENDING; + } + } + + else + ret_val = EOB_ACT_CONTINUE_SCAN; + + if (((yy_n_chars) + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { + /* Extend the array by 50%, plus the number we really need. */ + int new_size = (yy_n_chars) + number_to_move + ((yy_n_chars) >> 1); + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc( + (void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, (yy_size_t) new_size ); + if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_get_next_buffer()" ); + /* "- 2" to take care of EOB's */ + YY_CURRENT_BUFFER_LVALUE->yy_buf_size = (int) (new_size - 2); + } + + (yy_n_chars) += number_to_move; + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] = YY_END_OF_BUFFER_CHAR; + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] = YY_END_OF_BUFFER_CHAR; + + (yytext_ptr) = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[0]; + + return ret_val; +} + +/* yy_get_previous_state - get the state just before the EOB char was reached */ + + static yy_state_type yy_get_previous_state (void) +{ + yy_state_type yy_current_state; + char *yy_cp; + + yy_current_state = (yy_start); + + for ( yy_cp = (yytext_ptr) + YY_MORE_ADJ; yy_cp < (yy_c_buf_p); ++yy_cp ) + { + YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 1103 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + } + + return yy_current_state; +} + +/* yy_try_NUL_trans - try to make a transition on the NUL character + * + * synopsis + * next_state = yy_try_NUL_trans( current_state ); + */ + static yy_state_type yy_try_NUL_trans (yy_state_type yy_current_state ) +{ + int yy_is_jam; + char *yy_cp = (yy_c_buf_p); + + YY_CHAR yy_c = 1; + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 1103 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_is_jam = (yy_current_state == 1102); + + return yy_is_jam ? 0 : yy_current_state; +} + +#ifndef YY_NO_UNPUT + +#endif + +#ifndef YY_NO_INPUT +#ifdef __cplusplus + static int yyinput (void) +#else + static int input (void) +#endif + +{ + int c; + + *(yy_c_buf_p) = (yy_hold_char); + + if ( *(yy_c_buf_p) == YY_END_OF_BUFFER_CHAR ) + { + /* yy_c_buf_p now points to the character we want to return. + * If this occurs *before* the EOB characters, then it's a + * valid NUL; if not, then we've hit the end of the buffer. + */ + if ( (yy_c_buf_p) < &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] ) + /* This was really a NUL. */ + *(yy_c_buf_p) = '\0'; + + else + { /* need more input */ + int offset = (int) ((yy_c_buf_p) - (yytext_ptr)); + ++(yy_c_buf_p); + + switch ( yy_get_next_buffer( ) ) + { + case EOB_ACT_LAST_MATCH: + /* This happens because yy_g_n_b() + * sees that we've accumulated a + * token and flags that we need to + * try matching the token before + * proceeding. But for input(), + * there's no matching to consider. + * So convert the EOB_ACT_LAST_MATCH + * to EOB_ACT_END_OF_FILE. + */ + + /* Reset buffer status. */ + yyrestart( yyin ); + + /*FALLTHROUGH*/ + + case EOB_ACT_END_OF_FILE: + { + if ( yywrap( ) ) + return 0; + + if ( ! (yy_did_buffer_switch_on_eof) ) + YY_NEW_FILE; +#ifdef __cplusplus + return yyinput(); +#else + return input(); +#endif + } + + case EOB_ACT_CONTINUE_SCAN: + (yy_c_buf_p) = (yytext_ptr) + offset; + break; + } + } + } + + c = *(unsigned char *) (yy_c_buf_p); /* cast for 8-bit char's */ + *(yy_c_buf_p) = '\0'; /* preserve yytext */ + (yy_hold_char) = *++(yy_c_buf_p); + + return c; +} +#endif /* ifndef YY_NO_INPUT */ + +/** Immediately switch to a different input stream. + * @param input_file A readable stream. + * + * @note This function does not reset the start condition to @c INITIAL . + */ + void yyrestart (FILE * input_file ) +{ + + if ( ! YY_CURRENT_BUFFER ){ + yyensure_buffer_stack (); + YY_CURRENT_BUFFER_LVALUE = + yy_create_buffer( yyin, YY_BUF_SIZE ); + } + + yy_init_buffer( YY_CURRENT_BUFFER, input_file ); + yy_load_buffer_state( ); +} + +/** Switch to a different input buffer. + * @param new_buffer The new input buffer. + * + */ + void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ) +{ + + /* TODO. We should be able to replace this entire function body + * with + * yypop_buffer_state(); + * yypush_buffer_state(new_buffer); + */ + yyensure_buffer_stack (); + if ( YY_CURRENT_BUFFER == new_buffer ) + return; + + if ( YY_CURRENT_BUFFER ) + { + /* Flush out information for old buffer. */ + *(yy_c_buf_p) = (yy_hold_char); + YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p); + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + YY_CURRENT_BUFFER_LVALUE = new_buffer; + yy_load_buffer_state( ); + + /* We don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + (yy_did_buffer_switch_on_eof) = 1; +} + +static void yy_load_buffer_state (void) +{ + (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; + (yytext_ptr) = (yy_c_buf_p) = YY_CURRENT_BUFFER_LVALUE->yy_buf_pos; + yyin = YY_CURRENT_BUFFER_LVALUE->yy_input_file; + (yy_hold_char) = *(yy_c_buf_p); +} + +/** Allocate and initialize an input buffer state. + * @param file A readable stream. + * @param size The character buffer size in bytes. When in doubt, use @c YY_BUF_SIZE. + * + * @return the allocated buffer state. + */ + YY_BUFFER_STATE yy_create_buffer (FILE * file, int size ) +{ + YY_BUFFER_STATE b; + + b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_buf_size = size; + + /* yy_ch_buf has to be 2 characters longer than the size given because + * we need to put in 2 end-of-buffer characters. + */ + b->yy_ch_buf = (char *) yyalloc( (yy_size_t) (b->yy_buf_size + 2) ); + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_is_our_buffer = 1; + + yy_init_buffer( b, file ); + + return b; +} + +/** Destroy the buffer. + * @param b a buffer created with yy_create_buffer() + * + */ + void yy_delete_buffer (YY_BUFFER_STATE b ) +{ + + if ( ! b ) + return; + + if ( b == YY_CURRENT_BUFFER ) /* Not sure if we should pop here. */ + YY_CURRENT_BUFFER_LVALUE = (YY_BUFFER_STATE) 0; + + if ( b->yy_is_our_buffer ) + yyfree( (void *) b->yy_ch_buf ); + + yyfree( (void *) b ); +} + +/* Initializes or reinitializes a buffer. + * This function is sometimes called more than once on the same buffer, + * such as during a yyrestart() or at EOF. + */ + static void yy_init_buffer (YY_BUFFER_STATE b, FILE * file ) + +{ + int oerrno = errno; + + yy_flush_buffer( b ); + + b->yy_input_file = file; + b->yy_fill_buffer = 1; + + /* If b is the current buffer, then yy_init_buffer was _probably_ + * called from yyrestart() or through yy_get_next_buffer. + * In that case, we don't want to reset the lineno or column. + */ + if (b != YY_CURRENT_BUFFER){ + b->yy_bs_lineno = 1; + b->yy_bs_column = 0; + } + + b->yy_is_interactive = 0; + + errno = oerrno; +} + +/** Discard all buffered characters. On the next scan, YY_INPUT will be called. + * @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER. + * + */ + void yy_flush_buffer (YY_BUFFER_STATE b ) +{ + if ( ! b ) + return; + + b->yy_n_chars = 0; + + /* We always need two end-of-buffer characters. The first causes + * a transition to the end-of-buffer state. The second causes + * a jam in that state. + */ + b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; + b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; + + b->yy_buf_pos = &b->yy_ch_buf[0]; + + b->yy_at_bol = 1; + b->yy_buffer_status = YY_BUFFER_NEW; + + if ( b == YY_CURRENT_BUFFER ) + yy_load_buffer_state( ); +} + +/** Pushes the new state onto the stack. The new state becomes + * the current state. This function will allocate the stack + * if necessary. + * @param new_buffer The new state. + * + */ +void yypush_buffer_state (YY_BUFFER_STATE new_buffer ) +{ + if (new_buffer == NULL) + return; + + yyensure_buffer_stack(); + + /* This block is copied from yy_switch_to_buffer. */ + if ( YY_CURRENT_BUFFER ) + { + /* Flush out information for old buffer. */ + *(yy_c_buf_p) = (yy_hold_char); + YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p); + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + /* Only push if top exists. Otherwise, replace top. */ + if (YY_CURRENT_BUFFER) + (yy_buffer_stack_top)++; + YY_CURRENT_BUFFER_LVALUE = new_buffer; + + /* copied from yy_switch_to_buffer. */ + yy_load_buffer_state( ); + (yy_did_buffer_switch_on_eof) = 1; +} + +/** Removes and deletes the top of the stack, if present. + * The next element becomes the new top. + * + */ +void yypop_buffer_state (void) +{ + if (!YY_CURRENT_BUFFER) + return; + + yy_delete_buffer(YY_CURRENT_BUFFER ); + YY_CURRENT_BUFFER_LVALUE = NULL; + if ((yy_buffer_stack_top) > 0) + --(yy_buffer_stack_top); + + if (YY_CURRENT_BUFFER) { + yy_load_buffer_state( ); + (yy_did_buffer_switch_on_eof) = 1; + } +} + +/* Allocates the stack if it does not exist. + * Guarantees space for at least one push. + */ +static void yyensure_buffer_stack (void) +{ + yy_size_t num_to_alloc; + + if (!(yy_buffer_stack)) { + + /* First allocation is just for 2 elements, since we don't know if this + * scanner will even need a stack. We use 2 instead of 1 to avoid an + * immediate realloc on the next call. + */ + num_to_alloc = 1; /* After all that talk, this was set to 1 anyways... */ + (yy_buffer_stack) = (struct yy_buffer_state**)yyalloc + (num_to_alloc * sizeof(struct yy_buffer_state*) + ); + if ( ! (yy_buffer_stack) ) + YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + + memset((yy_buffer_stack), 0, num_to_alloc * sizeof(struct yy_buffer_state*)); + + (yy_buffer_stack_max) = num_to_alloc; + (yy_buffer_stack_top) = 0; + return; + } + + if ((yy_buffer_stack_top) >= ((yy_buffer_stack_max)) - 1){ + + /* Increase the buffer to prepare for a possible push. */ + yy_size_t grow_size = 8 /* arbitrary grow size */; + + num_to_alloc = (yy_buffer_stack_max) + grow_size; + (yy_buffer_stack) = (struct yy_buffer_state**)yyrealloc + ((yy_buffer_stack), + num_to_alloc * sizeof(struct yy_buffer_state*) + ); + if ( ! (yy_buffer_stack) ) + YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + + /* zero only the new slots.*/ + memset((yy_buffer_stack) + (yy_buffer_stack_max), 0, grow_size * sizeof(struct yy_buffer_state*)); + (yy_buffer_stack_max) = num_to_alloc; + } +} + +/** Setup the input buffer state to scan directly from a user-specified character buffer. + * @param base the character buffer + * @param size the size in bytes of the character buffer + * + * @return the newly allocated buffer state object. + */ +YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size ) +{ + YY_BUFFER_STATE b; + + if ( size < 2 || + base[size-2] != YY_END_OF_BUFFER_CHAR || + base[size-1] != YY_END_OF_BUFFER_CHAR ) + /* They forgot to leave room for the EOB's. */ + return NULL; + + b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); + + b->yy_buf_size = (int) (size - 2); /* "- 2" to take care of EOB's */ + b->yy_buf_pos = b->yy_ch_buf = base; + b->yy_is_our_buffer = 0; + b->yy_input_file = NULL; + b->yy_n_chars = b->yy_buf_size; + b->yy_is_interactive = 0; + b->yy_at_bol = 1; + b->yy_fill_buffer = 0; + b->yy_buffer_status = YY_BUFFER_NEW; + + yy_switch_to_buffer( b ); + + return b; +} + +/** Setup the input buffer state to scan a string. The next call to yylex() will + * scan from a @e copy of @a str. + * @param yystr a NUL-terminated string to scan + * + * @return the newly allocated buffer state object. + * @note If you want to scan bytes that may contain NUL values, then use + * yy_scan_bytes() instead. + */ +YY_BUFFER_STATE yy_scan_string (const char * yystr ) +{ + + return yy_scan_bytes( yystr, (int) strlen(yystr) ); +} + +/** Setup the input buffer state to scan the given bytes. The next call to yylex() will + * scan from a @e copy of @a bytes. + * @param yybytes the byte buffer to scan + * @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes. + * + * @return the newly allocated buffer state object. + */ +YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len ) +{ + YY_BUFFER_STATE b; + char *buf; + yy_size_t n; + int i; + + /* Get memory for full buffer, including space for trailing EOB's. */ + n = (yy_size_t) (_yybytes_len + 2); + buf = (char *) yyalloc( n ); + if ( ! buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); + + for ( i = 0; i < _yybytes_len; ++i ) + buf[i] = yybytes[i]; + + buf[_yybytes_len] = buf[_yybytes_len+1] = YY_END_OF_BUFFER_CHAR; + + b = yy_scan_buffer( buf, n ); + if ( ! b ) + YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); + + /* It's okay to grow etc. this buffer, and we should throw it + * away when we're done. + */ + b->yy_is_our_buffer = 1; + + return b; +} + +#ifndef YY_EXIT_FAILURE +#define YY_EXIT_FAILURE 2 +#endif + +static void yynoreturn yy_fatal_error (const char* msg ) +{ + fprintf( stderr, "%s\n", msg ); + exit( YY_EXIT_FAILURE ); +} + +/* Redefine yyless() so it works in section 3 code. */ + +#undef yyless +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + int yyless_macro_arg = (n); \ + YY_LESS_LINENO(yyless_macro_arg);\ + yytext[yyleng] = (yy_hold_char); \ + (yy_c_buf_p) = yytext + yyless_macro_arg; \ + (yy_hold_char) = *(yy_c_buf_p); \ + *(yy_c_buf_p) = '\0'; \ + yyleng = yyless_macro_arg; \ + } \ + while ( 0 ) + +/* Accessor methods (get/set functions) to struct members. */ + +/** Get the current line number. + * + */ +int yyget_lineno (void) +{ + + return yylineno; +} + +/** Get the input stream. + * + */ +FILE *yyget_in (void) +{ + return yyin; +} + +/** Get the output stream. + * + */ +FILE *yyget_out (void) +{ + return yyout; +} + +/** Get the length of the current token. + * + */ +int yyget_leng (void) +{ + return yyleng; +} + +/** Get the current token. + * + */ + +char *yyget_text (void) +{ + return yytext; +} + +/** Set the current line number. + * @param _line_number line number + * + */ +void yyset_lineno (int _line_number ) +{ + + yylineno = _line_number; +} + +/** Set the input stream. This does not discard the current + * input buffer. + * @param _in_str A readable stream. + * + * @see yy_switch_to_buffer + */ +void yyset_in (FILE * _in_str ) +{ + yyin = _in_str ; +} + +void yyset_out (FILE * _out_str ) +{ + yyout = _out_str ; +} + +int yyget_debug (void) +{ + return yy_flex_debug; +} + +void yyset_debug (int _bdebug ) +{ + yy_flex_debug = _bdebug ; +} + +static int yy_init_globals (void) +{ + /* Initialization is the same as for the non-reentrant scanner. + * This function is called from yylex_destroy(), so don't allocate here. + */ + + (yy_buffer_stack) = NULL; + (yy_buffer_stack_top) = 0; + (yy_buffer_stack_max) = 0; + (yy_c_buf_p) = NULL; + (yy_init) = 0; + (yy_start) = 0; + +/* Defined in main.c */ +#ifdef YY_STDINIT + yyin = stdin; + yyout = stdout; +#else + yyin = NULL; + yyout = NULL; +#endif + + /* For future reference: Set errno on error, since we are called by + * yylex_init() + */ + return 0; +} + +/* yylex_destroy is for both reentrant and non-reentrant scanners. */ +int yylex_destroy (void) +{ + + /* Pop the buffer stack, destroying each element. */ + while(YY_CURRENT_BUFFER){ + yy_delete_buffer( YY_CURRENT_BUFFER ); + YY_CURRENT_BUFFER_LVALUE = NULL; + yypop_buffer_state(); + } + + /* Destroy the stack itself. */ + yyfree((yy_buffer_stack) ); + (yy_buffer_stack) = NULL; + + /* Reset the globals. This is important in a non-reentrant scanner so the next time + * yylex() is called, initialization will occur. */ + yy_init_globals( ); + + return 0; +} + +/* + * Internal utility routines. + */ + +#ifndef yytext_ptr +static void yy_flex_strncpy (char* s1, const char * s2, int n ) +{ + + int i; + for ( i = 0; i < n; ++i ) + s1[i] = s2[i]; +} +#endif + +#ifdef YY_NEED_STRLEN +static int yy_flex_strlen (const char * s ) +{ + int n; + for ( n = 0; s[n]; ++n ) + ; + + return n; +} +#endif + +void *yyalloc (yy_size_t size ) +{ + return malloc(size); +} + +void *yyrealloc (void * ptr, yy_size_t size ) +{ + + /* The cast to (char *) in the following accommodates both + * implementations that use char* generic pointers, and those + * that use void* generic pointers. It works with the latter + * because both ANSI C and C++ allow castless assignment from + * any pointer type to void*, and deal with argument conversions + * as though doing an assignment. + */ + return realloc(ptr, size); +} + +void yyfree (void * ptr ) +{ + free( (char *) ptr ); /* see yyrealloc() for (char *) cast */ +} + +#define YYTABLES_NAME "yytables" + +#line 167 "css.l" + + diff --git a/src/css.l b/src/css.l new file mode 100644 index 0000000..bae065d --- /dev/null +++ b/src/css.l @@ -0,0 +1,167 @@ +%option case-insensitive +%option noyywrap +%option never-interactive +%option nounput + +%top{ +/* config.h must precede flex's inclusion of <stdio.h> + in order for its _GNU_SOURCE definition to take effect. */ +#include <config.h> +} + +%{ +/* Lex source for CSS tokenizing. + Taken from http://www.w3.org/TR/CSS21/grammar.html#q2 + Copyright (C) 2006, 2009-2011, 2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#define YY_NO_INPUT + +#include "css-tokens.h" + +#if defined __clang__ || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5) + #pragma GCC diagnostic ignored "-Wunknown-pragmas" // clang mourns about the next one + #pragma GCC diagnostic ignored "-Wunused-function" + #pragma GCC diagnostic ignored "-Wunused-macros" + #pragma GCC diagnostic ignored "-Wunused-parameter" + #pragma GCC diagnostic ignored "-Wsign-compare" + #pragma GCC diagnostic ignored "-Wswitch-default" + #pragma GCC diagnostic ignored "-Wunreachable-code" // clang + #pragma clang diagnostic ignored "-Wshorten-64-to-32" + #ifndef __clang__ + #pragma GCC diagnostic ignored "-Wsuggest-attribute=pure" + #endif +#endif + +%} + +h [0-9a-f] +nonascii [\240-\377] +unicode \\{h}{1,6}(\r\n|[ \t\r\n\f])? +escape {unicode}|\\[^\r\n\f0-9a-f] +nmstart [_a-z]|{nonascii}|{escape} +nmchar [_a-z0-9-]|{nonascii}|{escape} +string1 \"([^\n\r\f\\"]|\\{nl}|{escape})*\" +string2 \'([^\n\r\f\\']|\\{nl}|{escape})*\' +badstring1 \"([^\n\r\f\\"]|\\{nl}|{escape})*\\? +badstring2 \'([^\n\r\f\\']|\\{nl}|{escape})*\\? +badcomment1 \/\*[^*]*\*+([^/*][^*]*\*+)* +badcomment2 \/\*[^*]*(\*+[^/*][^*]*)* +baduri1 url\({w}([!#$%&*-\[\]-~]|{nonascii}|{escape})*{w} +baduri2 url\({w}{string}{w} +baduri3 url\({w}{badstring} +comment \/\*[^*]*\*+([^/*][^*]*\*+)*\/ +ident -?{nmstart}{nmchar}* +name {nmchar}+ +num [0-9]+|[0-9]*"."[0-9]+ +string {string1}|{string2} +badstring {badstring1}|{badstring2} +badcomment {badcomment1}|{badcomment2} +baduri {baduri1}|{baduri2}|{baduri3} +url ([!#$%&*-~]|{nonascii}|{escape})* +s [ \t\r\n\f]+ +w {s}? +nl \n|\r\n|\r|\f + +A a|\\0{0,4}(41|61)(\r\n|[ \t\r\n\f])? +C c|\\0{0,4}(43|63)(\r\n|[ \t\r\n\f])? +D d|\\0{0,4}(44|64)(\r\n|[ \t\r\n\f])? +E e|\\0{0,4}(45|65)(\r\n|[ \t\r\n\f])? +G g|\\0{0,4}(47|67)(\r\n|[ \t\r\n\f])?|\\g +H h|\\0{0,4}(48|68)(\r\n|[ \t\r\n\f])?|\\h +I i|\\0{0,4}(49|69)(\r\n|[ \t\r\n\f])?|\\i +K k|\\0{0,4}(4b|6b)(\r\n|[ \t\r\n\f])?|\\k +L l|\\0{0,4}(4c|6c)(\r\n|[ \t\r\n\f])?|\\l +M m|\\0{0,4}(4d|6d)(\r\n|[ \t\r\n\f])?|\\m +N n|\\0{0,4}(4e|6e)(\r\n|[ \t\r\n\f])?|\\n +O o|\\0{0,4}(4f|6f)(\r\n|[ \t\r\n\f])?|\\o +P p|\\0{0,4}(50|70)(\r\n|[ \t\r\n\f])?|\\p +R r|\\0{0,4}(52|72)(\r\n|[ \t\r\n\f])?|\\r +S s|\\0{0,4}(53|73)(\r\n|[ \t\r\n\f])?|\\s +T t|\\0{0,4}(54|74)(\r\n|[ \t\r\n\f])?|\\t +U u|\\0{0,4}(55|75)(\r\n|[ \t\r\n\f])?|\\u +X x|\\0{0,4}(58|78)(\r\n|[ \t\r\n\f])?|\\x +Z z|\\0{0,4}(5a|7a)(\r\n|[ \t\r\n\f])?|\\z + +%% + +{s} {return S;} + +{comment} {return COMMENT;} +#\/\*[^*]*\*+([^/*][^*]*\*+)*\/ /* ignore comments */ +{badcomment} /* unclosed comment at EOF */ + +"<!--" {return CDO;} +"-->" {return CDC;} +"~=" {return INCLUDES;} +"|=" {return DASHMATCH;} + +{string} {return STRING;} +{badstring} {return BAD_STRING;} + +{ident} {return IDENT;} + +"#"{name} {return HASH;} + +@{I}{M}{P}{O}{R}{T} {return IMPORT_SYM;} +@{P}{A}{G}{E} {return PAGE_SYM;} +@{M}{E}{D}{I}{A} {return MEDIA_SYM;} +"@charset " {return CHARSET_SYM;} + +"!"({w}|{comment})*{I}{M}{P}{O}{R}{T}{A}{N}{T} {return IMPORTANT_SYM;} + +{num}{E}{M} {return EMS;} +{num}{E}{X} {return EXS;} +{num}{P}{X} {return LENGTH;} +{num}{C}{M} {return LENGTH;} +{num}{M}{M} {return LENGTH;} +{num}{I}{N} {return LENGTH;} +{num}{P}{T} {return LENGTH;} +{num}{P}{C} {return LENGTH;} +{num}{D}{E}{G} {return ANGLE;} +{num}{R}{A}{D} {return ANGLE;} +{num}{G}{R}{A}{D} {return ANGLE;} +{num}{M}{S} {return TIME;} +{num}{S} {return TIME;} +{num}{H}{Z} {return FREQ;} +{num}{K}{H}{Z} {return FREQ;} +{num}{ident} {return DIMENSION;} + +{num}% {return PERCENTAGE;} +{num} {return NUMBER;} + +"url("{w}{string}{w}")" {return URI;} +"url("{w}{url}{w}")" {return URI;} +{baduri} {return BAD_URI;} + +{ident}"(" {return FUNCTION;} + +. {return *yytext;} + +%% diff --git a/src/css_.c b/src/css_.c new file mode 100644 index 0000000..76e4bb9 --- /dev/null +++ b/src/css_.c @@ -0,0 +1,3933 @@ +#include "wget.h" +#line 1 "css.c" +/* config.h must precede flex's inclusion of <stdio.h> + in order for its _GNU_SOURCE definition to take effect. */ +#include <config.h> + +#line 6 "css.c" + +#define YY_INT_ALIGNED short int + +/* A lexical scanner generated by flex */ + +#define FLEX_SCANNER +#define YY_FLEX_MAJOR_VERSION 2 +#define YY_FLEX_MINOR_VERSION 6 +#define YY_FLEX_SUBMINOR_VERSION 4 +#if YY_FLEX_SUBMINOR_VERSION > 0 +#define FLEX_BETA +#endif + +/* First, we deal with platform-specific or compiler-specific issues. */ + +/* begin standard C headers. */ +#include <stdio.h> +#include <string.h> +#include <errno.h> +#include <stdlib.h> + +/* end standard C headers. */ + +/* flex integer type definitions */ + +#ifndef FLEXINT_H +#define FLEXINT_H + +/* C99 systems have <inttypes.h>. Non-C99 systems may or may not. */ + +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + +/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h, + * if you want the limit (max/min) macros for int types. + */ +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS 1 +#endif + +#include <inttypes.h> +typedef int8_t flex_int8_t; +typedef uint8_t flex_uint8_t; +typedef int16_t flex_int16_t; +typedef uint16_t flex_uint16_t; +typedef int32_t flex_int32_t; +typedef uint32_t flex_uint32_t; +#else +typedef signed char flex_int8_t; +typedef short int flex_int16_t; +typedef int flex_int32_t; +typedef unsigned char flex_uint8_t; +typedef unsigned short int flex_uint16_t; +typedef unsigned int flex_uint32_t; + +/* Limits of integral types. */ +#ifndef INT8_MIN +#define INT8_MIN (-128) +#endif +#ifndef INT16_MIN +#define INT16_MIN (-32767-1) +#endif +#ifndef INT32_MIN +#define INT32_MIN (-2147483647-1) +#endif +#ifndef INT8_MAX +#define INT8_MAX (127) +#endif +#ifndef INT16_MAX +#define INT16_MAX (32767) +#endif +#ifndef INT32_MAX +#define INT32_MAX (2147483647) +#endif +#ifndef UINT8_MAX +#define UINT8_MAX (255U) +#endif +#ifndef UINT16_MAX +#define UINT16_MAX (65535U) +#endif +#ifndef UINT32_MAX +#define UINT32_MAX (4294967295U) +#endif + +#ifndef SIZE_MAX +#define SIZE_MAX (~(size_t)0) +#endif + +#endif /* ! C99 */ + +#endif /* ! FLEXINT_H */ + +/* begin standard C++ headers. */ + +/* TODO: this is always defined, so inline it */ +#define yyconst const + +#if defined(__GNUC__) && __GNUC__ >= 3 +#define yynoreturn __attribute__((__noreturn__)) +#else +#define yynoreturn +#endif + +/* Returned upon end-of-file. */ +#define YY_NULL 0 + +/* Promotes a possibly negative, possibly signed char to an + * integer in range [0..255] for use as an array index. + */ +#define YY_SC_TO_UI(c) ((YY_CHAR) (c)) + +/* Enter a start condition. This macro really ought to take a parameter, + * but we do it the disgusting crufty way forced on us by the ()-less + * definition of BEGIN. + */ +#define BEGIN (yy_start) = 1 + 2 * +/* Translate the current start state into a value that can be later handed + * to BEGIN to return to the state. The YYSTATE alias is for lex + * compatibility. + */ +#define YY_START (((yy_start) - 1) / 2) +#define YYSTATE YY_START +/* Action number for EOF rule of a given start state. */ +#define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) +/* Special action meaning "start processing a new file". */ +#define YY_NEW_FILE yyrestart( yyin ) +#define YY_END_OF_BUFFER_CHAR 0 + +/* Size of default input buffer. */ +#ifndef YY_BUF_SIZE +#ifdef __ia64__ +/* On IA-64, the buffer size is 16k, not 8k. + * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case. + * Ditto for the __ia64__ case accordingly. + */ +#define YY_BUF_SIZE 32768 +#else +#define YY_BUF_SIZE 16384 +#endif /* __ia64__ */ +#endif + +/* The state buf must be large enough to hold one state per character in the main buffer. + */ +#define YY_STATE_BUF_SIZE ((YY_BUF_SIZE + 2) * sizeof(yy_state_type)) + +#ifndef YY_TYPEDEF_YY_BUFFER_STATE +#define YY_TYPEDEF_YY_BUFFER_STATE +typedef struct yy_buffer_state *YY_BUFFER_STATE; +#endif + +#ifndef YY_TYPEDEF_YY_SIZE_T +#define YY_TYPEDEF_YY_SIZE_T +typedef size_t yy_size_t; +#endif + +extern int yyleng; + +extern FILE *yyin, *yyout; + +#define EOB_ACT_CONTINUE_SCAN 0 +#define EOB_ACT_END_OF_FILE 1 +#define EOB_ACT_LAST_MATCH 2 + + #define YY_LESS_LINENO(n) + #define YY_LINENO_REWIND_TO(ptr) + +/* Return all but the first "n" matched characters back to the input stream. */ +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + int yyless_macro_arg = (n); \ + YY_LESS_LINENO(yyless_macro_arg);\ + *yy_cp = (yy_hold_char); \ + YY_RESTORE_YY_MORE_OFFSET \ + (yy_c_buf_p) = yy_cp = yy_bp + yyless_macro_arg - YY_MORE_ADJ; \ + YY_DO_BEFORE_ACTION; /* set up yytext again */ \ + } \ + while ( 0 ) +#define unput(c) yyunput( c, (yytext_ptr) ) + +#ifndef YY_STRUCT_YY_BUFFER_STATE +#define YY_STRUCT_YY_BUFFER_STATE +struct yy_buffer_state + { + FILE *yy_input_file; + + char *yy_ch_buf; /* input buffer */ + char *yy_buf_pos; /* current position in input buffer */ + + /* Size of input buffer in bytes, not including room for EOB + * characters. + */ + int yy_buf_size; + + /* Number of characters read into yy_ch_buf, not including EOB + * characters. + */ + int yy_n_chars; + + /* Whether we "own" the buffer - i.e., we know we created it, + * and can realloc() it to grow it, and should free() it to + * delete it. + */ + int yy_is_our_buffer; + + /* Whether this is an "interactive" input source; if so, and + * if we're using stdio for input, then we want to use getc() + * instead of fread(), to make sure we stop fetching input after + * each newline. + */ + int yy_is_interactive; + + /* Whether we're considered to be at the beginning of a line. + * If so, '^' rules will be active on the next match, otherwise + * not. + */ + int yy_at_bol; + + int yy_bs_lineno; /**< The line count. */ + int yy_bs_column; /**< The column count. */ + + /* Whether to try to fill the input buffer when we reach the + * end of it. + */ + int yy_fill_buffer; + + int yy_buffer_status; + +#define YY_BUFFER_NEW 0 +#define YY_BUFFER_NORMAL 1 + /* When an EOF's been seen but there's still some text to process + * then we mark the buffer as YY_EOF_PENDING, to indicate that we + * shouldn't try reading from the input source any more. We might + * still have a bunch of tokens to match, though, because of + * possible backing-up. + * + * When we actually see the EOF, we change the status to "new" + * (via yyrestart()), so that the user can continue scanning by + * just pointing yyin at a new input file. + */ +#define YY_BUFFER_EOF_PENDING 2 + + }; +#endif /* !YY_STRUCT_YY_BUFFER_STATE */ + +/* Stack of input buffers. */ +static size_t yy_buffer_stack_top = 0; /**< index of top of stack. */ +static size_t yy_buffer_stack_max = 0; /**< capacity of stack. */ +static YY_BUFFER_STATE * yy_buffer_stack = NULL; /**< Stack as an array. */ + +/* We provide macros for accessing buffer states in case in the + * future we want to put the buffer states in a more general + * "scanner state". + * + * Returns the top of the stack, or NULL. + */ +#define YY_CURRENT_BUFFER ( (yy_buffer_stack) \ + ? (yy_buffer_stack)[(yy_buffer_stack_top)] \ + : NULL) +/* Same as previous macro, but useful when we know that the buffer stack is not + * NULL or when we need an lvalue. For internal use only. + */ +#define YY_CURRENT_BUFFER_LVALUE (yy_buffer_stack)[(yy_buffer_stack_top)] + +/* yy_hold_char holds the character lost when yytext is formed. */ +static char yy_hold_char; +static int yy_n_chars; /* number of characters read into yy_ch_buf */ +int yyleng; + +/* Points to current character in buffer. */ +static char *yy_c_buf_p = NULL; +static int yy_init = 0; /* whether we need to initialize */ +static int yy_start = 0; /* start state number */ + +/* Flag which is used to allow yywrap()'s to do buffer switches + * instead of setting up a fresh yyin. A bit of a hack ... + */ +static int yy_did_buffer_switch_on_eof; + +void yyrestart ( FILE *input_file ); +void yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer ); +YY_BUFFER_STATE yy_create_buffer ( FILE *file, int size ); +void yy_delete_buffer ( YY_BUFFER_STATE b ); +void yy_flush_buffer ( YY_BUFFER_STATE b ); +void yypush_buffer_state ( YY_BUFFER_STATE new_buffer ); +void yypop_buffer_state ( void ); + +static void yyensure_buffer_stack ( void ); +static void yy_load_buffer_state ( void ); +static void yy_init_buffer ( YY_BUFFER_STATE b, FILE *file ); +#define YY_FLUSH_BUFFER yy_flush_buffer( YY_CURRENT_BUFFER ) + +YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size ); +YY_BUFFER_STATE yy_scan_string ( const char *yy_str ); +YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, int len ); + +void *yyalloc ( yy_size_t ); +void *yyrealloc ( void *, yy_size_t ); +void yyfree ( void * ); + +#define yy_new_buffer yy_create_buffer +#define yy_set_interactive(is_interactive) \ + { \ + if ( ! YY_CURRENT_BUFFER ){ \ + yyensure_buffer_stack (); \ + YY_CURRENT_BUFFER_LVALUE = \ + yy_create_buffer( yyin, YY_BUF_SIZE ); \ + } \ + YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \ + } +#define yy_set_bol(at_bol) \ + { \ + if ( ! YY_CURRENT_BUFFER ){\ + yyensure_buffer_stack (); \ + YY_CURRENT_BUFFER_LVALUE = \ + yy_create_buffer( yyin, YY_BUF_SIZE ); \ + } \ + YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \ + } +#define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol) + +/* Begin user sect3 */ + +#define yywrap() (/*CONSTCOND*/1) +#define YY_SKIP_YYWRAP +typedef flex_uint8_t YY_CHAR; + +FILE *yyin = NULL, *yyout = NULL; + +typedef int yy_state_type; + +extern int yylineno; +int yylineno = 1; + +extern char *yytext; +#ifdef yytext_ptr +#undef yytext_ptr +#endif +#define yytext_ptr yytext + +static yy_state_type yy_get_previous_state ( void ); +static yy_state_type yy_try_NUL_trans ( yy_state_type current_state ); +static int yy_get_next_buffer ( void ); +static void yynoreturn yy_fatal_error ( const char* msg ); + +/* Done after the current pattern has been matched and before the + * corresponding action - sets up yytext. + */ +#define YY_DO_BEFORE_ACTION \ + (yytext_ptr) = yy_bp; \ + yyleng = (int) (yy_cp - yy_bp); \ + (yy_hold_char) = *yy_cp; \ + *yy_cp = '\0'; \ + (yy_c_buf_p) = yy_cp; +#define YY_NUM_RULES 41 +#define YY_END_OF_BUFFER 42 +/* This struct is not used in this scanner, + but its presence is necessary. */ +struct yy_trans_info + { + flex_int32_t yy_verify; + flex_int32_t yy_nxt; + }; +static const flex_int16_t yy_accept[1103] = + { 0, + 0, 0, 42, 40, 1, 1, 40, 10, 40, 10, + 40, 40, 40, 35, 40, 40, 11, 11, 40, 40, + 40, 1, 0, 0, 0, 0, 10, 9, 10, 12, + 0, 0, 10, 10, 0, 11, 0, 35, 4, 34, + 0, 0, 35, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 30, 0, 0, 0, 0, 0, + 0, 0, 39, 11, 0, 11, 11, 11, 8, 7, + 0, 0, 0, 0, 0, 0, 0, 10, 10, 10, + 0, 12, 12, 10, 10, 10, 6, 4, 4, 0, + 33, 0, 21, 0, 33, 0, 18, 19, 0, 33, + + 0, 31, 0, 23, 0, 33, 0, 22, 29, 0, + 25, 24, 20, 0, 33, 0, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 11, 11, 11, + 11, 11, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 10, 10, 10, 0, 0, 12, 12, 10, + 10, 10, 4, 2, 33, 33, 33, 33, 33, 21, + 26, 0, 33, 33, 33, 33, 33, 33, 33, 33, + 18, 19, 33, 0, 33, 33, 33, 33, 33, 33, + + 33, 31, 33, 33, 33, 23, 32, 0, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 22, 29, 33, + 33, 33, 33, 33, 24, 20, 27, 0, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 30, 33, + 33, 5, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 11, 11, 38, 11, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 10, 0, 3, 12, 10, 4, 4, 33, + + 33, 33, 33, 33, 21, 21, 33, 33, 33, 26, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 18, + 19, 18, 28, 0, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 31, 31, 33, 33, 33, 23, + 23, 33, 33, 33, 32, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 22, 29, 22, 33, 33, 33, + 33, 33, 25, 24, 20, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 30, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 25, 33, 33, 33, 30, 30, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 11, 38, + 38, 38, 38, 37, 0, 11, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10, 0, 0, 12, 10, 33, 33, 33, 33, 21, + 21, 21, 21, 33, 33, 33, 26, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 18, 19, + + 18, 18, 18, 19, 19, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 31, 31, 31, 31, 33, 33, 33, 23, + 23, 23, 23, 33, 33, 33, 32, 32, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 22, + 29, 22, 22, 22, 29, 29, 33, 33, 33, 33, + 33, 25, 24, 20, 25, 25, 24, 24, 20, 20, + 33, 33, 33, 27, 33, 33, 33, 33, 33, 33, + 27, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 30, 33, 33, + + 33, 25, 33, 27, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 11, 38, 38, 38, + 38, 38, 38, 38, 38, 0, 38, 37, 38, 38, + 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 10, 12, 10, 33, 33, 33, + 33, 21, 21, 33, 33, 33, 26, 26, 26, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 18, 19, + 18, 33, 33, 33, 28, 33, 33, 33, 33, 33, + + 33, 28, 33, 33, 33, 33, 33, 28, 33, 33, + 33, 31, 31, 33, 33, 33, 23, 23, 33, 33, + 33, 32, 32, 32, 32, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 22, 29, 22, 33, 33, 33, + 33, 33, 25, 24, 20, 33, 33, 33, 27, 27, + 27, 33, 33, 33, 33, 27, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 33, 30, 33, 33, 33, 25, 33, 27, 0, 13, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 14, 0, 0, 0, 0, 11, 38, 36, + 38, 38, 38, 38, 38, 38, 38, 38, 38, 0, + 38, 38, 37, 38, 0, 11, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 10, 12, + 10, 33, 33, 33, 21, 21, 33, 33, 33, 26, + 33, 33, 33, 33, 33, 33, 33, 18, 19, 18, + 33, 33, 33, 28, 28, 28, 33, 33, 33, 33, + 33, 33, 33, 33, 28, 33, 33, 31, 31, 33, + 33, 23, 23, 33, 33, 33, 32, 32, 33, 33, + 33, 33, 33, 33, 33, 22, 29, 22, 33, 33, + + 33, 33, 25, 24, 20, 33, 33, 33, 27, 33, + 33, 33, 27, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 30, 33, 33, 33, 25, 33, + 27, 0, 0, 0, 0, 13, 0, 0, 0, 0, + 0, 0, 0, 15, 0, 0, 0, 0, 0, 14, + 14, 0, 11, 38, 38, 38, 38, 38, 38, 38, + 0, 0, 38, 38, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 33, 21, 21, 33, 33, + 26, 33, 18, 19, 18, 33, 33, 33, 28, 33, + 33, 33, 33, 33, 28, 31, 31, 23, 23, 33, + + 33, 32, 32, 33, 22, 29, 22, 25, 24, 20, + 33, 33, 27, 33, 27, 16, 0, 13, 0, 0, + 0, 0, 0, 15, 15, 0, 0, 38, 38, 38, + 0, 0, 0, 0, 38, 17, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 26, 33, 33, + 28, 33, 32, 32, 27, 0, 13, 13, 0, 0, + 38, 38, 38, 0, 0, 38, 0, 0, 0, 17, + 0, 0, 0, 0, 0, 0, 0, 28, 0, 38, + 38, 38, 0, 38, 0, 17, 0, 0, 0, 0, + 38, 38, 0, 38, 0, 17, 17, 0, 0, 0, + + 0, 0 + } ; + +static const YY_CHAR yy_ec[256] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, + 1, 4, 5, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 6, 7, 8, 9, 10, 11, 10, 12, 13, + 14, 15, 10, 10, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 10, 10, 29, + 30, 31, 10, 32, 33, 34, 35, 36, 37, 38, + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 42, 49, 50, 51, 52, 42, 42, 53, 42, 54, + 10, 55, 10, 10, 42, 10, 56, 57, 58, 59, + + 60, 61, 62, 63, 64, 42, 65, 66, 67, 68, + 69, 70, 42, 71, 72, 73, 74, 42, 42, 75, + 42, 76, 10, 77, 10, 78, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, + 79, 79, 79, 79, 79 + } ; + +static const YY_CHAR yy_meta[80] = + { 0, + 1, 2, 3, 3, 3, 2, 2, 4, 2, 2, + 2, 4, 5, 2, 2, 6, 2, 7, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 2, 2, + 2, 2, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 2, 2, 9 + } ; + +static const flex_int16_t yy_base[1137] = + { 0, + 0, 0, 4195, 7110, 78, 83, 88, 87, 78, 85, + 82, 88, 4161, 142, 4151, 90, 86, 206, 259, 4118, + 4083, 98, 234, 4083, 72, 109, 116, 7110, 318, 100, + 4082, 361, 208, 420, 4017, 92, 463, 205, 4024, 7110, + 3977, 222, 0, 3974, 209, 89, 257, 202, 180, 245, + 248, 259, 355, 272, 3955, 524, 3987, 83, 280, 311, + 274, 585, 7110, 117, 637, 348, 210, 697, 7110, 7110, + 3963, 302, 382, 243, 3940, 3933, 371, 251, 356, 757, + 3945, 101, 817, 246, 357, 877, 7110, 3938, 252, 920, + 3894, 974, 3886, 1017, 397, 447, 3861, 3860, 661, 376, + + 904, 3850, 721, 3841, 448, 451, 1040, 3840, 3825, 1045, + 3813, 3796, 3787, 1063, 569, 600, 3784, 1137, 484, 1180, + 613, 1221, 748, 380, 566, 347, 591, 907, 654, 462, + 3767, 3789, 118, 559, 779, 633, 593, 588, 628, 782, + 654, 3776, 775, 3775, 802, 772, 682, 336, 1279, 345, + 447, 1322, 3754, 254, 695, 837, 658, 581, 632, 656, + 806, 1012, 456, 862, 1365, 3744, 256, 903, 1408, 278, + 964, 1451, 3730, 7110, 3689, 1511, 839, 751, 790, 3655, + 3633, 1086, 865, 3648, 3645, 868, 812, 3627, 850, 3599, + 3559, 3554, 895, 897, 344, 3587, 3580, 885, 900, 892, + + 922, 3526, 1001, 929, 943, 3484, 3483, 1165, 1002, 3500, + 3499, 1037, 1074, 945, 3485, 969, 3476, 3439, 3424, 1096, + 3450, 410, 3442, 438, 3403, 3397, 3396, 1109, 1104, 3416, + 3396, 745, 1554, 1126, 1597, 1199, 1638, 1237, 1696, 1591, + 1451, 1224, 1636, 1495, 1673, 1680, 1762, 1836, 1318, 1740, + 1703, 7110, 573, 1049, 1161, 1158, 970, 987, 1131, 1017, + 1192, 1245, 3385, 3371, 1046, 1269, 1252, 3369, 3358, 1342, + 1600, 1403, 1552, 1788, 1263, 1891, 1796, 1934, 3362, 1868, + 1238, 1307, 1248, 3350, 3304, 1055, 1352, 1779, 1931, 1391, + 1392, 1393, 1983, 3284, 7110, 2026, 2069, 3253, 347, 892, + + 2112, 1392, 1108, 1110, 1248, 1488, 1500, 3239, 3204, 3174, + 1541, 3194, 3177, 2112, 1624, 1119, 3145, 1176, 3142, 1861, + 1873, 1878, 3104, 1704, 1607, 3071, 2999, 1394, 2951, 2939, + 1979, 1503, 1180, 1209, 1896, 1901, 1669, 1259, 1273, 1936, + 1971, 1707, 1278, 1296, 2904, 1691, 2927, 2900, 2022, 1731, + 1336, 2867, 1403, 2856, 1994, 2027, 2063, 1753, 2851, 716, + 2848, 950, 2068, 2106, 2117, 1786, 2836, 2831, 1793, 2833, + 2795, 2150, 2191, 1807, 2234, 2138, 2275, 2156, 2333, 2228, + 2237, 2273, 2344, 2358, 2367, 2378, 2449, 2523, 2189, 2433, + 2413, 1498, 2121, 2020, 2269, 2451, 1769, 2278, 1797, 2271, + + 1426, 1684, 1625, 2322, 1465, 2311, 2342, 2464, 2148, 2457, + 2261, 1528, 2388, 2423, 2759, 1179, 1347, 2149, 2296, 2245, + 2683, 2676, 1875, 1831, 2535, 2548, 1910, 2480, 2177, 2645, + 2637, 2383, 2555, 7110, 2446, 2468, 2600, 2595, 2508, 2546, + 2570, 2561, 2456, 2552, 2533, 2540, 1981, 2561, 2614, 2651, + 2674, 741, 457, 7110, 2729, 2807, 2597, 661, 2134, 2596, + 2579, 1548, 1591, 2330, 2405, 2814, 2604, 1589, 2641, 97, + 2882, 2584, 771, 2942, 3003, 3046, 2605, 1637, 1675, 2688, + 2695, 2537, 1235, 2639, 2556, 2555, 2710, 2662, 2543, 2538, + 2819, 1925, 2402, 2684, 1849, 2508, 1979, 2507, 2715, 2720, + + 2844, 2463, 1448, 2462, 1598, 2832, 2475, 2463, 2837, 2420, + 2412, 2894, 2582, 2374, 2365, 3042, 2023, 2628, 2816, 2853, + 2025, 2109, 2889, 2919, 2312, 1614, 2857, 2229, 2270, 2929, + 3047, 2262, 1615, 2931, 2423, 2439, 3052, 3083, 3071, 2289, + 2269, 3106, 2300, 2407, 3094, 2461, 2233, 2530, 2232, 3119, + 3124, 3129, 2191, 1714, 2183, 1734, 3117, 2186, 1033, 2175, + 1110, 3142, 3147, 3152, 2135, 1803, 2132, 2181, 2103, 2338, + 3076, 2090, 2077, 3160, 3148, 2035, 2003, 3172, 3132, 3134, + 0, 3213, 1864, 3230, 3173, 3254, 3181, 3312, 3378, 3437, + 3511, 3581, 3656, 3723, 3785, 3859, 3933, 3219, 3988, 4050, + + 3236, 2813, 3269, 2888, 2606, 2901, 3277, 3160, 2608, 2666, + 3251, 3162, 3338, 3190, 2881, 3264, 7110, 3191, 3285, 1994, + 1985, 2987, 3302, 3350, 3326, 3303, 3358, 3340, 1965, 1964, + 3341, 3369, 3367, 3038, 3373, 3009, 4107, 3394, 1039, 3416, + 4166, 696, 4225, 3442, 3463, 3481, 3497, 3517, 4285, 4346, + 4407, 3059, 3408, 3193, 1957, 1942, 3295, 3430, 3539, 3544, + 3283, 2258, 3485, 311, 4467, 4510, 4553, 4596, 590, 2669, + 3080, 3327, 3553, 3451, 1920, 1919, 3559, 1876, 2372, 805, + 1897, 1896, 3572, 3260, 3261, 1801, 3311, 1794, 3565, 3614, + 3619, 3505, 1764, 1751, 3626, 3572, 1712, 1704, 3635, 3537, + + 3597, 0, 842, 1683, 1668, 3640, 3543, 0, 962, 3354, + 3355, 3645, 3661, 1143, 3420, 3436, 3672, 3681, 3673, 3477, + 3547, 3701, 3710, 1613, 2478, 1302, 1608, 1607, 3716, 3707, + 3553, 1605, 3554, 1537, 3732, 3737, 3748, 3732, 1529, 1796, + 1502, 1830, 3762, 3770, 3777, 3774, 1498, 1459, 3798, 1394, + 3135, 1339, 1384, 1374, 3811, 0, 4639, 3784, 1740, 3822, + 2122, 3822, 3869, 3895, 3916, 3941, 3970, 3978, 3995, 4699, + 4002, 3880, 4083, 4105, 3722, 0, 3634, 0, 1410, 7110, + 3904, 3810, 1349, 1342, 3782, 3868, 4066, 4635, 3883, 3654, + 3856, 1427, 3908, 1333, 1308, 3944, 4176, 3996, 3768, 3982, + + 1575, 4001, 4071, 4011, 3853, 4024, 1763, 4756, 4088, 7110, + 1487, 3423, 4816, 896, 3208, 4876, 4262, 4444, 4919, 4504, + 4547, 4591, 4677, 4979, 5040, 5101, 3896, 4040, 4023, 1274, + 1266, 3798, 4080, 4509, 4041, 4065, 4639, 1821, 4709, 5144, + 5187, 5230, 3632, 3726, 4208, 4213, 2076, 1260, 1207, 4231, + 1176, 1123, 4793, 3760, 1108, 3762, 1099, 4236, 4322, 4327, + 4093, 1043, 1038, 4332, 994, 3434, 2212, 1028, 1002, 4913, + 992, 951, 4552, 3833, 0, 3946, 3953, 4474, 4479, 4041, + 4059, 4645, 4716, 2284, 4091, 4106, 4721, 4733, 929, 840, + 4919, 4115, 838, 4116, 837, 4738, 4743, 4756, 811, 2240, + + 777, 3157, 4763, 4798, 4803, 2371, 745, 718, 4823, 658, + 646, 5138, 0, 4956, 5225, 5181, 5269, 5279, 5291, 5296, + 5303, 5315, 5386, 5143, 5350, 5369, 5375, 5394, 5408, 5448, + 5413, 642, 4130, 618, 583, 7110, 4253, 5467, 4134, 4243, + 5276, 2587, 4250, 4854, 4277, 4128, 4675, 2903, 4432, 7110, + 526, 2948, 4859, 1544, 4963, 5522, 1357, 3705, 5565, 5608, + 5472, 5669, 5477, 5730, 4466, 5171, 4581, 508, 493, 4435, + 5496, 4865, 4634, 5275, 3053, 5506, 5511, 5565, 478, 449, + 4864, 5767, 5602, 5607, 5772, 3203, 451, 445, 4924, 443, + 442, 5777, 5790, 5795, 5805, 5812, 5835, 5853, 5859, 4140, + + 4164, 4984, 5017, 5866, 5871, 5876, 5881, 5889, 5894, 5899, + 411, 381, 5022, 5907, 5913, 7110, 4866, 5027, 5214, 4678, + 5521, 3400, 4944, 7110, 370, 3468, 3483, 5950, 5993, 6036, + 5987, 6030, 6096, 0, 6139, 7110, 5502, 5131, 4161, 4218, + 4832, 5304, 6035, 5313, 4441, 5307, 3527, 5992, 344, 285, + 5490, 6133, 6073, 6138, 6176, 5382, 7110, 296, 3629, 3675, + 6213, 6256, 6299, 6199, 6342, 6385, 5403, 237, 230, 7110, + 5415, 6212, 6255, 5439, 3976, 5380, 3852, 6293, 3928, 6428, + 6471, 6514, 6557, 6600, 5441, 5531, 5500, 5100, 5886, 4057, + 6643, 6686, 6729, 5612, 5628, 7110, 133, 4058, 6772, 4181, + + 6336, 7110, 6833, 6837, 6846, 6850, 6855, 6864, 6873, 6882, + 6891, 6900, 112, 6904, 6913, 6922, 6931, 6940, 6949, 6958, + 6967, 6976, 6984, 6993, 7002, 7011, 7020, 7029, 7038, 7047, + 7056, 7065, 7074, 7083, 7092, 7100 + } ; + +static const flex_int16_t yy_def[1137] = + { 0, + 1102, 1, 1102, 1102, 1102, 1102, 1102, 1103, 1104, 1105, + 1106, 1102, 1102, 1102, 1102, 1102, 1107, 1107, 1108, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1103, 1102, 1109, 1104, + 1102, 1110, 1105, 1111, 1102, 1107, 1108, 14, 1112, 1102, + 1113, 1102, 14, 1114, 1114, 1114, 1114, 1114, 1114, 1114, + 1114, 1114, 1114, 1114, 1114, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1107, 1115, 1107, 1107, 1107, 1102, 1102, + 1116, 1102, 1102, 1102, 1102, 1102, 1102, 1103, 1103, 1103, + 1117, 1104, 1104, 1105, 1105, 1105, 1102, 1112, 1118, 56, + 1114, 1119, 1114, 1119, 1114, 94, 1114, 1114, 94, 1114, + + 94, 1114, 94, 1114, 94, 1114, 94, 1114, 1114, 94, + 1114, 1114, 1114, 94, 1114, 94, 1114, 1114, 118, 118, + 118, 118, 118, 1114, 1114, 1114, 1114, 1114, 1114, 1114, + 1114, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1107, 68, 1107, + 1107, 68, 1116, 1120, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1103, 1103, 80, 1117, 1121, 1104, 83, 1105, + 1105, 86, 1122, 1102, 1114, 118, 176, 176, 176, 1114, + 1114, 94, 176, 176, 176, 176, 176, 176, 176, 176, + 1114, 1114, 1114, 94, 176, 176, 176, 1114, 176, 176, + + 176, 1114, 176, 176, 176, 1114, 1114, 94, 176, 176, + 176, 1114, 176, 176, 176, 176, 176, 1114, 1114, 176, + 176, 176, 176, 176, 1114, 1114, 1114, 94, 176, 176, + 176, 1114, 118, 233, 233, 233, 233, 233, 233, 239, + 239, 239, 239, 239, 239, 239, 239, 233, 248, 248, + 239, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1107, 68, 1123, 68, 1124, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 80, 1125, 1102, 83, 86, 1122, 1126, 1114, + + 176, 301, 301, 301, 301, 301, 176, 176, 176, 1114, + 301, 301, 301, 301, 301, 301, 301, 301, 301, 301, + 301, 301, 1114, 94, 176, 176, 176, 301, 301, 301, + 301, 301, 301, 301, 301, 301, 301, 301, 301, 301, + 301, 176, 176, 176, 1114, 301, 301, 301, 301, 301, + 301, 301, 301, 301, 301, 301, 301, 301, 301, 301, + 301, 301, 301, 301, 301, 176, 176, 176, 301, 301, + 301, 301, 233, 373, 373, 373, 373, 373, 373, 379, + 379, 379, 379, 379, 379, 379, 379, 373, 388, 388, + 379, 1114, 1114, 1114, 1114, 373, 1114, 1114, 1114, 1114, + + 1114, 1114, 1114, 1114, 1114, 1114, 1114, 1114, 1114, 1114, + 373, 1114, 1114, 373, 1114, 1114, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 276, 1123, + 1123, 1127, 1128, 1102, 1102, 276, 1129, 1130, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1131, 1132, 1133, 1102, 86, 301, 476, 476, 476, 476, + 476, 1114, 1114, 301, 301, 301, 301, 476, 476, 476, + 476, 1114, 1114, 476, 476, 476, 476, 476, 476, 476, + + 476, 1114, 1114, 1114, 1114, 176, 176, 176, 301, 301, + 301, 301, 476, 476, 476, 476, 1114, 1114, 476, 476, + 476, 476, 476, 476, 1114, 1114, 476, 476, 476, 476, + 476, 1114, 1114, 301, 301, 301, 301, 301, 476, 476, + 476, 476, 1114, 1114, 476, 476, 476, 476, 476, 476, + 476, 476, 1114, 1114, 1114, 1114, 476, 476, 476, 476, + 476, 476, 476, 476, 1114, 1114, 1114, 1114, 1114, 1114, + 301, 301, 301, 301, 476, 476, 476, 476, 1114, 1114, + 476, 373, 582, 582, 582, 582, 582, 582, 582, 582, + 582, 582, 582, 582, 590, 590, 582, 582, 582, 590, + + 582, 582, 582, 582, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 276, 1102, 1127, 1102, + 1134, 1128, 1135, 1123, 1123, 1102, 1123, 1123, 1123, 1102, + 456, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1131, 474, 475, 476, 668, 668, + 668, 668, 668, 476, 476, 476, 476, 1114, 1114, 668, + 668, 668, 668, 668, 668, 668, 668, 668, 668, 668, + 668, 301, 301, 301, 301, 476, 476, 476, 476, 1114, + + 1114, 476, 668, 668, 668, 668, 668, 668, 668, 668, + 668, 668, 668, 668, 668, 668, 668, 668, 476, 476, + 476, 476, 476, 1114, 1114, 668, 668, 668, 668, 668, + 668, 668, 668, 668, 668, 668, 668, 668, 668, 668, + 668, 668, 668, 668, 668, 476, 476, 476, 476, 1114, + 1114, 668, 668, 668, 668, 668, 582, 757, 757, 757, + 757, 757, 757, 757, 757, 757, 757, 757, 757, 757, + 757, 757, 757, 757, 757, 757, 757, 757, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 276, 1102, 1102, + 1127, 1127, 1127, 1128, 1128, 1128, 1123, 1123, 649, 1136, + 1136, 1123, 1136, 649, 1102, 651, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1131, 474, + 475, 668, 842, 842, 842, 842, 668, 668, 668, 668, + 842, 842, 842, 842, 842, 842, 842, 842, 842, 842, + 476, 476, 476, 476, 1114, 1114, 668, 668, 668, 668, + 842, 842, 842, 842, 842, 842, 842, 842, 842, 842, + 842, 842, 842, 668, 668, 668, 668, 668, 842, 842, + 842, 842, 842, 842, 842, 842, 842, 842, 842, 842, + + 842, 842, 842, 842, 842, 668, 668, 668, 668, 842, + 842, 842, 842, 1114, 1114, 1114, 1114, 1114, 1114, 1114, + 1114, 1114, 1114, 1114, 1114, 1114, 1114, 1114, 1114, 1114, + 1114, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 826, 1127, 1127, 813, 1128, 1128, 816, 649, + 1136, 1102, 1136, 824, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1114, 1114, 1114, 842, 842, + 842, 1114, 1114, 1114, 1114, 668, 668, 668, 668, 842, + 842, 842, 1114, 1114, 1114, 1114, 1114, 1114, 1114, 842, + + 842, 842, 842, 1114, 1114, 1114, 1114, 1114, 1114, 1114, + 842, 842, 842, 1114, 1114, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 813, 816, 649, + 1136, 1136, 1136, 962, 824, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1114, 842, 842, + 842, 1114, 1114, 1114, 1114, 1102, 1102, 1102, 1102, 1102, + 813, 816, 649, 1136, 1033, 824, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1114, 1102, 813, + 816, 649, 1033, 824, 1102, 1102, 1102, 1102, 1102, 1102, + 813, 816, 1033, 1082, 1102, 1102, 1102, 1102, 1033, 1102, + + 1136, 0, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102 + } ; + +static const flex_int16_t yy_nxt[7190] = + { 0, + 4, 5, 6, 5, 5, 5, 7, 8, 9, 4, + 4, 10, 4, 4, 4, 11, 12, 13, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 15, 4, + 4, 16, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 18, 17, 17, 19, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 18, 17, 17, 20, 21, 17, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 23, + 23, 23, 23, 23, 28, 31, 28, 35, 63, 22, + + 22, 22, 22, 22, 63, 24, 38, 38, 38, 38, + 38, 38, 38, 38, 38, 38, 72, 1102, 1102, 75, + 44, 76, 133, 28, 58, 95, 73, 74, 25, 63, + 59, 75, 32, 76, 60, 1096, 37, 61, 72, 34, + 65, 29, 26, 96, 62, 133, 65, 58, 95, 77, + 253, 25, 40, 59, 32, 32, 60, 41, 42, 61, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, + 29, 65, 77, 253, 44, 44, 45, 46, 47, 44, + 48, 49, 50, 44, 51, 44, 52, 44, 44, 53, + 54, 55, 44, 44, 44, 44, 56, 44, 44, 45, + + 46, 47, 44, 48, 49, 50, 51, 44, 52, 44, + 44, 53, 54, 55, 44, 44, 44, 44, 63, 28, + 44, 1102, 63, 38, 38, 38, 38, 38, 38, 38, + 38, 38, 38, 102, 103, 23, 23, 23, 23, 23, + 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, + 100, 24, 1086, 93, 66, 102, 101, 28, 28, 1086, + 65, 161, 34, 94, 65, 75, 89, 76, 154, 174, + 167, 280, 100, 295, 25, 93, 66, 68, 68, 68, + 68, 68, 68, 68, 68, 68, 68, 106, 26, 28, + 104, 68, 68, 68, 68, 68, 68, 25, 1057, 105, + + 34, 97, 107, 108, 115, 29, 138, 1078, 109, 98, + 106, 99, 104, 110, 68, 68, 68, 68, 68, 68, + 27, 27, 79, 97, 134, 108, 116, 115, 139, 138, + 109, 98, 34, 158, 135, 159, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 134, 136, 63, 155, + 80, 80, 80, 80, 80, 80, 156, 277, 27, 33, + 63, 299, 328, 28, 174, 137, 1078, 329, 28, 330, + 136, 155, 1024, 80, 80, 80, 80, 80, 80, 83, + 83, 83, 83, 83, 83, 83, 83, 83, 83, 111, + 65, 150, 104, 83, 83, 83, 83, 83, 83, 65, + + 157, 105, 65, 1055, 158, 112, 159, 113, 193, 114, + 29, 34, 111, 150, 104, 72, 83, 83, 83, 83, + 83, 83, 33, 33, 85, 73, 160, 112, 100, 113, + 194, 193, 364, 1055, 101, 181, 365, 72, 86, 86, + 86, 86, 86, 86, 86, 86, 86, 86, 160, 67, + 100, 182, 86, 86, 86, 86, 86, 86, 181, 63, + 364, 1052, 1052, 28, 365, 183, 203, 1051, 640, 184, + 204, 185, 205, 1051, 1048, 86, 86, 86, 86, 86, + 86, 68, 68, 68, 68, 68, 68, 68, 68, 68, + 68, 175, 175, 206, 115, 68, 68, 68, 68, 68, + + 68, 65, 234, 1048, 207, 208, 234, 234, 234, 234, + 29, 643, 1043, 175, 175, 206, 116, 115, 68, 68, + 68, 68, 68, 68, 117, 117, 207, 1043, 950, 117, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 118, 119, 119, 119, 120, 121, 122, 123, + 119, 119, 117, 117, 117, 117, 119, 119, 119, 119, + 119, 119, 124, 125, 126, 117, 127, 117, 128, 117, + 117, 129, 130, 131, 117, 117, 117, 117, 117, 119, + 119, 119, 119, 119, 119, 124, 125, 126, 127, 117, + 128, 117, 117, 129, 130, 131, 117, 117, 117, 117, + + 117, 117, 117, 140, 227, 1018, 254, 141, 142, 143, + 144, 262, 843, 255, 844, 263, 288, 264, 229, 102, + 103, 417, 230, 228, 231, 145, 265, 227, 254, 146, + 106, 247, 147, 248, 249, 234, 234, 234, 234, 288, + 1018, 102, 266, 417, 175, 107, 267, 1016, 145, 265, + 268, 146, 269, 106, 147, 149, 149, 149, 149, 149, + 149, 149, 149, 149, 149, 1014, 175, 289, 260, 149, + 149, 149, 149, 149, 149, 458, 287, 1014, 280, 186, + 158, 271, 159, 187, 188, 189, 190, 261, 111, 272, + 289, 260, 149, 149, 149, 149, 149, 149, 67, 67, + + 67, 151, 67, 155, 112, 191, 113, 640, 114, 63, + 156, 111, 272, 192, 138, 152, 152, 152, 152, 152, + 152, 152, 152, 152, 152, 155, 112, 191, 113, 152, + 152, 152, 152, 152, 152, 192, 139, 138, 563, 199, + 1013, 281, 564, 176, 200, 176, 201, 117, 640, 282, + 643, 65, 152, 152, 152, 152, 152, 152, 163, 78, + 78, 164, 163, 281, 28, 175, 247, 1013, 248, 249, + 234, 234, 234, 234, 202, 165, 165, 165, 165, 165, + 165, 165, 165, 165, 165, 473, 305, 175, 295, 165, + 165, 165, 165, 165, 165, 641, 202, 256, 1008, 92, + + 270, 257, 271, 258, 141, 142, 143, 144, 136, 305, + 274, 29, 165, 165, 165, 165, 165, 165, 82, 82, + 82, 168, 82, 259, 290, 306, 137, 851, 75, 852, + 76, 136, 1008, 274, 1102, 169, 169, 169, 169, 169, + 169, 169, 169, 169, 169, 259, 134, 320, 306, 169, + 169, 169, 169, 169, 169, 283, 135, 302, 1006, 1006, + 284, 303, 285, 304, 78, 871, 1004, 872, 134, 28, + 320, 32, 169, 169, 169, 169, 169, 169, 170, 84, + 84, 171, 170, 311, 286, 322, 315, 312, 28, 313, + 316, 317, 318, 319, 175, 172, 172, 172, 172, 172, + + 172, 172, 172, 172, 172, 82, 286, 640, 322, 172, + 172, 172, 172, 172, 172, 325, 29, 193, 332, 326, + 1102, 327, 195, 333, 335, 334, 176, 196, 176, 197, + 323, 34, 172, 172, 172, 172, 172, 172, 119, 194, + 193, 175, 119, 119, 119, 119, 92, 335, 175, 324, + 643, 108, 198, 323, 336, 1004, 109, 32, 117, 117, + 117, 110, 117, 175, 117, 340, 84, 117, 117, 117, + 175, 993, 563, 108, 198, 28, 564, 336, 109, 341, + 355, 117, 117, 117, 117, 876, 117, 877, 340, 117, + 117, 117, 176, 176, 176, 176, 176, 176, 176, 176, + + 176, 176, 341, 355, 357, 425, 176, 176, 176, 176, + 176, 176, 993, 291, 291, 291, 292, 291, 34, 337, + 346, 992, 426, 338, 347, 339, 348, 357, 425, 176, + 176, 176, 176, 176, 176, 177, 176, 176, 176, 178, + 176, 179, 176, 176, 176, 426, 640, 992, 92, 176, + 176, 176, 176, 176, 176, 744, 72, 427, 209, 745, + 989, 180, 210, 213, 211, 989, 73, 214, 215, 216, + 217, 428, 176, 176, 176, 176, 176, 176, 72, 212, + 427, 220, 434, 180, 175, 221, 222, 223, 224, 218, + 207, 208, 350, 641, 219, 418, 351, 352, 353, 354, + + 435, 281, 212, 419, 307, 434, 175, 175, 308, 282, + 309, 218, 207, 225, 358, 226, 219, 418, 359, 360, + 361, 362, 369, 281, 310, 984, 370, 366, 371, 175, + 175, 367, 744, 368, 984, 225, 745, 226, 117, 117, + 117, 232, 117, 480, 374, 481, 982, 310, 374, 374, + 374, 374, 175, 175, 499, 233, 234, 234, 234, 235, + 236, 237, 238, 234, 234, 880, 480, 881, 481, 234, + 234, 234, 234, 234, 234, 175, 424, 499, 254, 420, + 257, 415, 258, 342, 421, 255, 422, 176, 343, 176, + 344, 92, 234, 234, 234, 234, 234, 234, 234, 982, + + 254, 239, 240, 241, 234, 242, 243, 244, 423, 175, + 429, 501, 523, 245, 430, 246, 431, 387, 345, 388, + 389, 374, 374, 374, 374, 399, 399, 399, 400, 399, + 423, 175, 981, 92, 501, 523, 245, 482, 246, 234, + 345, 524, 239, 240, 241, 234, 242, 243, 244, 482, + 482, 482, 483, 482, 250, 387, 251, 388, 389, 374, + 374, 374, 374, 432, 524, 148, 465, 263, 91, 264, + 440, 284, 100, 285, 268, 63, 269, 250, 101, 251, + 148, 148, 148, 275, 148, 981, 459, 436, 971, 92, + 91, 437, 460, 438, 100, 530, 971, 276, 276, 276, + + 276, 276, 276, 276, 276, 276, 276, 439, 459, 531, + 537, 276, 276, 276, 276, 276, 276, 65, 530, 415, + 415, 415, 416, 415, 889, 461, 890, 944, 538, 462, + 439, 463, 531, 537, 276, 276, 276, 276, 276, 276, + 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, + 374, 538, 944, 464, 278, 278, 278, 278, 278, 278, + 442, 910, 938, 911, 141, 142, 143, 144, 640, 938, + 467, 550, 92, 374, 158, 464, 159, 278, 278, 278, + 278, 278, 278, 293, 293, 293, 293, 293, 293, 293, + 293, 293, 293, 912, 550, 291, 605, 293, 293, 293, + + 293, 293, 293, 912, 445, 445, 445, 446, 445, 470, + 477, 643, 513, 75, 478, 76, 479, 514, 605, 515, + 293, 293, 293, 293, 293, 293, 296, 296, 296, 296, + 296, 296, 296, 296, 296, 296, 72, 72, 552, 136, + 296, 296, 296, 296, 296, 296, 73, 73, 92, 257, + 502, 258, 397, 397, 397, 398, 397, 137, 72, 72, + 932, 552, 136, 296, 296, 296, 296, 296, 296, 297, + 297, 297, 297, 297, 297, 297, 297, 297, 297, 102, + 103, 909, 932, 297, 297, 297, 297, 297, 297, 482, + 482, 482, 483, 482, 640, 97, 403, 403, 403, 404, + + 403, 102, 92, 98, 106, 99, 297, 297, 297, 297, + 297, 297, 175, 175, 175, 300, 175, 97, 484, 107, + 909, 520, 485, 903, 486, 98, 521, 106, 522, 301, + 301, 301, 301, 301, 301, 301, 301, 301, 301, 91, + 104, 641, 93, 301, 301, 301, 301, 301, 301, 105, + 903, 640, 94, 447, 447, 447, 448, 447, 897, 488, + 115, 91, 104, 489, 93, 490, 301, 301, 301, 301, + 301, 301, 373, 374, 374, 374, 375, 376, 377, 378, + 374, 374, 116, 115, 138, 659, 374, 374, 374, 374, + 374, 374, 394, 394, 394, 395, 394, 263, 641, 264, + + 504, 443, 443, 443, 444, 443, 139, 138, 659, 374, + 374, 374, 374, 374, 374, 374, 525, 532, 379, 380, + 381, 374, 382, 383, 384, 509, 897, 396, 660, 510, + 385, 511, 386, 891, 891, 91, 155, 401, 401, 401, + 402, 401, 494, 156, 134, 96, 495, 496, 497, 498, + 396, 660, 92, 385, 135, 386, 374, 91, 155, 379, + 380, 381, 374, 382, 383, 384, 134, 92, 92, 92, + 104, 390, 672, 391, 405, 405, 405, 406, 405, 105, + 91, 407, 407, 407, 408, 407, 401, 527, 873, 102, + 103, 528, 104, 529, 390, 672, 391, 392, 392, 392, + + 393, 392, 91, 873, 407, 407, 407, 408, 407, 539, + 673, 102, 106, 540, 374, 541, 553, 91, 374, 374, + 374, 374, 506, 870, 108, 534, 507, 107, 508, 109, + 535, 870, 536, 673, 110, 106, 555, 102, 103, 91, + 93, 405, 405, 405, 406, 405, 108, 108, 175, 545, + 94, 109, 109, 546, 547, 548, 549, 110, 923, 102, + 924, 925, 93, 409, 409, 409, 410, 409, 92, 108, + 175, 557, 374, 864, 109, 558, 559, 560, 561, 106, + 468, 468, 468, 469, 468, 268, 864, 269, 92, 445, + 445, 445, 446, 445, 107, 374, 411, 450, 450, 450, + + 450, 450, 106, 452, 571, 565, 91, 453, 572, 454, + 573, 575, 112, 97, 113, 576, 114, 577, 904, 411, + 859, 98, 905, 99, 136, 583, 155, 859, 91, 583, + 583, 583, 583, 156, 112, 97, 113, 412, 412, 412, + 413, 412, 137, 98, 284, 100, 285, 136, 155, 614, + 455, 101, 904, 257, 374, 258, 905, 92, 374, 374, + 374, 374, 502, 502, 502, 503, 502, 100, 414, 23, + 23, 23, 23, 23, 504, 504, 504, 505, 504, 502, + 502, 502, 503, 502, 689, 24, 757, 757, 757, 757, + 116, 414, 148, 148, 148, 275, 148, 525, 525, 525, + + 526, 525, 525, 525, 525, 526, 525, 689, 25, 449, + 449, 449, 449, 449, 449, 449, 449, 449, 449, 853, + 853, 418, 26, 449, 449, 449, 449, 449, 449, 419, + 92, 25, 468, 468, 468, 469, 468, 532, 532, 532, + 533, 532, 617, 418, 850, 850, 449, 449, 449, 449, + 449, 449, 456, 456, 456, 456, 456, 456, 456, 456, + 456, 456, 834, 181, 618, 617, 456, 456, 456, 456, + 456, 456, 532, 532, 532, 533, 532, 834, 155, 182, + 517, 517, 517, 518, 517, 156, 181, 803, 803, 456, + 456, 456, 456, 456, 456, 553, 553, 553, 554, 553, + + 155, 471, 471, 471, 471, 471, 471, 471, 471, 471, + 471, 519, 797, 138, 691, 471, 471, 471, 471, 471, + 471, 797, 755, 543, 543, 543, 544, 543, 555, 555, + 555, 556, 555, 194, 519, 139, 138, 691, 471, 471, + 471, 471, 471, 471, 474, 474, 474, 474, 474, 474, + 474, 474, 474, 474, 755, 193, 95, 712, 474, 474, + 474, 474, 474, 474, 553, 553, 553, 554, 553, 565, + 565, 565, 566, 565, 96, 207, 208, 194, 193, 95, + 712, 474, 474, 474, 474, 474, 474, 475, 475, 475, + 475, 475, 475, 475, 475, 475, 475, 207, 979, 749, + + 980, 475, 475, 475, 475, 475, 475, 567, 567, 567, + 568, 567, 749, 492, 492, 492, 493, 492, 569, 569, + 569, 570, 569, 392, 475, 475, 475, 475, 475, 475, + 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + 923, 713, 924, 925, 476, 476, 476, 476, 476, 476, + 181, 579, 579, 579, 580, 579, 596, 92, 597, 598, + 583, 583, 583, 583, 713, 93, 182, 476, 476, 476, + 476, 476, 476, 181, 596, 94, 597, 598, 583, 583, + 583, 583, 111, 567, 652, 581, 92, 93, 653, 92, + 415, 415, 415, 416, 415, 623, 743, 606, 112, 430, + + 113, 431, 114, 607, 228, 111, 652, 743, 581, 582, + 583, 583, 583, 584, 585, 586, 587, 583, 583, 606, + 112, 583, 113, 583, 583, 583, 583, 583, 583, 394, + 394, 394, 395, 394, 990, 92, 991, 92, 397, 397, + 397, 398, 397, 92, 583, 92, 583, 583, 583, 583, + 583, 583, 583, 736, 736, 588, 589, 590, 583, 591, + 592, 593, 1009, 612, 601, 717, 1010, 594, 421, 595, + 422, 394, 91, 399, 399, 399, 399, 400, 399, 583, + 397, 97, 96, 583, 583, 583, 583, 601, 717, 98, + 594, 99, 595, 583, 91, 729, 588, 589, 590, 583, + + 591, 592, 593, 97, 281, 95, 718, 1000, 599, 1001, + 600, 98, 282, 405, 608, 729, 92, 91, 609, 100, + 610, 100, 97, 96, 403, 101, 281, 101, 95, 718, + 98, 599, 99, 600, 392, 392, 392, 393, 392, 91, + 569, 100, 611, 100, 97, 401, 401, 401, 402, 401, + 106, 583, 98, 207, 208, 583, 583, 583, 583, 403, + 403, 403, 404, 403, 611, 107, 92, 104, 405, 405, + 405, 406, 405, 106, 678, 207, 105, 93, 459, 407, + 407, 407, 408, 407, 460, 706, 108, 94, 91, 104, + 412, 109, 92, 1011, 706, 1012, 110, 102, 103, 93, + + 459, 625, 91, 104, 492, 263, 106, 264, 108, 543, + 91, 91, 105, 109, 407, 407, 407, 408, 407, 102, + 115, 107, 108, 661, 91, 104, 92, 109, 284, 106, + 285, 699, 110, 91, 405, 405, 405, 406, 405, 699, + 181, 583, 116, 115, 108, 583, 583, 583, 583, 109, + 409, 409, 409, 410, 409, 722, 182, 108, 604, 409, + 207, 208, 109, 181, 628, 583, 407, 110, 629, 583, + 630, 723, 106, 583, 583, 583, 583, 228, 722, 108, + 724, 604, 207, 602, 109, 695, 631, 107, 583, 181, + 437, 111, 438, 91, 723, 106, 735, 695, 619, 112, + + 134, 113, 620, 114, 621, 182, 602, 112, 108, 113, + 135, 114, 181, 109, 111, 91, 92, 92, 110, 735, + 622, 112, 134, 113, 412, 412, 412, 413, 412, 112, + 108, 113, 92, 690, 690, 109, 615, 615, 615, 616, + 615, 583, 445, 622, 434, 583, 583, 583, 583, 615, + 615, 615, 616, 615, 443, 603, 626, 626, 626, 627, + 626, 683, 435, 447, 633, 737, 683, 434, 268, 136, + 269, 634, 634, 634, 635, 634, 136, 116, 603, 636, + 677, 677, 254, 141, 142, 143, 144, 137, 737, 255, + 260, 92, 136, 138, 137, 254, 134, 658, 473, 136, + + 703, 462, 255, 463, 254, 704, 135, 705, 265, 261, + 421, 458, 422, 260, 654, 139, 138, 254, 134, 655, + 632, 656, 664, 669, 266, 632, 158, 670, 159, 671, + 517, 265, 637, 637, 637, 637, 637, 637, 637, 637, + 637, 637, 779, 468, 657, 787, 637, 637, 637, 637, + 637, 637, 450, 450, 450, 450, 450, 674, 452, 624, + 193, 675, 453, 676, 454, 779, 657, 624, 787, 637, + 637, 637, 637, 637, 637, 638, 638, 638, 638, 638, + 680, 1102, 194, 193, 681, 1102, 682, 454, 155, 482, + 482, 482, 483, 482, 613, 156, 482, 482, 482, 483, + + 482, 613, 684, 788, 845, 455, 685, 686, 687, 688, + 155, 678, 678, 678, 679, 678, 502, 502, 502, 503, + 502, 504, 504, 504, 505, 504, 788, 845, 455, 644, + 645, 646, 646, 646, 645, 647, 644, 647, 647, 647, + 644, 644, 648, 647, 647, 647, 647, 649, 649, 649, + 649, 649, 649, 649, 649, 649, 649, 647, 647, 647, + 647, 649, 649, 649, 649, 649, 649, 647, 647, 647, + 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + 647, 647, 647, 650, 649, 649, 649, 649, 649, 649, + 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + + 647, 647, 647, 647, 647, 647, 647, 647, 67, 67, + 67, 151, 67, 92, 578, 662, 662, 662, 663, 662, + 492, 492, 492, 493, 492, 651, 651, 651, 651, 651, + 651, 651, 651, 651, 651, 757, 757, 757, 757, 651, + 651, 651, 651, 651, 651, 502, 502, 502, 503, 502, + 692, 708, 578, 574, 693, 696, 694, 181, 574, 697, + 281, 698, 651, 651, 651, 651, 651, 651, 282, 562, + 324, 709, 562, 182, 708, 714, 710, 551, 711, 715, + 181, 716, 281, 163, 78, 78, 164, 163, 551, 28, + 525, 525, 525, 526, 525, 700, 700, 700, 701, 700, + + 665, 665, 665, 665, 665, 665, 665, 665, 665, 665, + 757, 757, 757, 757, 665, 665, 665, 665, 665, 665, + 525, 525, 525, 526, 525, 430, 542, 431, 254, 702, + 532, 532, 532, 533, 532, 255, 29, 665, 665, 665, + 665, 665, 665, 82, 82, 82, 168, 82, 324, 719, + 254, 780, 702, 542, 720, 781, 721, 30, 92, 516, + 666, 666, 666, 666, 666, 666, 666, 666, 666, 666, + 437, 516, 438, 780, 666, 666, 666, 666, 666, 666, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 32, 666, 666, 666, + + 666, 666, 666, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 512, 617, + 30, 667, 667, 667, 667, 667, 667, 667, 667, 667, + 667, 141, 142, 143, 144, 667, 667, 667, 667, 667, + 667, 618, 617, 517, 517, 517, 518, 517, 532, 532, + 532, 533, 532, 724, 724, 724, 725, 724, 667, 667, + 667, 667, 667, 667, 668, 668, 668, 668, 668, 668, + 668, 668, 668, 668, 707, 462, 265, 463, 668, 668, + 668, 668, 668, 668, 724, 724, 724, 725, 724, 726, + 512, 827, 266, 727, 746, 728, 194, 707, 747, 265, + + 748, 668, 668, 668, 668, 668, 668, 543, 543, 543, + 544, 543, 730, 828, 827, 846, 731, 732, 733, 734, + 553, 553, 553, 554, 553, 555, 555, 555, 556, 555, + 553, 553, 553, 554, 553, 738, 579, 750, 846, 739, + 740, 741, 742, 565, 565, 565, 566, 565, 567, 567, + 567, 568, 567, 569, 569, 569, 570, 569, 92, 207, + 208, 750, 750, 750, 751, 750, 752, 227, 500, 227, + 753, 500, 754, 579, 579, 579, 580, 579, 786, 1009, + 789, 207, 609, 1010, 610, 421, 228, 422, 228, 92, + 227, 770, 227, 771, 772, 757, 757, 757, 757, 770, + + 491, 771, 772, 757, 757, 757, 757, 756, 792, 793, + 642, 833, 257, 794, 258, 795, 655, 491, 656, 640, + 415, 415, 415, 416, 415, 1049, 228, 1050, 92, 487, + 756, 757, 757, 757, 757, 758, 759, 760, 761, 757, + 757, 757, 757, 757, 757, 757, 757, 757, 757, 757, + 757, 762, 763, 764, 757, 765, 766, 767, 757, 757, + 757, 757, 643, 768, 487, 769, 615, 299, 757, 757, + 757, 757, 757, 757, 181, 762, 763, 764, 757, 765, + 766, 767, 854, 855, 856, 857, 768, 773, 769, 774, + 182, 757, 757, 757, 757, 782, 858, 181, 473, 606, + + 783, 838, 784, 796, 778, 607, 284, 620, 285, 621, + 773, 254, 774, 392, 392, 392, 393, 392, 255, 858, + 798, 606, 466, 228, 430, 785, 431, 778, 482, 482, + 482, 483, 482, 254, 757, 757, 757, 757, 260, 790, + 790, 790, 791, 790, 801, 652, 860, 785, 263, 653, + 264, 799, 799, 799, 800, 799, 93, 261, 802, 804, + 626, 260, 629, 437, 630, 438, 94, 652, 466, 860, + 805, 805, 805, 806, 805, 634, 458, 441, 93, 394, + 394, 394, 395, 394, 418, 807, 878, 879, 441, 268, + 427, 269, 419, 260, 433, 638, 638, 638, 638, 638, + + 757, 757, 757, 757, 428, 434, 418, 454, 433, 878, + 879, 265, 261, 427, 775, 372, 260, 809, 809, 809, + 809, 809, 609, 435, 610, 639, 829, 266, 434, 810, + 640, 830, 96, 831, 265, 372, 865, 775, 397, 397, + 397, 398, 397, 638, 638, 638, 638, 638, 835, 1102, + 92, 92, 462, 1102, 463, 454, 882, 92, 832, 757, + 757, 757, 757, 363, 638, 638, 638, 638, 638, 847, + 1102, 363, 883, 848, 1102, 849, 454, 641, 92, 882, + 832, 97, 646, 646, 646, 646, 646, 662, 92, 98, + 620, 99, 621, 92, 454, 883, 455, 356, 638, 638, + + 638, 638, 638, 97, 1102, 629, 356, 630, 1102, 887, + 454, 98, 399, 399, 399, 400, 399, 455, 638, 638, + 638, 638, 638, 861, 1102, 349, 349, 862, 1102, 863, + 454, 281, 887, 757, 757, 757, 757, 92, 92, 282, + 836, 836, 836, 837, 836, 836, 836, 836, 837, 836, + 655, 455, 656, 281, 482, 482, 482, 483, 482, 100, + 678, 678, 678, 679, 678, 101, 502, 502, 502, 503, + 502, 455, 323, 492, 492, 492, 493, 492, 875, 888, + 92, 100, 401, 401, 401, 402, 401, 459, 896, 898, + 867, 324, 459, 460, 868, 323, 869, 324, 460, 700, + + 331, 875, 888, 757, 757, 757, 757, 331, 92, 459, + 181, 896, 898, 92, 459, 504, 504, 504, 505, 504, + 502, 502, 502, 503, 502, 321, 182, 865, 865, 865, + 866, 865, 323, 181, 102, 103, 700, 700, 700, 701, + 700, 517, 517, 517, 518, 517, 525, 525, 525, 526, + 525, 324, 783, 321, 784, 323, 102, 403, 403, 403, + 404, 403, 525, 525, 525, 526, 525, 977, 314, 931, + 708, 314, 874, 532, 532, 532, 533, 532, 757, 757, + 757, 757, 532, 532, 532, 533, 532, 92, 228, 324, + 977, 884, 931, 708, 194, 874, 885, 794, 886, 795, + + 418, 104, 724, 724, 724, 725, 724, 814, 419, 92, + 105, 724, 724, 724, 725, 724, 640, 543, 543, 543, + 544, 543, 418, 104, 405, 405, 405, 406, 405, 892, + 893, 894, 895, 553, 553, 553, 554, 553, 555, 555, + 555, 556, 555, 92, 299, 757, 757, 757, 757, 553, + 553, 553, 554, 553, 899, 900, 901, 902, 167, 643, + 181, 978, 106, 565, 565, 565, 566, 565, 154, 207, + 208, 567, 567, 567, 568, 567, 182, 107, 569, 569, + 569, 570, 569, 181, 978, 106, 407, 407, 407, 408, + 407, 207, 906, 273, 273, 983, 907, 985, 908, 750, + + 750, 750, 751, 750, 252, 915, 916, 917, 427, 918, + 919, 920, 579, 579, 579, 580, 579, 921, 983, 922, + 985, 92, 428, 392, 392, 392, 393, 392, 937, 108, + 827, 427, 780, 783, 109, 784, 781, 91, 92, 110, + 921, 92, 922, 915, 916, 917, 913, 918, 919, 920, + 92, 108, 828, 827, 780, 926, 109, 927, 790, 91, + 409, 409, 409, 410, 409, 228, 93, 92, 995, 913, + 394, 394, 394, 395, 394, 830, 94, 831, 926, 92, + 927, 415, 415, 415, 416, 415, 939, 324, 93, 434, + 609, 995, 610, 776, 92, 92, 397, 397, 397, 398, + + 397, 942, 418, 91, 92, 928, 421, 435, 422, 112, + 419, 113, 434, 114, 92, 92, 776, 399, 399, 399, + 400, 399, 933, 96, 418, 91, 943, 934, 928, 935, + 794, 112, 795, 113, 412, 412, 412, 413, 412, 97, + 92, 965, 401, 401, 401, 402, 401, 98, 92, 99, + 966, 934, 89, 935, 936, 757, 757, 757, 757, 167, + 162, 97, 945, 965, 100, 777, 620, 162, 621, 98, + 101, 403, 403, 403, 404, 403, 936, 154, 996, 405, + 405, 405, 406, 405, 799, 997, 100, 116, 777, 405, + 405, 405, 406, 405, 102, 103, 407, 407, 407, 408, + + 407, 996, 132, 412, 412, 412, 413, 412, 997, 92, + 757, 757, 757, 757, 948, 104, 102, 106, 430, 949, + 431, 965, 427, 629, 105, 630, 805, 106, 92, 952, + 966, 90, 107, 437, 930, 438, 428, 104, 89, 108, + 106, 970, 107, 965, 109, 427, 830, 87, 831, 110, + 106, 407, 407, 407, 408, 407, 116, 930, 967, 975, + 434, 108, 968, 462, 969, 463, 109, 940, 940, 940, + 941, 940, 950, 950, 950, 951, 950, 998, 435, 968, + 1039, 969, 1040, 434, 405, 405, 405, 406, 405, 809, + 809, 809, 809, 809, 108, 999, 81, 71, 972, 109, + + 998, 810, 91, 655, 110, 656, 407, 407, 407, 408, + 407, 986, 70, 459, 606, 987, 108, 988, 999, 460, + 607, 109, 106, 1002, 91, 808, 808, 808, 808, 808, + 808, 808, 808, 808, 808, 459, 606, 107, 1003, 808, + 808, 808, 808, 808, 808, 106, 1002, 69, 1017, 108, + 1005, 1007, 1022, 934, 109, 935, 609, 57, 610, 110, + 617, 1003, 808, 808, 808, 808, 808, 808, 639, 639, + 812, 108, 1053, 1005, 1007, 39, 109, 946, 946, 946, + 947, 946, 618, 617, 813, 813, 813, 813, 813, 813, + 813, 813, 813, 813, 1102, 1053, 1054, 1072, 813, 813, + + 813, 813, 813, 813, 1068, 1102, 1069, 1102, 617, 482, + 482, 482, 483, 482, 482, 482, 482, 483, 482, 1054, + 1072, 813, 813, 813, 813, 813, 813, 642, 642, 815, + 618, 617, 678, 678, 678, 679, 678, 502, 502, 502, + 503, 502, 1102, 816, 816, 816, 816, 816, 816, 816, + 816, 816, 816, 1102, 1073, 1102, 1102, 816, 816, 816, + 816, 816, 816, 638, 638, 638, 638, 638, 1023, 1102, + 1102, 1019, 794, 1102, 795, 454, 783, 1073, 784, 1102, + 816, 816, 816, 816, 816, 816, 817, 817, 817, 818, + 817, 606, 1102, 1102, 1102, 1026, 1102, 607, 454, 620, + + 1102, 621, 1102, 819, 819, 819, 819, 819, 819, 819, + 819, 819, 819, 606, 1102, 1102, 455, 819, 819, 819, + 819, 819, 819, 504, 504, 504, 505, 504, 502, 502, + 502, 503, 502, 865, 865, 865, 866, 865, 1102, 455, + 819, 819, 819, 819, 819, 819, 820, 821, 638, 638, + 638, 821, 822, 820, 822, 822, 822, 820, 820, 823, + 822, 822, 822, 822, 824, 824, 824, 824, 824, 824, + 824, 824, 824, 824, 822, 822, 822, 822, 824, 824, + 824, 824, 824, 824, 822, 822, 822, 822, 822, 822, + 822, 822, 822, 822, 822, 822, 822, 822, 822, 822, + + 825, 824, 824, 824, 824, 824, 824, 822, 822, 822, + 822, 822, 822, 822, 822, 822, 822, 822, 822, 822, + 822, 822, 822, 822, 822, 826, 826, 826, 826, 826, + 826, 826, 826, 826, 826, 1102, 1102, 1102, 1102, 826, + 826, 826, 826, 826, 826, 638, 817, 638, 638, 638, + 1027, 1102, 1102, 1044, 629, 1102, 630, 454, 830, 1102, + 831, 1102, 826, 826, 826, 826, 826, 826, 163, 78, + 78, 164, 163, 827, 28, 525, 525, 525, 526, 525, + 525, 525, 525, 526, 525, 839, 839, 839, 839, 839, + 839, 839, 839, 839, 839, 828, 827, 1102, 455, 839, + + 839, 839, 839, 839, 839, 646, 646, 646, 646, 646, + 973, 973, 973, 974, 973, 1102, 1036, 454, 1102, 1102, + 1037, 29, 839, 839, 839, 839, 839, 839, 840, 840, + 840, 840, 840, 840, 840, 840, 840, 840, 1036, 1102, + 1102, 1102, 840, 840, 840, 840, 840, 840, 638, 638, + 638, 638, 638, 517, 517, 517, 518, 517, 962, 652, + 454, 1102, 1102, 653, 1102, 840, 840, 840, 840, 840, + 840, 841, 841, 841, 841, 841, 841, 841, 841, 841, + 841, 652, 1102, 1102, 994, 841, 841, 841, 841, 841, + 841, 1102, 638, 638, 638, 638, 638, 1102, 1102, 1042, + + 1102, 962, 1102, 968, 454, 969, 194, 994, 841, 841, + 841, 841, 841, 841, 842, 842, 842, 842, 842, 842, + 842, 842, 842, 842, 1102, 1102, 1102, 1102, 842, 842, + 842, 842, 842, 842, 1102, 1102, 940, 940, 940, 941, + 940, 836, 1102, 1102, 1102, 455, 532, 532, 532, 533, + 532, 842, 842, 842, 842, 842, 842, 914, 914, 914, + 914, 914, 914, 914, 914, 914, 914, 1102, 1102, 1102, + 1102, 914, 914, 914, 914, 914, 914, 946, 646, 646, + 646, 646, 646, 606, 652, 1102, 1102, 459, 653, 607, + 454, 1102, 1102, 460, 914, 914, 914, 914, 914, 914, + + 409, 409, 409, 410, 409, 606, 652, 617, 1102, 459, + 163, 78, 78, 164, 163, 1102, 28, 532, 532, 532, + 533, 532, 724, 724, 724, 725, 724, 1102, 780, 618, + 617, 962, 781, 929, 724, 724, 724, 725, 724, 553, + 553, 553, 554, 553, 555, 555, 555, 556, 555, 112, + 780, 113, 1102, 114, 1102, 1102, 929, 553, 553, 553, + 554, 553, 1102, 29, 565, 565, 565, 566, 565, 1102, + 1102, 112, 1102, 113, 953, 953, 953, 953, 953, 953, + 953, 953, 953, 953, 1102, 1102, 1102, 1102, 953, 953, + 953, 953, 953, 953, 492, 492, 492, 493, 492, 567, + + 567, 567, 568, 567, 569, 569, 569, 570, 569, 1102, + 1102, 953, 953, 953, 953, 953, 953, 954, 811, 811, + 955, 954, 1102, 640, 750, 750, 750, 751, 750, 1102, + 1102, 181, 1102, 1102, 956, 956, 956, 956, 956, 956, + 956, 956, 956, 956, 1102, 1102, 1102, 182, 956, 956, + 956, 956, 956, 956, 181, 1024, 1024, 1024, 1025, 1024, + 148, 148, 148, 275, 148, 678, 678, 678, 679, 678, + 641, 956, 956, 956, 956, 956, 956, 957, 814, 814, + 958, 957, 1036, 1047, 1056, 1102, 1037, 640, 655, 934, + 656, 935, 1102, 1102, 959, 959, 959, 959, 959, 959, + + 959, 959, 959, 959, 1036, 1102, 1102, 1102, 959, 959, + 959, 959, 959, 959, 700, 700, 700, 701, 700, 1102, + 543, 543, 543, 544, 543, 865, 865, 865, 866, 865, + 643, 959, 959, 959, 959, 959, 959, 960, 960, 960, + 960, 960, 960, 960, 960, 960, 960, 1102, 875, 1102, + 1102, 960, 960, 960, 960, 960, 960, 117, 117, 117, + 232, 117, 1060, 1102, 1102, 811, 794, 324, 795, 1102, + 640, 875, 207, 208, 960, 960, 960, 960, 960, 960, + 821, 821, 821, 963, 821, 724, 724, 724, 725, 724, + 1102, 1102, 1102, 1102, 207, 1102, 1102, 964, 964, 964, + + 964, 964, 964, 964, 964, 964, 964, 1102, 1102, 1102, + 92, 964, 964, 964, 964, 964, 964, 641, 724, 724, + 724, 725, 724, 750, 750, 750, 751, 750, 1057, 1057, + 1057, 1058, 1057, 1102, 964, 964, 964, 964, 964, 964, + 644, 645, 646, 646, 646, 645, 647, 644, 647, 647, + 647, 644, 644, 648, 647, 647, 647, 647, 649, 649, + 649, 649, 649, 649, 649, 649, 649, 649, 647, 647, + 647, 647, 649, 649, 649, 649, 649, 649, 647, 647, + 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + 647, 647, 647, 647, 650, 649, 649, 649, 649, 649, + + 649, 647, 647, 647, 647, 647, 647, 647, 647, 647, + 647, 647, 647, 647, 647, 647, 647, 647, 647, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 1102, + 1102, 1102, 1102, 64, 64, 64, 64, 64, 64, 579, + 579, 579, 580, 579, 412, 412, 412, 413, 412, 1071, + 1036, 1102, 1102, 1039, 1037, 1040, 64, 64, 64, 64, + 64, 64, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 1036, 1015, 1102, 115, 30, 30, 30, 30, + 30, 30, 394, 394, 394, 395, 394, 1102, 1102, 1038, + 1102, 1102, 228, 1039, 1102, 1040, 1015, 116, 115, 30, + + 30, 30, 30, 30, 30, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 1102, 1041, 95, 1102, 33, + 33, 33, 33, 33, 33, 1102, 392, 392, 392, 393, + 392, 1102, 1059, 1102, 1102, 96, 1102, 783, 1041, 784, + 95, 1102, 33, 33, 33, 33, 33, 33, 976, 976, + 976, 976, 976, 976, 976, 976, 976, 976, 1102, 1102, + 1102, 1102, 976, 976, 976, 976, 976, 976, 1102, 93, + 397, 397, 397, 398, 397, 1102, 1102, 973, 940, 94, + 399, 399, 399, 400, 399, 976, 976, 976, 976, 976, + 976, 93, 401, 401, 401, 402, 401, 403, 403, 403, + + 404, 403, 1102, 1102, 405, 405, 405, 406, 405, 1045, + 1102, 1102, 1102, 97, 1102, 1102, 407, 407, 407, 408, + 407, 98, 1074, 99, 606, 652, 968, 100, 969, 653, + 607, 1077, 1102, 101, 1102, 97, 830, 1102, 831, 827, + 1102, 104, 106, 98, 102, 103, 606, 652, 1102, 100, + 105, 415, 415, 415, 416, 415, 1102, 107, 1102, 108, + 1102, 828, 827, 104, 109, 106, 102, 1102, 1102, 110, + 405, 405, 405, 406, 405, 1102, 407, 407, 407, 408, + 407, 108, 1075, 1102, 1102, 1102, 109, 409, 409, 409, + 410, 409, 1102, 1102, 1102, 117, 117, 117, 232, 117, + + 1079, 1102, 1102, 1102, 92, 934, 1102, 935, 106, 117, + 117, 117, 232, 117, 117, 117, 117, 232, 117, 108, + 111, 1085, 1102, 107, 109, 965, 1068, 1102, 1069, 110, + 1102, 106, 181, 1087, 966, 1102, 112, 1039, 113, 1040, + 114, 108, 1102, 111, 1102, 1102, 109, 965, 182, 117, + 117, 117, 232, 117, 1102, 181, 1102, 1090, 112, 1095, + 113, 968, 92, 969, 1068, 1102, 1069, 92, 1020, 1020, + 1020, 1021, 1020, 646, 646, 646, 646, 646, 638, 821, + 638, 638, 638, 227, 1102, 454, 1102, 1102, 1102, 1102, + 454, 865, 865, 865, 866, 865, 1102, 1045, 1045, 1045, + + 1046, 1045, 228, 1102, 1102, 1102, 227, 175, 175, 175, + 300, 175, 482, 482, 482, 483, 482, 780, 1098, 1102, + 1067, 781, 1039, 1020, 1040, 1068, 962, 1069, 827, 1102, + 1102, 962, 1096, 1096, 1096, 1097, 1096, 1102, 1102, 780, + 1028, 1028, 1028, 1028, 1028, 1028, 1028, 1028, 1028, 1028, + 828, 827, 1070, 1102, 1028, 1028, 1028, 1028, 1028, 1028, + 92, 1102, 1102, 1102, 1102, 92, 482, 482, 482, 483, + 482, 780, 1102, 1102, 1070, 781, 1102, 1028, 1028, 1028, + 1028, 1028, 1028, 1029, 1029, 1029, 1029, 1029, 1029, 1029, + 1029, 1029, 1029, 780, 1102, 1102, 1102, 1029, 1029, 1029, + + 1029, 1029, 1029, 502, 502, 502, 503, 502, 504, 504, + 504, 505, 504, 821, 821, 821, 963, 821, 1102, 92, + 1029, 1029, 1029, 1029, 1029, 1029, 1030, 1030, 1030, 1030, + 1030, 1030, 1030, 1030, 1030, 1030, 1102, 1102, 1102, 1102, + 1030, 1030, 1030, 1030, 1030, 1030, 1100, 1102, 1102, 1102, + 1102, 1068, 1102, 1069, 1102, 1102, 92, 1102, 1102, 1102, + 1102, 92, 1102, 1030, 1030, 1030, 1030, 1030, 1030, 820, + 1031, 646, 646, 646, 1031, 1032, 820, 1032, 1032, 1032, + 820, 820, 823, 1032, 1032, 1032, 1032, 1033, 1033, 1033, + 1033, 1033, 1033, 1033, 1033, 1033, 1033, 1032, 1032, 1032, + + 1032, 1033, 1033, 1033, 1033, 1033, 1033, 1032, 1032, 1032, + 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, + 1032, 1032, 1032, 1034, 1033, 1033, 1033, 1033, 1033, 1033, + 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, + 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1032, 1035, 1035, + 1035, 1035, 1035, 1035, 1035, 1035, 1035, 1035, 1102, 1102, + 1102, 1102, 1035, 1035, 1035, 1035, 1035, 1035, 492, 492, + 492, 493, 492, 502, 502, 502, 503, 502, 700, 700, + 700, 701, 700, 1102, 1102, 1035, 1035, 1035, 1035, 1035, + 1035, 517, 517, 517, 518, 517, 175, 175, 175, 300, + + 175, 1102, 1102, 1102, 1102, 181, 175, 175, 175, 300, + 175, 1102, 995, 525, 525, 525, 526, 525, 1102, 1102, + 1102, 182, 193, 1102, 1102, 1102, 92, 1102, 181, 1102, + 323, 324, 1102, 1102, 1102, 995, 525, 525, 525, 526, + 525, 1102, 1102, 1102, 194, 193, 1102, 1102, 1102, 324, + 1102, 1102, 1102, 323, 532, 532, 532, 533, 532, 92, + 532, 532, 532, 533, 532, 1102, 92, 543, 543, 543, + 544, 543, 553, 553, 553, 554, 553, 555, 555, 555, + 556, 555, 553, 553, 553, 554, 553, 1102, 1088, 92, + 565, 565, 565, 566, 565, 567, 567, 567, 568, 567, + + 569, 569, 569, 570, 569, 1102, 1102, 92, 579, 579, + 579, 580, 579, 92, 175, 175, 175, 300, 175, 207, + 208, 1102, 1102, 1102, 1102, 92, 1102, 1102, 1102, 1102, + 92, 1102, 1102, 1102, 1102, 92, 1036, 1102, 1102, 1102, + 1037, 207, 227, 92, 1102, 1102, 1102, 1102, 92, 1102, + 1102, 1102, 1102, 92, 1102, 1102, 1102, 1102, 1036, 1102, + 1102, 228, 1102, 1102, 1102, 227, 1102, 92, 1061, 1061, + 1061, 1061, 1061, 1061, 1061, 1061, 1061, 1061, 1102, 1102, + 1102, 1102, 1061, 1061, 1061, 1061, 1061, 1061, 646, 646, + 646, 646, 646, 678, 678, 678, 679, 678, 1102, 1102, + + 454, 1102, 1102, 1102, 1102, 1061, 1061, 1061, 1061, 1061, + 1061, 1062, 1062, 1062, 1062, 1062, 1062, 1062, 1062, 1062, + 1062, 1102, 1102, 1102, 1102, 1062, 1062, 1062, 1062, 1062, + 1062, 646, 646, 646, 646, 646, 1075, 1075, 1075, 1076, + 1075, 962, 1102, 454, 1102, 1102, 92, 1102, 1062, 1062, + 1062, 1062, 1062, 1062, 1063, 1063, 1063, 1063, 1063, 1063, + 1063, 1063, 1063, 1063, 1102, 1102, 1102, 1102, 1063, 1063, + 1063, 1063, 1063, 1063, 724, 724, 724, 725, 724, 1102, + 965, 1102, 1102, 1102, 962, 1102, 1102, 1102, 1102, 966, + 1102, 1063, 1063, 1063, 1063, 1063, 1063, 1031, 1031, 1031, + + 1064, 1031, 965, 1102, 1102, 1102, 1102, 1102, 1102, 454, + 1102, 1102, 1102, 1102, 1065, 1065, 1065, 1065, 1065, 1065, + 1065, 1065, 1065, 1065, 1102, 1102, 1102, 92, 1065, 1065, + 1065, 1065, 1065, 1065, 700, 700, 700, 701, 700, 724, + 724, 724, 725, 724, 1102, 1102, 1102, 1102, 1102, 1102, + 962, 1065, 1065, 1065, 1065, 1065, 1065, 1066, 1066, 1066, + 1066, 1066, 1066, 1066, 1066, 1066, 1066, 1102, 323, 1102, + 1102, 1066, 1066, 1066, 1066, 1066, 1066, 750, 750, 750, + 751, 750, 1102, 1102, 1102, 1102, 1102, 324, 1102, 1102, + 1102, 323, 92, 1102, 1066, 1066, 1066, 1066, 1066, 1066, + + 646, 1031, 646, 646, 646, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 454, 1088, 1088, 1088, 1089, 1088, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 92, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, + 1080, 1102, 1102, 1102, 1102, 1080, 1080, 1080, 1080, 1080, + 1080, 1102, 1102, 962, 1102, 1102, 1088, 1088, 1088, 1089, + 1088, 1102, 1036, 1102, 1102, 1102, 1037, 1102, 1080, 1080, + 1080, 1080, 1080, 1080, 1081, 1081, 1081, 1081, 1081, 1081, + 1081, 1081, 1081, 1081, 1036, 1102, 1102, 1102, 1081, 1081, + 1081, 1081, 1081, 1081, 865, 865, 865, 866, 865, 1102, + + 1102, 1102, 1102, 1102, 1102, 1036, 1102, 1102, 1102, 1037, + 1102, 1081, 1081, 1081, 1081, 1081, 1081, 1082, 1082, 1082, + 1082, 1082, 1082, 1082, 1082, 1082, 1082, 1036, 1102, 1102, + 1102, 1082, 1082, 1082, 1082, 1082, 1082, 1031, 1031, 1031, + 1064, 1031, 1102, 1102, 1102, 1102, 1102, 92, 1102, 454, + 1102, 1102, 1102, 1102, 1082, 1082, 1082, 1082, 1082, 1082, + 1083, 1083, 1083, 1083, 1083, 1083, 1083, 1083, 1083, 1083, + 1102, 1102, 1102, 1102, 1083, 1083, 1083, 1083, 1083, 1083, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 962, 1102, 1102, 1102, 1102, 1102, 1102, 1083, 1083, 1083, + + 1083, 1083, 1083, 1084, 1084, 1084, 1084, 1084, 1084, 1084, + 1084, 1084, 1084, 1102, 1102, 1102, 1102, 1084, 1084, 1084, + 1084, 1084, 1084, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1084, 1084, 1084, 1084, 1084, 1084, 1091, 1091, 1091, 1091, + 1091, 1091, 1091, 1091, 1091, 1091, 1102, 1102, 1102, 1102, + 1091, 1091, 1091, 1091, 1091, 1091, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1091, 1091, 1091, 1091, 1091, 1091, 1092, + 1092, 1092, 1092, 1092, 1092, 1092, 1092, 1092, 1092, 1102, + + 1102, 1102, 1102, 1092, 1092, 1092, 1092, 1092, 1092, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1092, 1092, 1092, 1092, + 1092, 1092, 451, 451, 451, 451, 451, 451, 451, 451, + 451, 451, 1102, 1102, 1102, 1102, 451, 451, 451, 451, + 451, 451, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 451, + 451, 451, 451, 451, 451, 1093, 1093, 1093, 1093, 1093, + 1093, 1093, 1093, 1093, 1093, 1102, 1102, 1102, 1102, 1093, + 1093, 1093, 1093, 1093, 1093, 1102, 1102, 1102, 1102, 1102, + + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1093, 1093, 1093, 1093, 1093, 1093, 1094, 1094, + 1094, 1094, 1094, 1094, 1094, 1094, 1094, 1094, 1102, 1102, + 1102, 1102, 1094, 1094, 1094, 1094, 1094, 1094, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1094, 1094, 1094, 1094, 1094, + 1094, 639, 639, 639, 639, 639, 639, 639, 639, 639, + 639, 1102, 1102, 1102, 1102, 639, 639, 639, 639, 639, + 639, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 639, 639, + + 639, 639, 639, 639, 642, 642, 642, 642, 642, 642, + 642, 642, 642, 642, 1102, 1102, 1102, 1102, 642, 642, + 642, 642, 642, 642, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 642, 642, 642, 642, 642, 642, 1099, 1099, 1099, + 1099, 1099, 1099, 1099, 1099, 1099, 1099, 1102, 1102, 1102, + 1102, 1099, 1099, 1099, 1099, 1099, 1099, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1099, 1099, 1099, 1099, 1099, 1099, + 1101, 1101, 1101, 1101, 1101, 1101, 1101, 1101, 1101, 1101, + + 1102, 1102, 1102, 1102, 1101, 1101, 1101, 1101, 1101, 1101, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1101, 1101, 1101, + 1101, 1101, 1101, 27, 27, 1102, 27, 27, 27, 27, + 27, 27, 30, 30, 30, 30, 33, 33, 1102, 33, + 33, 33, 33, 33, 33, 36, 1102, 1102, 36, 64, + 64, 1102, 64, 64, 67, 67, 1102, 67, 67, 67, + 67, 67, 67, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 82, 82, 1102, 82, 82, 82, 82, 82, + 82, 84, 84, 84, 84, 84, 84, 84, 84, 84, + + 88, 88, 88, 88, 88, 88, 88, 88, 88, 91, + 1102, 91, 91, 148, 148, 1102, 148, 148, 148, 148, + 148, 148, 153, 153, 153, 153, 153, 153, 153, 153, + 153, 166, 166, 166, 166, 166, 166, 166, 166, 166, + 173, 173, 173, 173, 173, 173, 173, 173, 173, 175, + 175, 1102, 175, 175, 175, 175, 175, 175, 279, 279, + 279, 279, 279, 279, 279, 279, 279, 294, 294, 294, + 294, 294, 294, 294, 294, 294, 298, 298, 298, 298, + 298, 298, 298, 298, 298, 451, 451, 451, 1102, 451, + 451, 451, 451, 457, 457, 457, 457, 457, 457, 457, + + 457, 457, 472, 472, 472, 472, 472, 472, 472, 472, + 472, 173, 173, 173, 173, 173, 173, 173, 173, 173, + 639, 639, 1102, 639, 639, 639, 639, 639, 639, 642, + 642, 1102, 642, 642, 642, 642, 642, 642, 457, 457, + 457, 457, 457, 457, 457, 457, 457, 279, 279, 279, + 279, 279, 279, 279, 279, 279, 27, 27, 27, 27, + 27, 27, 27, 27, 27, 472, 472, 472, 472, 472, + 472, 472, 472, 472, 294, 294, 294, 294, 294, 294, + 294, 294, 294, 811, 811, 811, 811, 811, 811, 811, + 811, 811, 814, 814, 814, 814, 814, 814, 814, 814, + + 814, 961, 961, 1102, 1102, 961, 961, 961, 961, 3, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102 + } ; + +static const flex_int16_t yy_chk[7190] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, + 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, + 7, 7, 7, 7, 8, 9, 10, 11, 17, 22, + + 22, 22, 22, 22, 36, 7, 12, 12, 12, 12, + 12, 12, 12, 12, 12, 12, 25, 30, 82, 470, + 1113, 470, 58, 27, 16, 46, 25, 26, 7, 64, + 16, 26, 9, 26, 16, 1097, 11, 16, 25, 10, + 17, 8, 7, 46, 16, 58, 36, 16, 46, 26, + 133, 7, 14, 16, 30, 82, 16, 14, 14, 16, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 27, 64, 26, 133, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 18, 33, + 14, 38, 67, 38, 38, 38, 38, 38, 38, 38, + 38, 38, 38, 49, 49, 23, 23, 23, 23, 23, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 48, 23, 1069, 45, 18, 49, 48, 84, 78, 1068, + 18, 74, 33, 45, 67, 74, 89, 74, 154, 89, + 167, 154, 48, 167, 23, 45, 18, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 51, 23, 170, + 50, 19, 19, 19, 19, 19, 19, 23, 1058, 50, + + 84, 47, 51, 52, 54, 78, 61, 1050, 52, 47, + 51, 47, 50, 52, 19, 19, 19, 19, 19, 19, + 29, 29, 29, 47, 59, 52, 54, 54, 61, 61, + 52, 47, 170, 664, 59, 664, 29, 29, 29, 29, + 29, 29, 29, 29, 29, 29, 59, 60, 148, 72, + 29, 29, 29, 29, 29, 29, 72, 150, 79, 85, + 66, 299, 195, 79, 299, 60, 1049, 195, 85, 195, + 60, 72, 1025, 29, 29, 29, 29, 29, 29, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 53, + 148, 66, 126, 32, 32, 32, 32, 32, 32, 150, + + 73, 126, 66, 1012, 73, 53, 73, 53, 100, 53, + 79, 85, 53, 66, 126, 77, 32, 32, 32, 32, + 32, 32, 34, 34, 34, 77, 73, 53, 124, 53, + 100, 100, 222, 1011, 124, 95, 222, 77, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 73, 151, + 124, 95, 34, 34, 34, 34, 34, 34, 95, 151, + 224, 991, 990, 163, 224, 96, 105, 988, 453, 96, + 105, 96, 105, 987, 980, 34, 34, 34, 34, 34, + 34, 37, 37, 37, 37, 37, 37, 37, 37, 37, + 37, 96, 105, 105, 130, 37, 37, 37, 37, 37, + + 37, 151, 119, 979, 106, 106, 119, 119, 119, 119, + 163, 453, 969, 96, 105, 105, 130, 130, 37, 37, + 37, 37, 37, 37, 56, 56, 106, 968, 951, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + + 56, 56, 56, 62, 115, 935, 134, 62, 62, 62, + 62, 137, 669, 134, 669, 137, 158, 137, 116, 125, + 125, 253, 116, 115, 116, 62, 138, 115, 134, 62, + 127, 121, 62, 121, 121, 121, 121, 121, 121, 158, + 934, 125, 138, 253, 116, 127, 139, 932, 62, 138, + 139, 62, 139, 127, 62, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 911, 116, 159, 136, 65, + 65, 65, 65, 65, 65, 458, 157, 910, 458, 99, + 157, 141, 157, 99, 99, 99, 99, 136, 129, 141, + 159, 136, 65, 65, 65, 65, 65, 65, 68, 68, + + 68, 68, 68, 160, 129, 99, 129, 642, 129, 68, + 160, 129, 141, 99, 147, 68, 68, 68, 68, 68, + 68, 68, 68, 68, 68, 160, 129, 99, 129, 68, + 68, 68, 68, 68, 68, 99, 147, 147, 360, 103, + 908, 155, 360, 103, 103, 103, 103, 232, 452, 155, + 642, 68, 68, 68, 68, 68, 68, 68, 80, 80, + 80, 80, 80, 155, 80, 103, 123, 907, 123, 123, + 123, 123, 123, 123, 103, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 473, 178, 103, 473, 80, + 80, 80, 80, 80, 80, 452, 103, 135, 901, 232, + + 140, 135, 143, 135, 140, 140, 140, 140, 146, 178, + 143, 80, 80, 80, 80, 80, 80, 80, 83, 83, + 83, 83, 83, 135, 161, 179, 146, 680, 161, 680, + 161, 146, 899, 143, 83, 83, 83, 83, 83, 83, + 83, 83, 83, 83, 83, 135, 145, 187, 179, 83, + 83, 83, 83, 83, 83, 156, 145, 177, 895, 893, + 156, 177, 156, 177, 164, 703, 890, 703, 145, 164, + 187, 83, 83, 83, 83, 83, 83, 83, 86, 86, + 86, 86, 86, 183, 156, 189, 186, 183, 86, 183, + 186, 186, 186, 186, 300, 86, 86, 86, 86, 86, + + 86, 86, 86, 86, 86, 168, 156, 814, 189, 86, + 86, 86, 86, 86, 86, 194, 164, 198, 199, 194, + 168, 194, 101, 199, 200, 199, 101, 101, 101, 101, + 193, 86, 86, 86, 86, 86, 86, 86, 90, 198, + 198, 194, 90, 90, 90, 90, 300, 200, 101, 193, + 814, 128, 101, 193, 201, 889, 128, 168, 90, 90, + 90, 128, 90, 194, 90, 204, 171, 90, 90, 90, + 101, 872, 362, 128, 101, 171, 362, 201, 128, 205, + 214, 90, 90, 90, 90, 709, 90, 709, 204, 90, + 90, 90, 92, 92, 92, 92, 92, 92, 92, 92, + + 92, 92, 205, 214, 216, 257, 92, 92, 92, 92, + 92, 92, 871, 162, 162, 162, 162, 162, 171, 203, + 209, 869, 258, 203, 209, 203, 209, 216, 257, 92, + 92, 92, 92, 92, 92, 94, 94, 94, 94, 94, + 94, 94, 94, 94, 94, 258, 639, 868, 865, 94, + 94, 94, 94, 94, 94, 559, 162, 260, 107, 559, + 863, 94, 107, 110, 107, 862, 162, 110, 110, 110, + 110, 260, 94, 94, 94, 94, 94, 94, 162, 107, + 260, 114, 265, 94, 107, 114, 114, 114, 114, 110, + 212, 212, 213, 639, 110, 254, 213, 213, 213, 213, + + 265, 286, 107, 254, 182, 265, 107, 114, 182, 286, + 182, 110, 212, 114, 220, 114, 110, 254, 220, 220, + 220, 220, 229, 286, 182, 857, 229, 228, 229, 114, + 182, 228, 561, 228, 855, 114, 561, 114, 118, 118, + 118, 118, 118, 303, 234, 304, 852, 182, 234, 234, + 234, 234, 182, 228, 316, 118, 118, 118, 118, 118, + 118, 118, 118, 118, 118, 714, 303, 714, 304, 118, + 118, 118, 118, 118, 118, 228, 256, 316, 259, 255, + 256, 416, 256, 208, 255, 259, 255, 208, 208, 208, + 208, 118, 118, 118, 118, 118, 118, 118, 120, 851, + + 259, 120, 120, 120, 120, 120, 120, 120, 255, 208, + 261, 318, 333, 120, 261, 120, 261, 236, 208, 236, + 236, 236, 236, 236, 236, 242, 242, 242, 242, 242, + 255, 208, 849, 416, 318, 333, 120, 483, 120, 122, + 208, 334, 122, 122, 122, 122, 122, 122, 122, 305, + 305, 305, 305, 305, 122, 238, 122, 238, 238, 238, + 238, 238, 238, 262, 334, 275, 283, 262, 242, 262, + 267, 283, 242, 283, 267, 275, 267, 122, 242, 122, + 149, 149, 149, 149, 149, 848, 281, 266, 831, 483, + 242, 266, 281, 266, 242, 338, 830, 149, 149, 149, + + 149, 149, 149, 149, 149, 149, 149, 266, 281, 339, + 343, 149, 149, 149, 149, 149, 149, 275, 338, 249, + 249, 249, 249, 249, 726, 282, 726, 795, 344, 282, + 266, 282, 339, 343, 149, 149, 149, 149, 149, 149, + 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, + 249, 344, 794, 282, 152, 152, 152, 152, 152, 152, + 270, 752, 784, 752, 270, 270, 270, 270, 957, 783, + 287, 351, 249, 249, 287, 282, 287, 152, 152, 152, + 152, 152, 152, 165, 165, 165, 165, 165, 165, 165, + 165, 165, 165, 754, 351, 292, 417, 165, 165, 165, + + 165, 165, 165, 753, 272, 272, 272, 272, 272, 290, + 302, 957, 328, 290, 302, 290, 302, 328, 417, 328, + 165, 165, 165, 165, 165, 165, 169, 169, 169, 169, + 169, 169, 169, 169, 169, 169, 291, 292, 353, 272, + 169, 169, 169, 169, 169, 169, 291, 292, 750, 792, + 503, 792, 241, 241, 241, 241, 241, 272, 291, 292, + 779, 353, 272, 169, 169, 169, 169, 169, 169, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 401, + 401, 748, 779, 172, 172, 172, 172, 172, 172, 306, + 306, 306, 306, 306, 811, 241, 244, 244, 244, 244, + + 244, 401, 503, 241, 405, 241, 172, 172, 172, 172, + 172, 172, 176, 176, 176, 176, 176, 241, 307, 405, + 747, 332, 307, 741, 307, 241, 332, 405, 332, 176, + 176, 176, 176, 176, 176, 176, 176, 176, 176, 244, + 244, 811, 392, 176, 176, 176, 176, 176, 176, 244, + 739, 954, 392, 273, 273, 273, 273, 273, 734, 311, + 412, 244, 244, 311, 392, 311, 176, 176, 176, 176, + 176, 176, 233, 233, 233, 233, 233, 233, 233, 233, + 233, 233, 412, 412, 273, 462, 233, 233, 233, 233, + 233, 233, 240, 240, 240, 240, 240, 801, 954, 801, + + 505, 271, 271, 271, 271, 271, 273, 273, 462, 233, + 233, 233, 233, 233, 233, 235, 526, 533, 235, 235, + 235, 235, 235, 235, 235, 325, 732, 240, 463, 325, + 235, 325, 235, 728, 727, 240, 468, 243, 243, 243, + 243, 243, 315, 468, 271, 240, 315, 315, 315, 315, + 240, 463, 505, 235, 271, 235, 237, 240, 468, 237, + 237, 237, 237, 237, 237, 237, 271, 724, 526, 533, + 403, 237, 478, 237, 245, 245, 245, 245, 245, 403, + 243, 246, 246, 246, 246, 246, 402, 337, 705, 243, + 243, 337, 403, 337, 237, 478, 237, 239, 239, 239, + + 239, 239, 243, 704, 251, 251, 251, 251, 251, 346, + 479, 243, 245, 346, 239, 346, 554, 245, 239, 239, + 239, 239, 324, 698, 246, 342, 324, 245, 324, 246, + 342, 697, 342, 479, 246, 245, 556, 402, 402, 245, + 239, 250, 250, 250, 250, 250, 246, 251, 324, 350, + 239, 246, 251, 350, 350, 350, 350, 251, 759, 402, + 759, 759, 239, 247, 247, 247, 247, 247, 554, 251, + 324, 358, 250, 694, 251, 358, 358, 358, 358, 250, + 288, 288, 288, 288, 288, 807, 693, 807, 556, 274, + 274, 274, 274, 274, 250, 250, 247, 277, 277, 277, + + 277, 277, 250, 277, 366, 566, 247, 277, 366, 277, + 366, 369, 247, 397, 247, 369, 247, 369, 740, 247, + 688, 397, 740, 397, 274, 374, 288, 686, 247, 374, + 374, 374, 374, 288, 247, 397, 247, 248, 248, 248, + 248, 248, 274, 397, 838, 399, 838, 274, 288, 424, + 277, 399, 742, 424, 248, 424, 742, 566, 248, 248, + 248, 248, 320, 320, 320, 320, 320, 399, 248, 280, + 280, 280, 280, 280, 321, 321, 321, 321, 321, 322, + 322, 322, 322, 322, 495, 280, 583, 583, 583, 583, + 248, 248, 276, 276, 276, 276, 276, 335, 335, 335, + + 335, 335, 336, 336, 336, 336, 336, 495, 280, 276, + 276, 276, 276, 276, 276, 276, 276, 276, 276, 682, + 681, 423, 280, 276, 276, 276, 276, 276, 276, 423, + 678, 280, 289, 289, 289, 289, 289, 340, 340, 340, + 340, 340, 427, 423, 676, 675, 276, 276, 276, 276, + 276, 276, 278, 278, 278, 278, 278, 278, 278, 278, + 278, 278, 656, 492, 427, 427, 278, 278, 278, 278, + 278, 278, 341, 341, 341, 341, 341, 655, 289, 492, + 331, 331, 331, 331, 331, 289, 492, 630, 629, 278, + 278, 278, 278, 278, 278, 355, 355, 355, 355, 355, + + 289, 293, 293, 293, 293, 293, 293, 293, 293, 293, + 293, 331, 621, 447, 497, 293, 293, 293, 293, 293, + 293, 620, 577, 349, 349, 349, 349, 349, 356, 356, + 356, 356, 356, 331, 331, 447, 447, 497, 293, 293, + 293, 293, 293, 293, 296, 296, 296, 296, 296, 296, + 296, 296, 296, 296, 576, 517, 394, 521, 296, 296, + 296, 296, 296, 296, 357, 357, 357, 357, 357, 363, + 363, 363, 363, 363, 394, 349, 349, 517, 517, 394, + 521, 296, 296, 296, 296, 296, 296, 297, 297, 297, + 297, 297, 297, 297, 297, 297, 297, 349, 847, 573, + + 847, 297, 297, 297, 297, 297, 297, 364, 364, 364, + 364, 364, 572, 314, 314, 314, 314, 314, 365, 365, + 365, 365, 365, 393, 297, 297, 297, 297, 297, 297, + 301, 301, 301, 301, 301, 301, 301, 301, 301, 301, + 761, 522, 761, 761, 301, 301, 301, 301, 301, 301, + 314, 372, 372, 372, 372, 372, 376, 569, 376, 376, + 376, 376, 376, 376, 522, 393, 314, 301, 301, 301, + 301, 301, 301, 314, 378, 393, 378, 378, 378, 378, + 378, 378, 409, 568, 459, 372, 567, 393, 459, 565, + 389, 389, 389, 389, 389, 429, 560, 418, 409, 429, + + 409, 429, 409, 418, 372, 409, 459, 558, 372, 373, + 373, 373, 373, 373, 373, 373, 373, 373, 373, 418, + 409, 389, 409, 373, 373, 373, 373, 373, 373, 380, + 380, 380, 380, 380, 867, 568, 867, 555, 381, 381, + 381, 381, 381, 389, 389, 553, 373, 373, 373, 373, + 373, 373, 375, 549, 547, 375, 375, 375, 375, 375, + 375, 375, 900, 420, 380, 528, 900, 375, 420, 375, + 420, 395, 380, 400, 382, 382, 382, 382, 382, 411, + 398, 381, 380, 411, 411, 411, 411, 380, 528, 381, + 375, 381, 375, 377, 380, 541, 377, 377, 377, 377, + + 377, 377, 377, 381, 662, 395, 529, 884, 377, 884, + 377, 381, 662, 406, 419, 540, 532, 382, 419, 400, + 419, 382, 398, 395, 404, 400, 662, 382, 395, 529, + 398, 377, 398, 377, 379, 379, 379, 379, 379, 382, + 570, 400, 419, 382, 398, 383, 383, 383, 383, 383, + 406, 379, 398, 543, 543, 379, 379, 379, 379, 384, + 384, 384, 384, 384, 419, 406, 525, 404, 385, 385, + 385, 385, 385, 406, 679, 543, 404, 379, 464, 386, + 386, 386, 386, 386, 464, 515, 407, 379, 383, 404, + 413, 407, 570, 906, 514, 906, 407, 383, 383, 379, + + 464, 432, 384, 384, 493, 432, 385, 432, 407, 544, + 383, 385, 384, 407, 391, 391, 391, 391, 391, 383, + 413, 385, 386, 465, 384, 384, 679, 386, 465, 385, + 465, 511, 386, 385, 390, 390, 390, 390, 390, 510, + 493, 414, 413, 413, 386, 414, 414, 414, 414, 386, + 387, 387, 387, 387, 387, 535, 493, 391, 414, 410, + 544, 544, 391, 493, 435, 390, 408, 391, 435, 396, + 435, 536, 390, 396, 396, 396, 396, 414, 535, 391, + 725, 414, 544, 387, 391, 508, 436, 390, 390, 396, + 436, 410, 436, 387, 536, 390, 546, 507, 428, 387, + + 443, 387, 428, 387, 428, 396, 387, 410, 408, 410, + 443, 410, 396, 408, 410, 387, 504, 502, 408, 546, + 428, 387, 443, 387, 388, 388, 388, 388, 388, 410, + 408, 410, 725, 498, 496, 408, 425, 425, 425, 425, + 425, 388, 446, 428, 439, 388, 388, 388, 388, 426, + 426, 426, 426, 426, 444, 388, 433, 433, 433, 433, + 433, 490, 439, 448, 440, 548, 489, 439, 440, 445, + 440, 441, 441, 441, 441, 441, 446, 388, 388, 442, + 486, 485, 425, 442, 442, 442, 442, 445, 548, 425, + 433, 482, 445, 448, 446, 426, 444, 461, 472, 446, + + 513, 461, 426, 461, 425, 513, 444, 513, 441, 433, + 942, 457, 942, 433, 460, 448, 448, 426, 444, 460, + 438, 460, 467, 477, 441, 437, 467, 477, 467, 477, + 518, 441, 449, 449, 449, 449, 449, 449, 449, 449, + 449, 449, 605, 469, 460, 609, 449, 449, 449, 449, + 449, 449, 450, 450, 450, 450, 450, 484, 450, 431, + 518, 484, 450, 484, 450, 605, 460, 430, 609, 449, + 449, 449, 449, 449, 449, 451, 451, 451, 451, 451, + 488, 451, 518, 518, 488, 451, 488, 451, 469, 480, + 480, 480, 480, 480, 422, 469, 481, 481, 481, 481, + + 481, 421, 494, 610, 670, 450, 494, 494, 494, 494, + 469, 487, 487, 487, 487, 487, 499, 499, 499, 499, + 499, 500, 500, 500, 500, 500, 610, 670, 451, 455, + 455, 455, 455, 455, 455, 455, 455, 455, 455, 455, + 455, 455, 455, 455, 455, 455, 455, 455, 455, 455, + 455, 455, 455, 455, 455, 455, 455, 455, 455, 455, + 455, 455, 455, 455, 455, 455, 455, 455, 455, 455, + 455, 455, 455, 455, 455, 455, 455, 455, 455, 455, + 455, 455, 455, 455, 455, 455, 455, 455, 455, 455, + 455, 455, 455, 455, 455, 455, 455, 455, 455, 455, + + 455, 455, 455, 455, 455, 455, 455, 455, 456, 456, + 456, 456, 456, 415, 371, 466, 466, 466, 466, 466, + 491, 491, 491, 491, 491, 456, 456, 456, 456, 456, + 456, 456, 456, 456, 456, 602, 602, 602, 602, 456, + 456, 456, 456, 456, 456, 501, 501, 501, 501, 501, + 506, 519, 370, 368, 506, 509, 506, 491, 367, 509, + 466, 509, 456, 456, 456, 456, 456, 456, 466, 361, + 519, 520, 359, 491, 519, 527, 520, 354, 520, 527, + 491, 527, 466, 471, 471, 471, 471, 471, 352, 471, + 523, 523, 523, 523, 523, 512, 512, 512, 512, 512, + + 471, 471, 471, 471, 471, 471, 471, 471, 471, 471, + 604, 604, 604, 604, 471, 471, 471, 471, 471, 471, + 524, 524, 524, 524, 524, 948, 348, 948, 615, 512, + 530, 530, 530, 530, 530, 615, 471, 471, 471, 471, + 471, 471, 471, 474, 474, 474, 474, 474, 512, 534, + 615, 606, 512, 347, 534, 606, 534, 474, 345, 330, + 474, 474, 474, 474, 474, 474, 474, 474, 474, 474, + 952, 329, 952, 606, 474, 474, 474, 474, 474, 474, + 474, 474, 474, 474, 474, 474, 474, 474, 474, 474, + 474, 474, 474, 474, 474, 474, 474, 474, 474, 474, + + 474, 474, 474, 474, 474, 474, 474, 474, 474, 474, + 474, 474, 474, 474, 474, 474, 474, 474, 327, 622, + 474, 475, 475, 475, 475, 475, 475, 475, 475, 475, + 475, 636, 636, 636, 636, 475, 475, 475, 475, 475, + 475, 622, 622, 516, 516, 516, 516, 516, 531, 531, + 531, 531, 531, 537, 537, 537, 537, 537, 475, 475, + 475, 475, 475, 475, 476, 476, 476, 476, 476, 476, + 476, 476, 476, 476, 516, 975, 634, 975, 476, 476, + 476, 476, 476, 476, 538, 538, 538, 538, 538, 539, + 326, 652, 634, 539, 571, 539, 516, 516, 571, 634, + + 571, 476, 476, 476, 476, 476, 476, 542, 542, 542, + 542, 542, 545, 652, 652, 671, 545, 545, 545, 545, + 550, 550, 550, 550, 550, 551, 551, 551, 551, 551, + 552, 552, 552, 552, 552, 557, 580, 751, 671, 557, + 557, 557, 557, 562, 562, 562, 562, 562, 563, 563, + 563, 563, 563, 564, 564, 564, 564, 564, 323, 542, + 542, 574, 574, 574, 574, 574, 575, 579, 319, 580, + 575, 317, 575, 578, 578, 578, 578, 578, 608, 902, + 612, 542, 608, 902, 608, 612, 579, 612, 580, 751, + 579, 585, 580, 585, 585, 585, 585, 585, 585, 587, + + 313, 587, 587, 587, 587, 587, 587, 578, 614, 618, + 815, 654, 614, 618, 614, 618, 654, 312, 654, 815, + 598, 598, 598, 598, 598, 986, 578, 986, 310, 309, + 578, 582, 582, 582, 582, 582, 582, 582, 582, 582, + 582, 598, 598, 598, 598, 582, 582, 582, 582, 582, + 582, 584, 584, 584, 584, 584, 584, 584, 601, 601, + 601, 601, 815, 584, 308, 584, 616, 298, 582, 582, + 582, 582, 582, 582, 601, 586, 586, 586, 586, 586, + 586, 586, 684, 684, 684, 684, 584, 586, 584, 586, + 601, 603, 603, 603, 603, 607, 685, 601, 294, 611, + + 607, 661, 607, 619, 603, 611, 661, 619, 661, 619, + 586, 616, 586, 588, 588, 588, 588, 588, 616, 685, + 623, 611, 285, 603, 623, 607, 623, 603, 672, 672, + 672, 672, 672, 616, 588, 588, 588, 588, 626, 613, + 613, 613, 613, 613, 625, 657, 687, 607, 625, 657, + 625, 624, 624, 624, 624, 624, 588, 626, 628, 631, + 627, 626, 628, 631, 628, 631, 588, 657, 284, 687, + 632, 632, 632, 632, 632, 635, 279, 269, 588, 589, + 589, 589, 589, 589, 613, 633, 710, 711, 268, 633, + 624, 633, 613, 627, 264, 638, 638, 638, 638, 638, + + 589, 589, 589, 589, 624, 632, 613, 638, 263, 710, + 711, 635, 627, 624, 589, 231, 627, 640, 640, 640, + 640, 640, 1022, 632, 1022, 812, 653, 635, 632, 640, + 812, 653, 589, 653, 635, 230, 866, 589, 590, 590, + 590, 590, 590, 644, 644, 644, 644, 644, 658, 644, + 227, 226, 658, 644, 658, 644, 715, 225, 653, 590, + 590, 590, 590, 223, 645, 645, 645, 645, 645, 674, + 645, 221, 716, 674, 645, 674, 645, 812, 219, 715, + 653, 590, 646, 646, 646, 646, 646, 663, 866, 590, + 1026, 590, 1026, 218, 646, 716, 644, 217, 647, 647, + + 647, 647, 647, 590, 647, 1027, 215, 1027, 647, 720, + 647, 590, 591, 591, 591, 591, 591, 645, 648, 648, + 648, 648, 648, 692, 648, 211, 210, 692, 648, 692, + 648, 663, 720, 591, 591, 591, 591, 207, 206, 663, + 659, 659, 659, 659, 659, 660, 660, 660, 660, 660, + 1047, 647, 1047, 663, 673, 673, 673, 673, 673, 591, + 677, 677, 677, 677, 677, 591, 689, 689, 689, 689, + 689, 648, 700, 683, 683, 683, 683, 683, 707, 721, + 202, 591, 592, 592, 592, 592, 592, 659, 731, 733, + 696, 700, 660, 659, 696, 700, 696, 707, 660, 701, + + 197, 707, 721, 592, 592, 592, 592, 196, 192, 659, + 683, 731, 733, 191, 660, 690, 690, 690, 690, 690, + 691, 691, 691, 691, 691, 190, 683, 695, 695, 695, + 695, 695, 701, 683, 592, 592, 699, 699, 699, 699, + 699, 706, 706, 706, 706, 706, 712, 712, 712, 712, + 712, 701, 1059, 188, 1059, 701, 592, 593, 593, 593, + 593, 593, 713, 713, 713, 713, 713, 843, 185, 777, + 699, 184, 706, 717, 717, 717, 717, 717, 593, 593, + 593, 593, 718, 718, 718, 718, 718, 181, 777, 699, + 843, 719, 777, 699, 706, 706, 719, 1060, 719, 1060, + + 790, 593, 722, 722, 722, 722, 722, 958, 790, 180, + 593, 723, 723, 723, 723, 723, 958, 729, 729, 729, + 729, 729, 790, 593, 594, 594, 594, 594, 594, 730, + 730, 730, 730, 735, 735, 735, 735, 735, 736, 736, + 736, 736, 736, 175, 173, 594, 594, 594, 594, 737, + 737, 737, 737, 737, 738, 738, 738, 738, 166, 958, + 775, 844, 594, 743, 743, 743, 743, 743, 153, 729, + 729, 744, 744, 744, 744, 744, 775, 594, 745, 745, + 745, 745, 745, 775, 844, 594, 595, 595, 595, 595, + 595, 729, 746, 144, 142, 854, 746, 856, 746, 749, + + 749, 749, 749, 749, 132, 758, 758, 758, 799, 758, + 758, 758, 755, 755, 755, 755, 755, 758, 854, 758, + 856, 131, 799, 762, 762, 762, 762, 762, 782, 595, + 832, 799, 785, 782, 595, 782, 785, 595, 117, 595, + 758, 113, 758, 760, 760, 760, 755, 760, 760, 760, + 112, 595, 832, 832, 785, 760, 595, 760, 791, 595, + 596, 596, 596, 596, 596, 755, 762, 111, 874, 755, + 763, 763, 763, 763, 763, 1077, 762, 1077, 760, 109, + 760, 772, 772, 772, 772, 772, 786, 874, 762, 805, + 786, 874, 786, 596, 108, 104, 764, 764, 764, 764, + + 764, 789, 791, 596, 102, 763, 789, 805, 789, 596, + 791, 596, 805, 596, 98, 97, 596, 765, 765, 765, + 765, 765, 781, 763, 791, 596, 793, 781, 763, 781, + 793, 596, 793, 596, 597, 597, 597, 597, 597, 764, + 93, 827, 766, 766, 766, 766, 766, 764, 91, 764, + 827, 1079, 88, 1079, 781, 597, 597, 597, 597, 81, + 76, 764, 796, 827, 765, 597, 796, 75, 796, 764, + 765, 767, 767, 767, 767, 767, 781, 71, 876, 768, + 768, 768, 768, 768, 800, 877, 765, 597, 597, 599, + 599, 599, 599, 599, 766, 766, 769, 769, 769, 769, + + 769, 876, 57, 771, 771, 771, 771, 771, 877, 55, + 599, 599, 599, 599, 798, 767, 766, 768, 798, 802, + 798, 1075, 800, 802, 767, 802, 806, 599, 44, 804, + 1075, 41, 768, 804, 771, 804, 800, 767, 39, 769, + 768, 829, 599, 1075, 769, 800, 829, 35, 829, 769, + 599, 600, 600, 600, 600, 600, 771, 771, 828, 835, + 806, 769, 828, 835, 828, 835, 769, 787, 787, 787, + 787, 787, 803, 803, 803, 803, 803, 880, 806, 1090, + 1098, 1090, 1098, 806, 773, 773, 773, 773, 773, 809, + 809, 809, 809, 809, 600, 881, 31, 24, 833, 600, + + 880, 809, 600, 833, 600, 833, 774, 774, 774, 774, + 774, 861, 21, 836, 787, 861, 600, 861, 881, 836, + 787, 600, 773, 885, 600, 637, 637, 637, 637, 637, + 637, 637, 637, 637, 637, 836, 787, 773, 886, 637, + 637, 637, 637, 637, 637, 773, 885, 20, 933, 774, + 892, 894, 939, 933, 774, 933, 939, 15, 939, 774, + 946, 886, 637, 637, 637, 637, 637, 637, 641, 641, + 641, 774, 1000, 892, 894, 13, 774, 797, 797, 797, + 797, 797, 946, 946, 641, 641, 641, 641, 641, 641, + 641, 641, 641, 641, 3, 1000, 1001, 1039, 641, 641, + + 641, 641, 641, 641, 1100, 0, 1100, 0, 797, 845, + 845, 845, 845, 845, 846, 846, 846, 846, 846, 1001, + 1039, 641, 641, 641, 641, 641, 641, 643, 643, 643, + 797, 797, 850, 850, 850, 850, 850, 858, 858, 858, + 858, 858, 0, 643, 643, 643, 643, 643, 643, 643, + 643, 643, 643, 0, 1040, 0, 0, 643, 643, 643, + 643, 643, 643, 817, 817, 817, 817, 817, 943, 817, + 0, 937, 943, 817, 943, 817, 937, 1040, 937, 0, + 643, 643, 643, 643, 643, 643, 649, 649, 649, 649, + 649, 940, 649, 0, 0, 945, 649, 940, 649, 945, + + 0, 945, 0, 649, 649, 649, 649, 649, 649, 649, + 649, 649, 649, 940, 0, 0, 817, 649, 649, 649, + 649, 649, 649, 859, 859, 859, 859, 859, 860, 860, + 860, 860, 860, 864, 864, 864, 864, 864, 0, 649, + 649, 649, 649, 649, 649, 649, 650, 650, 650, 650, + 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, + 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, + 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, + 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, + 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, + + 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, + 650, 650, 650, 650, 650, 650, 650, 650, 650, 650, + 650, 650, 650, 650, 650, 651, 651, 651, 651, 651, + 651, 651, 651, 651, 651, 0, 0, 0, 0, 651, + 651, 651, 651, 651, 651, 818, 818, 818, 818, 818, + 949, 818, 0, 970, 949, 818, 949, 818, 970, 0, + 970, 0, 651, 651, 651, 651, 651, 651, 665, 665, + 665, 665, 665, 1045, 665, 878, 878, 878, 878, 878, + 879, 879, 879, 879, 879, 665, 665, 665, 665, 665, + 665, 665, 665, 665, 665, 1045, 1045, 0, 818, 665, + + 665, 665, 665, 665, 665, 820, 820, 820, 820, 820, + 834, 834, 834, 834, 834, 0, 965, 820, 0, 0, + 965, 665, 665, 665, 665, 665, 665, 665, 666, 666, + 666, 666, 666, 666, 666, 666, 666, 666, 965, 0, + 0, 0, 666, 666, 666, 666, 666, 666, 821, 821, + 821, 821, 821, 873, 873, 873, 873, 873, 820, 834, + 821, 0, 0, 834, 0, 666, 666, 666, 666, 666, + 666, 667, 667, 667, 667, 667, 667, 667, 667, 667, + 667, 834, 0, 0, 873, 667, 667, 667, 667, 667, + 667, 0, 822, 822, 822, 822, 822, 0, 822, 967, + + 0, 821, 822, 967, 822, 967, 873, 873, 667, 667, + 667, 667, 667, 667, 668, 668, 668, 668, 668, 668, + 668, 668, 668, 668, 0, 0, 0, 0, 668, 668, + 668, 668, 668, 668, 0, 0, 788, 788, 788, 788, + 788, 837, 0, 0, 0, 822, 882, 882, 882, 882, + 882, 668, 668, 668, 668, 668, 668, 757, 757, 757, + 757, 757, 757, 757, 757, 757, 757, 0, 0, 0, + 0, 757, 757, 757, 757, 757, 757, 947, 823, 823, + 823, 823, 823, 788, 973, 0, 0, 837, 973, 788, + 823, 0, 0, 837, 757, 757, 757, 757, 757, 757, + + 770, 770, 770, 770, 770, 788, 973, 947, 0, 837, + 839, 839, 839, 839, 839, 0, 839, 883, 883, 883, + 883, 883, 887, 887, 887, 887, 887, 0, 1020, 947, + 947, 823, 1020, 770, 888, 888, 888, 888, 888, 896, + 896, 896, 896, 896, 897, 897, 897, 897, 897, 770, + 1020, 770, 0, 770, 0, 0, 770, 898, 898, 898, + 898, 898, 0, 839, 903, 903, 903, 903, 903, 0, + 0, 770, 0, 770, 808, 808, 808, 808, 808, 808, + 808, 808, 808, 808, 0, 0, 0, 0, 808, 808, + 808, 808, 808, 808, 853, 853, 853, 853, 853, 904, + + 904, 904, 904, 904, 905, 905, 905, 905, 905, 0, + 0, 808, 808, 808, 808, 808, 808, 813, 813, 813, + 813, 813, 0, 813, 909, 909, 909, 909, 909, 0, + 0, 853, 0, 0, 813, 813, 813, 813, 813, 813, + 813, 813, 813, 813, 0, 0, 0, 853, 813, 813, + 813, 813, 813, 813, 853, 944, 944, 944, 944, 944, + 953, 953, 953, 953, 953, 981, 981, 981, 981, 981, + 813, 813, 813, 813, 813, 813, 813, 816, 816, 816, + 816, 816, 1041, 972, 1017, 0, 1041, 816, 972, 1017, + 972, 1017, 0, 0, 816, 816, 816, 816, 816, 816, + + 816, 816, 816, 816, 1041, 0, 0, 0, 816, 816, + 816, 816, 816, 816, 870, 870, 870, 870, 870, 0, + 891, 891, 891, 891, 891, 989, 989, 989, 989, 989, + 816, 816, 816, 816, 816, 816, 816, 819, 819, 819, + 819, 819, 819, 819, 819, 819, 819, 0, 870, 0, + 0, 819, 819, 819, 819, 819, 819, 914, 914, 914, + 914, 914, 1023, 0, 0, 955, 1023, 870, 1023, 0, + 955, 870, 891, 891, 819, 819, 819, 819, 819, 819, + 824, 824, 824, 824, 824, 1002, 1002, 1002, 1002, 1002, + 0, 0, 0, 0, 891, 0, 0, 824, 824, 824, + + 824, 824, 824, 824, 824, 824, 824, 0, 0, 0, + 914, 824, 824, 824, 824, 824, 824, 955, 1003, 1003, + 1003, 1003, 1003, 1013, 1013, 1013, 1013, 1013, 1018, 1018, + 1018, 1018, 1018, 0, 824, 824, 824, 824, 824, 824, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, + + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 826, + 826, 826, 826, 826, 826, 826, 826, 826, 826, 0, + 0, 0, 0, 826, 826, 826, 826, 826, 826, 912, + 912, 912, 912, 912, 924, 924, 924, 924, 924, 1038, + 1088, 0, 0, 1038, 1088, 1038, 826, 826, 826, 826, + 826, 826, 840, 840, 840, 840, 840, 840, 840, 840, + 840, 840, 1088, 912, 0, 924, 840, 840, 840, 840, + 840, 840, 916, 916, 916, 916, 916, 0, 0, 966, + 0, 0, 912, 966, 0, 966, 912, 924, 924, 840, + + 840, 840, 840, 840, 840, 841, 841, 841, 841, 841, + 841, 841, 841, 841, 841, 0, 966, 916, 0, 841, + 841, 841, 841, 841, 841, 0, 915, 915, 915, 915, + 915, 0, 1019, 0, 0, 916, 0, 1019, 966, 1019, + 916, 0, 841, 841, 841, 841, 841, 841, 842, 842, + 842, 842, 842, 842, 842, 842, 842, 842, 0, 0, + 0, 0, 842, 842, 842, 842, 842, 842, 0, 915, + 917, 917, 917, 917, 917, 0, 0, 974, 941, 915, + 918, 918, 918, 918, 918, 842, 842, 842, 842, 842, + 842, 915, 919, 919, 919, 919, 919, 920, 920, 920, + + 920, 920, 0, 0, 921, 921, 921, 921, 921, 1046, + 0, 0, 0, 917, 0, 0, 922, 922, 922, 922, + 922, 917, 1042, 917, 941, 974, 1042, 918, 1042, 974, + 941, 1044, 0, 918, 0, 917, 1044, 0, 1044, 1046, + 0, 920, 921, 917, 919, 919, 941, 974, 0, 918, + 920, 925, 925, 925, 925, 925, 0, 921, 0, 922, + 0, 1046, 1046, 920, 922, 921, 919, 0, 0, 922, + 926, 926, 926, 926, 926, 0, 927, 927, 927, 927, + 927, 922, 1076, 0, 0, 0, 922, 923, 923, 923, + 923, 923, 0, 0, 0, 928, 928, 928, 928, 928, + + 1056, 0, 0, 0, 925, 1056, 0, 1056, 926, 929, + 929, 929, 929, 929, 931, 931, 931, 931, 931, 927, + 923, 1067, 0, 926, 927, 1076, 1067, 0, 1067, 927, + 0, 926, 928, 1071, 1076, 0, 923, 1071, 923, 1071, + 923, 927, 0, 923, 0, 0, 927, 1076, 928, 930, + 930, 930, 930, 930, 0, 928, 0, 1074, 923, 1085, + 923, 1074, 929, 1074, 1085, 0, 1085, 931, 938, 938, + 938, 938, 938, 961, 961, 961, 961, 961, 963, 963, + 963, 963, 963, 930, 0, 961, 0, 0, 0, 0, + 963, 1051, 1051, 1051, 1051, 1051, 0, 971, 971, 971, + + 971, 971, 930, 0, 0, 0, 930, 976, 976, 976, + 976, 976, 977, 977, 977, 977, 977, 938, 1087, 0, + 1037, 938, 1087, 1021, 1087, 1037, 961, 1037, 971, 0, + 0, 963, 1086, 1086, 1086, 1086, 1086, 0, 0, 938, + 956, 956, 956, 956, 956, 956, 956, 956, 956, 956, + 971, 971, 1037, 0, 956, 956, 956, 956, 956, 956, + 976, 0, 0, 0, 0, 977, 978, 978, 978, 978, + 978, 1021, 0, 0, 1037, 1021, 0, 956, 956, 956, + 956, 956, 956, 959, 959, 959, 959, 959, 959, 959, + 959, 959, 959, 1021, 0, 0, 0, 959, 959, 959, + + 959, 959, 959, 983, 983, 983, 983, 983, 984, 984, + 984, 984, 984, 1094, 1094, 1094, 1094, 1094, 0, 978, + 959, 959, 959, 959, 959, 959, 960, 960, 960, 960, + 960, 960, 960, 960, 960, 960, 0, 0, 0, 0, + 960, 960, 960, 960, 960, 960, 1095, 0, 0, 0, + 0, 1095, 0, 1095, 0, 0, 983, 0, 0, 0, + 0, 984, 0, 960, 960, 960, 960, 960, 960, 962, + 962, 962, 962, 962, 962, 962, 962, 962, 962, 962, + 962, 962, 962, 962, 962, 962, 962, 962, 962, 962, + 962, 962, 962, 962, 962, 962, 962, 962, 962, 962, + + 962, 962, 962, 962, 962, 962, 962, 962, 962, 962, + 962, 962, 962, 962, 962, 962, 962, 962, 962, 962, + 962, 962, 962, 962, 962, 962, 962, 962, 962, 962, + 962, 962, 962, 962, 962, 962, 962, 962, 962, 962, + 962, 962, 962, 962, 962, 962, 962, 962, 964, 964, + 964, 964, 964, 964, 964, 964, 964, 964, 0, 0, + 0, 0, 964, 964, 964, 964, 964, 964, 982, 982, + 982, 982, 982, 985, 985, 985, 985, 985, 992, 992, + 992, 992, 992, 0, 0, 964, 964, 964, 964, 964, + 964, 993, 993, 993, 993, 993, 994, 994, 994, 994, + + 994, 0, 0, 0, 0, 982, 995, 995, 995, 995, + 995, 0, 992, 996, 996, 996, 996, 996, 0, 0, + 0, 982, 993, 0, 0, 0, 985, 0, 982, 0, + 994, 992, 0, 0, 0, 992, 997, 997, 997, 997, + 997, 0, 0, 0, 993, 993, 0, 0, 0, 994, + 0, 0, 0, 994, 998, 998, 998, 998, 998, 995, + 999, 999, 999, 999, 999, 0, 996, 1004, 1004, 1004, + 1004, 1004, 1005, 1005, 1005, 1005, 1005, 1006, 1006, 1006, + 1006, 1006, 1007, 1007, 1007, 1007, 1007, 0, 1089, 997, + 1008, 1008, 1008, 1008, 1008, 1009, 1009, 1009, 1009, 1009, + + 1010, 1010, 1010, 1010, 1010, 0, 0, 998, 1014, 1014, + 1014, 1014, 1014, 999, 1015, 1015, 1015, 1015, 1015, 1004, + 1004, 0, 0, 0, 0, 1005, 0, 0, 0, 0, + 1006, 0, 0, 0, 0, 1007, 1089, 0, 0, 0, + 1089, 1004, 1014, 1008, 0, 0, 0, 0, 1009, 0, + 0, 0, 0, 1010, 0, 0, 0, 0, 1089, 0, + 0, 1014, 0, 0, 0, 1014, 0, 1015, 1028, 1028, + 1028, 1028, 1028, 1028, 1028, 1028, 1028, 1028, 0, 0, + 0, 0, 1028, 1028, 1028, 1028, 1028, 1028, 1031, 1031, + 1031, 1031, 1031, 1048, 1048, 1048, 1048, 1048, 0, 0, + + 1031, 0, 0, 0, 0, 1028, 1028, 1028, 1028, 1028, + 1028, 1029, 1029, 1029, 1029, 1029, 1029, 1029, 1029, 1029, + 1029, 0, 0, 0, 0, 1029, 1029, 1029, 1029, 1029, + 1029, 1032, 1032, 1032, 1032, 1032, 1043, 1043, 1043, 1043, + 1043, 1031, 0, 1032, 0, 0, 1048, 0, 1029, 1029, + 1029, 1029, 1029, 1029, 1030, 1030, 1030, 1030, 1030, 1030, + 1030, 1030, 1030, 1030, 0, 0, 0, 0, 1030, 1030, + 1030, 1030, 1030, 1030, 1053, 1053, 1053, 1053, 1053, 0, + 1043, 0, 0, 0, 1032, 0, 0, 0, 0, 1043, + 0, 1030, 1030, 1030, 1030, 1030, 1030, 1033, 1033, 1033, + + 1033, 1033, 1043, 0, 0, 0, 0, 0, 0, 1033, + 0, 0, 0, 0, 1033, 1033, 1033, 1033, 1033, 1033, + 1033, 1033, 1033, 1033, 0, 0, 0, 1053, 1033, 1033, + 1033, 1033, 1033, 1033, 1052, 1052, 1052, 1052, 1052, 1054, + 1054, 1054, 1054, 1054, 0, 0, 0, 0, 0, 0, + 1033, 1033, 1033, 1033, 1033, 1033, 1033, 1035, 1035, 1035, + 1035, 1035, 1035, 1035, 1035, 1035, 1035, 0, 1052, 0, + 0, 1035, 1035, 1035, 1035, 1035, 1035, 1055, 1055, 1055, + 1055, 1055, 0, 0, 0, 0, 0, 1052, 0, 0, + 0, 1052, 1054, 0, 1035, 1035, 1035, 1035, 1035, 1035, + + 1064, 1064, 1064, 1064, 1064, 0, 0, 0, 0, 0, + 0, 0, 1064, 1072, 1072, 1072, 1072, 1072, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1055, 1061, 1061, 1061, 1061, 1061, 1061, 1061, 1061, 1061, + 1061, 0, 0, 0, 0, 1061, 1061, 1061, 1061, 1061, + 1061, 0, 0, 1064, 0, 0, 1073, 1073, 1073, 1073, + 1073, 0, 1072, 0, 0, 0, 1072, 0, 1061, 1061, + 1061, 1061, 1061, 1061, 1062, 1062, 1062, 1062, 1062, 1062, + 1062, 1062, 1062, 1062, 1072, 0, 0, 0, 1062, 1062, + 1062, 1062, 1062, 1062, 1078, 1078, 1078, 1078, 1078, 0, + + 0, 0, 0, 0, 0, 1073, 0, 0, 0, 1073, + 0, 1062, 1062, 1062, 1062, 1062, 1062, 1063, 1063, 1063, + 1063, 1063, 1063, 1063, 1063, 1063, 1063, 1073, 0, 0, + 0, 1063, 1063, 1063, 1063, 1063, 1063, 1101, 1101, 1101, + 1101, 1101, 0, 0, 0, 0, 0, 1078, 0, 1101, + 0, 0, 0, 0, 1063, 1063, 1063, 1063, 1063, 1063, + 1065, 1065, 1065, 1065, 1065, 1065, 1065, 1065, 1065, 1065, + 0, 0, 0, 0, 1065, 1065, 1065, 1065, 1065, 1065, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1101, 0, 0, 0, 0, 0, 0, 1065, 1065, 1065, + + 1065, 1065, 1065, 1066, 1066, 1066, 1066, 1066, 1066, 1066, + 1066, 1066, 1066, 0, 0, 0, 0, 1066, 1066, 1066, + 1066, 1066, 1066, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1066, 1066, 1066, 1066, 1066, 1066, 1080, 1080, 1080, 1080, + 1080, 1080, 1080, 1080, 1080, 1080, 0, 0, 0, 0, + 1080, 1080, 1080, 1080, 1080, 1080, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1080, 1080, 1080, 1080, 1080, 1080, 1081, + 1081, 1081, 1081, 1081, 1081, 1081, 1081, 1081, 1081, 0, + + 0, 0, 0, 1081, 1081, 1081, 1081, 1081, 1081, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1081, 1081, 1081, 1081, + 1081, 1081, 1082, 1082, 1082, 1082, 1082, 1082, 1082, 1082, + 1082, 1082, 0, 0, 0, 0, 1082, 1082, 1082, 1082, + 1082, 1082, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, + 1082, 1082, 1082, 1082, 1082, 1083, 1083, 1083, 1083, 1083, + 1083, 1083, 1083, 1083, 1083, 0, 0, 0, 0, 1083, + 1083, 1083, 1083, 1083, 1083, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1083, 1083, 1083, 1083, 1083, 1083, 1084, 1084, + 1084, 1084, 1084, 1084, 1084, 1084, 1084, 1084, 0, 0, + 0, 0, 1084, 1084, 1084, 1084, 1084, 1084, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1084, 1084, 1084, 1084, 1084, + 1084, 1091, 1091, 1091, 1091, 1091, 1091, 1091, 1091, 1091, + 1091, 0, 0, 0, 0, 1091, 1091, 1091, 1091, 1091, + 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1091, 1091, + + 1091, 1091, 1091, 1091, 1092, 1092, 1092, 1092, 1092, 1092, + 1092, 1092, 1092, 1092, 0, 0, 0, 0, 1092, 1092, + 1092, 1092, 1092, 1092, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1092, 1092, 1092, 1092, 1092, 1092, 1093, 1093, 1093, + 1093, 1093, 1093, 1093, 1093, 1093, 1093, 0, 0, 0, + 0, 1093, 1093, 1093, 1093, 1093, 1093, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1093, 1093, 1093, 1093, 1093, 1093, + 1099, 1099, 1099, 1099, 1099, 1099, 1099, 1099, 1099, 1099, + + 0, 0, 0, 0, 1099, 1099, 1099, 1099, 1099, 1099, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1099, 1099, 1099, + 1099, 1099, 1099, 1103, 1103, 0, 1103, 1103, 1103, 1103, + 1103, 1103, 1104, 1104, 1104, 1104, 1105, 1105, 0, 1105, + 1105, 1105, 1105, 1105, 1105, 1106, 0, 0, 1106, 1107, + 1107, 0, 1107, 1107, 1108, 1108, 0, 1108, 1108, 1108, + 1108, 1108, 1108, 1109, 1109, 1109, 1109, 1109, 1109, 1109, + 1109, 1109, 1110, 1110, 0, 1110, 1110, 1110, 1110, 1110, + 1110, 1111, 1111, 1111, 1111, 1111, 1111, 1111, 1111, 1111, + + 1112, 1112, 1112, 1112, 1112, 1112, 1112, 1112, 1112, 1114, + 0, 1114, 1114, 1115, 1115, 0, 1115, 1115, 1115, 1115, + 1115, 1115, 1116, 1116, 1116, 1116, 1116, 1116, 1116, 1116, + 1116, 1117, 1117, 1117, 1117, 1117, 1117, 1117, 1117, 1117, + 1118, 1118, 1118, 1118, 1118, 1118, 1118, 1118, 1118, 1119, + 1119, 0, 1119, 1119, 1119, 1119, 1119, 1119, 1120, 1120, + 1120, 1120, 1120, 1120, 1120, 1120, 1120, 1121, 1121, 1121, + 1121, 1121, 1121, 1121, 1121, 1121, 1122, 1122, 1122, 1122, + 1122, 1122, 1122, 1122, 1122, 1123, 1123, 1123, 0, 1123, + 1123, 1123, 1123, 1124, 1124, 1124, 1124, 1124, 1124, 1124, + + 1124, 1124, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, + 1125, 1126, 1126, 1126, 1126, 1126, 1126, 1126, 1126, 1126, + 1127, 1127, 0, 1127, 1127, 1127, 1127, 1127, 1127, 1128, + 1128, 0, 1128, 1128, 1128, 1128, 1128, 1128, 1129, 1129, + 1129, 1129, 1129, 1129, 1129, 1129, 1129, 1130, 1130, 1130, + 1130, 1130, 1130, 1130, 1130, 1130, 1131, 1131, 1131, 1131, + 1131, 1131, 1131, 1131, 1131, 1132, 1132, 1132, 1132, 1132, + 1132, 1132, 1132, 1132, 1133, 1133, 1133, 1133, 1133, 1133, + 1133, 1133, 1133, 1134, 1134, 1134, 1134, 1134, 1134, 1134, + 1134, 1134, 1135, 1135, 1135, 1135, 1135, 1135, 1135, 1135, + + 1135, 1136, 1136, 0, 0, 1136, 1136, 1136, 1136, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, + 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102, 1102 + } ; + +static yy_state_type yy_last_accepting_state; +static char *yy_last_accepting_cpos; + +extern int yy_flex_debug; +int yy_flex_debug = 0; + +/* The intent behind this definition is that it'll catch + * any uses of REJECT which flex missed. + */ +#define REJECT reject_used_but_not_detected +#define yymore() yymore_used_but_not_detected +#define YY_MORE_ADJ 0 +#define YY_RESTORE_YY_MORE_OFFSET +char *yytext; +#line 1 "css.l" + +#line 13 "css.l" +/* Lex source for CSS tokenizing. + Taken from http://www.w3.org/TR/CSS21/grammar.html#q2 + Copyright (C) 2006, 2009-2011, 2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#define YY_NO_INPUT + +#include "css-tokens.h" + +#if defined __clang__ || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5) + #pragma GCC diagnostic ignored "-Wunknown-pragmas" // clang mourns about the next one + #pragma GCC diagnostic ignored "-Wunused-function" + #pragma GCC diagnostic ignored "-Wunused-macros" + #pragma GCC diagnostic ignored "-Wunused-parameter" + #pragma GCC diagnostic ignored "-Wsign-compare" + #pragma GCC diagnostic ignored "-Wswitch-default" + #pragma GCC diagnostic ignored "-Wunreachable-code" // clang + #pragma clang diagnostic ignored "-Wshorten-64-to-32" + #ifndef __clang__ + #pragma GCC diagnostic ignored "-Wsuggest-attribute=pure" + #endif +#endif + +#line 2452 "css.c" +#line 2453 "css.c" + +#define INITIAL 0 + +#ifndef YY_NO_UNISTD_H +/* Special case for "unistd.h", since it is non-ANSI. We include it way + * down here because we want the user's section 1 to have been scanned first. + * The user has a chance to override it with an option. + */ +#include <unistd.h> +#endif + +#ifndef YY_EXTRA_TYPE +#define YY_EXTRA_TYPE void * +#endif + +static int yy_init_globals ( void ); + +/* Accessor methods to globals. + These are made visible to non-reentrant scanners for convenience. */ + +int yylex_destroy ( void ); + +int yyget_debug ( void ); + +void yyset_debug ( int debug_flag ); + +YY_EXTRA_TYPE yyget_extra ( void ); + +void yyset_extra ( YY_EXTRA_TYPE user_defined ); + +FILE *yyget_in ( void ); + +void yyset_in ( FILE * _in_str ); + +FILE *yyget_out ( void ); + +void yyset_out ( FILE * _out_str ); + + int yyget_leng ( void ); + +char *yyget_text ( void ); + +int yyget_lineno ( void ); + +void yyset_lineno ( int _line_number ); + +/* Macros after this point can all be overridden by user definitions in + * section 1. + */ + +#ifndef YY_SKIP_YYWRAP +#ifdef __cplusplus +extern "C" int yywrap ( void ); +#else +extern int yywrap ( void ); +#endif +#endif + +#ifndef YY_NO_UNPUT + +#endif + +#ifndef yytext_ptr +static void yy_flex_strncpy ( char *, const char *, int ); +#endif + +#ifdef YY_NEED_STRLEN +static int yy_flex_strlen ( const char * ); +#endif + +#ifndef YY_NO_INPUT +#ifdef __cplusplus +static int yyinput ( void ); +#else +static int input ( void ); +#endif + +#endif + +/* Amount of stuff to slurp up with each read. */ +#ifndef YY_READ_BUF_SIZE +#ifdef __ia64__ +/* On IA-64, the buffer size is 16k, not 8k */ +#define YY_READ_BUF_SIZE 16384 +#else +#define YY_READ_BUF_SIZE 8192 +#endif /* __ia64__ */ +#endif + +/* Copy whatever the last rule matched to the standard output. */ +#ifndef ECHO +/* This used to be an fputs(), but since the string might contain NUL's, + * we now use fwrite(). + */ +#define ECHO do { if (fwrite( yytext, (size_t) yyleng, 1, yyout )) {} } while (0) +#endif + +/* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, + * is returned in "result". + */ +#ifndef YY_INPUT +#define YY_INPUT(buf,result,max_size) \ + if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \ + { \ + int c = '*'; \ + int n; \ + for ( n = 0; n < max_size && \ + (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ + buf[n] = (char) c; \ + if ( c == '\n' ) \ + buf[n++] = (char) c; \ + if ( c == EOF && ferror( yyin ) ) \ + YY_FATAL_ERROR( "input in flex scanner failed" ); \ + result = n; \ + } \ + else \ + { \ + errno=0; \ + while ( (result = (int) fread(buf, 1, (yy_size_t) max_size, yyin)) == 0 && ferror(yyin)) \ + { \ + if( errno != EINTR) \ + { \ + YY_FATAL_ERROR( "input in flex scanner failed" ); \ + break; \ + } \ + errno=0; \ + clearerr(yyin); \ + } \ + }\ +\ + +#endif + +/* No semi-colon after return; correct usage is to write "yyterminate();" - + * we don't want an extra ';' after the "return" because that will cause + * some compilers to complain about unreachable statements. + */ +#ifndef yyterminate +#define yyterminate() return YY_NULL +#endif + +/* Number of entries by which start-condition stack grows. */ +#ifndef YY_START_STACK_INCR +#define YY_START_STACK_INCR 25 +#endif + +/* Report a fatal error. */ +#ifndef YY_FATAL_ERROR +#define YY_FATAL_ERROR(msg) yy_fatal_error( msg ) +#endif + +/* end tables serialization structures and prototypes */ + +/* Default declaration of generated scanner - a define so the user can + * easily add parameters. + */ +#ifndef YY_DECL +#define YY_DECL_IS_OURS 1 + +extern int yylex (void); + +#define YY_DECL int yylex (void) +#endif /* !YY_DECL */ + +/* Code executed at the beginning of each rule, after yytext and yyleng + * have been set up. + */ +#ifndef YY_USER_ACTION +#define YY_USER_ACTION +#endif + +/* Code executed at the end of each rule. */ +#ifndef YY_BREAK +#define YY_BREAK /*LINTED*/break; +#endif + +#define YY_RULE_SETUP \ + YY_USER_ACTION + +/** The main scanner function which does all the work. + */ +YY_DECL +{ + yy_state_type yy_current_state; + char *yy_cp, *yy_bp; + int yy_act; + + if ( !(yy_init) ) + { + (yy_init) = 1; + +#ifdef YY_USER_INIT + YY_USER_INIT; +#endif + + if ( ! (yy_start) ) + (yy_start) = 1; /* first start state */ + + if ( ! yyin ) + yyin = stdin; + + if ( ! yyout ) + yyout = stdout; + + if ( ! YY_CURRENT_BUFFER ) { + yyensure_buffer_stack (); + YY_CURRENT_BUFFER_LVALUE = + yy_create_buffer( yyin, YY_BUF_SIZE ); + } + + yy_load_buffer_state( ); + } + + { +#line 112 "css.l" + + +#line 2671 "css.c" + + while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ + { + yy_cp = (yy_c_buf_p); + + /* Support of yytext. */ + *yy_cp = (yy_hold_char); + + /* yy_bp points to the position in yy_ch_buf of the start of + * the current run. + */ + yy_bp = yy_cp; + + yy_current_state = (yy_start); +yy_match: + do + { + YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)] ; + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 1103 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + ++yy_cp; + } + while ( yy_current_state != 1102 ); + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + +yy_find_action: + yy_act = yy_accept[yy_current_state]; + + YY_DO_BEFORE_ACTION; + +do_action: /* This label is used only to access EOF actions. */ + + switch ( yy_act ) + { /* beginning of action switch */ + case 0: /* must back up */ + /* undo the effects of YY_DO_BEFORE_ACTION */ + *yy_cp = (yy_hold_char); + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + goto yy_find_action; + +case 1: +/* rule 1 can match eol */ +YY_RULE_SETUP +#line 114 "css.l" +{return S;} + YY_BREAK +case 2: +/* rule 2 can match eol */ +YY_RULE_SETUP +#line 116 "css.l" +{return COMMENT;} + YY_BREAK +case 3: +/* rule 3 can match eol */ +YY_RULE_SETUP +#line 117 "css.l" +/* ignore comments */ + YY_BREAK +case 4: +/* rule 4 can match eol */ +YY_RULE_SETUP +#line 118 "css.l" +/* unclosed comment at EOF */ + YY_BREAK +case 5: +YY_RULE_SETUP +#line 120 "css.l" +{return CDO;} + YY_BREAK +case 6: +YY_RULE_SETUP +#line 121 "css.l" +{return CDC;} + YY_BREAK +case 7: +YY_RULE_SETUP +#line 122 "css.l" +{return INCLUDES;} + YY_BREAK +case 8: +YY_RULE_SETUP +#line 123 "css.l" +{return DASHMATCH;} + YY_BREAK +case 9: +/* rule 9 can match eol */ +YY_RULE_SETUP +#line 125 "css.l" +{return STRING;} + YY_BREAK +case 10: +/* rule 10 can match eol */ +YY_RULE_SETUP +#line 126 "css.l" +{return BAD_STRING;} + YY_BREAK +case 11: +/* rule 11 can match eol */ +YY_RULE_SETUP +#line 128 "css.l" +{return IDENT;} + YY_BREAK +case 12: +/* rule 12 can match eol */ +YY_RULE_SETUP +#line 130 "css.l" +{return HASH;} + YY_BREAK +case 13: +/* rule 13 can match eol */ +YY_RULE_SETUP +#line 132 "css.l" +{return IMPORT_SYM;} + YY_BREAK +case 14: +/* rule 14 can match eol */ +YY_RULE_SETUP +#line 133 "css.l" +{return PAGE_SYM;} + YY_BREAK +case 15: +/* rule 15 can match eol */ +YY_RULE_SETUP +#line 134 "css.l" +{return MEDIA_SYM;} + YY_BREAK +case 16: +YY_RULE_SETUP +#line 135 "css.l" +{return CHARSET_SYM;} + YY_BREAK +case 17: +/* rule 17 can match eol */ +YY_RULE_SETUP +#line 137 "css.l" +{return IMPORTANT_SYM;} + YY_BREAK +case 18: +/* rule 18 can match eol */ +YY_RULE_SETUP +#line 139 "css.l" +{return EMS;} + YY_BREAK +case 19: +/* rule 19 can match eol */ +YY_RULE_SETUP +#line 140 "css.l" +{return EXS;} + YY_BREAK +case 20: +/* rule 20 can match eol */ +YY_RULE_SETUP +#line 141 "css.l" +{return LENGTH;} + YY_BREAK +case 21: +/* rule 21 can match eol */ +YY_RULE_SETUP +#line 142 "css.l" +{return LENGTH;} + YY_BREAK +case 22: +/* rule 22 can match eol */ +YY_RULE_SETUP +#line 143 "css.l" +{return LENGTH;} + YY_BREAK +case 23: +/* rule 23 can match eol */ +YY_RULE_SETUP +#line 144 "css.l" +{return LENGTH;} + YY_BREAK +case 24: +/* rule 24 can match eol */ +YY_RULE_SETUP +#line 145 "css.l" +{return LENGTH;} + YY_BREAK +case 25: +/* rule 25 can match eol */ +YY_RULE_SETUP +#line 146 "css.l" +{return LENGTH;} + YY_BREAK +case 26: +/* rule 26 can match eol */ +YY_RULE_SETUP +#line 147 "css.l" +{return ANGLE;} + YY_BREAK +case 27: +/* rule 27 can match eol */ +YY_RULE_SETUP +#line 148 "css.l" +{return ANGLE;} + YY_BREAK +case 28: +/* rule 28 can match eol */ +YY_RULE_SETUP +#line 149 "css.l" +{return ANGLE;} + YY_BREAK +case 29: +/* rule 29 can match eol */ +YY_RULE_SETUP +#line 150 "css.l" +{return TIME;} + YY_BREAK +case 30: +/* rule 30 can match eol */ +YY_RULE_SETUP +#line 151 "css.l" +{return TIME;} + YY_BREAK +case 31: +/* rule 31 can match eol */ +YY_RULE_SETUP +#line 152 "css.l" +{return FREQ;} + YY_BREAK +case 32: +/* rule 32 can match eol */ +YY_RULE_SETUP +#line 153 "css.l" +{return FREQ;} + YY_BREAK +case 33: +/* rule 33 can match eol */ +YY_RULE_SETUP +#line 154 "css.l" +{return DIMENSION;} + YY_BREAK +case 34: +YY_RULE_SETUP +#line 156 "css.l" +{return PERCENTAGE;} + YY_BREAK +case 35: +YY_RULE_SETUP +#line 157 "css.l" +{return NUMBER;} + YY_BREAK +case 36: +/* rule 36 can match eol */ +YY_RULE_SETUP +#line 159 "css.l" +{return URI;} + YY_BREAK +case 37: +/* rule 37 can match eol */ +YY_RULE_SETUP +#line 160 "css.l" +{return URI;} + YY_BREAK +case 38: +/* rule 38 can match eol */ +YY_RULE_SETUP +#line 161 "css.l" +{return BAD_URI;} + YY_BREAK +case 39: +/* rule 39 can match eol */ +YY_RULE_SETUP +#line 163 "css.l" +{return FUNCTION;} + YY_BREAK +case 40: +YY_RULE_SETUP +#line 165 "css.l" +{return *yytext;} + YY_BREAK +case 41: +YY_RULE_SETUP +#line 167 "css.l" +ECHO; + YY_BREAK +#line 2961 "css.c" +case YY_STATE_EOF(INITIAL): + yyterminate(); + + case YY_END_OF_BUFFER: + { + /* Amount of text matched not including the EOB char. */ + int yy_amount_of_matched_text = (int) (yy_cp - (yytext_ptr)) - 1; + + /* Undo the effects of YY_DO_BEFORE_ACTION. */ + *yy_cp = (yy_hold_char); + YY_RESTORE_YY_MORE_OFFSET + + if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_NEW ) + { + /* We're scanning a new file or input source. It's + * possible that this happened because the user + * just pointed yyin at a new source and called + * yylex(). If so, then we have to assure + * consistency between YY_CURRENT_BUFFER and our + * globals. Here is the right place to do so, because + * this is the first action (other than possibly a + * back-up) that will match for the new input source. + */ + (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; + YY_CURRENT_BUFFER_LVALUE->yy_input_file = yyin; + YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = YY_BUFFER_NORMAL; + } + + /* Note that here we test for yy_c_buf_p "<=" to the position + * of the first EOB in the buffer, since yy_c_buf_p will + * already have been incremented past the NUL character + * (since all states make transitions on EOB to the + * end-of-buffer state). Contrast this with the test + * in input(). + */ + if ( (yy_c_buf_p) <= &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] ) + { /* This was really a NUL. */ + yy_state_type yy_next_state; + + (yy_c_buf_p) = (yytext_ptr) + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state( ); + + /* Okay, we're now positioned to make the NUL + * transition. We couldn't have + * yy_get_previous_state() go ahead and do it + * for us because it doesn't know how to deal + * with the possibility of jamming (and we don't + * want to build jamming into it because then it + * will run more slowly). + */ + + yy_next_state = yy_try_NUL_trans( yy_current_state ); + + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + + if ( yy_next_state ) + { + /* Consume the NUL. */ + yy_cp = ++(yy_c_buf_p); + yy_current_state = yy_next_state; + goto yy_match; + } + + else + { + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + goto yy_find_action; + } + } + + else switch ( yy_get_next_buffer( ) ) + { + case EOB_ACT_END_OF_FILE: + { + (yy_did_buffer_switch_on_eof) = 0; + + if ( yywrap( ) ) + { + /* Note: because we've taken care in + * yy_get_next_buffer() to have set up + * yytext, we can now set up + * yy_c_buf_p so that if some total + * hoser (like flex itself) wants to + * call the scanner after we return the + * YY_NULL, it'll still work - another + * YY_NULL will get returned. + */ + (yy_c_buf_p) = (yytext_ptr) + YY_MORE_ADJ; + + yy_act = YY_STATE_EOF(YY_START); + goto do_action; + } + + else + { + if ( ! (yy_did_buffer_switch_on_eof) ) + YY_NEW_FILE; + } + break; + } + + case EOB_ACT_CONTINUE_SCAN: + (yy_c_buf_p) = + (yytext_ptr) + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state( ); + + yy_cp = (yy_c_buf_p); + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + goto yy_match; + + case EOB_ACT_LAST_MATCH: + (yy_c_buf_p) = + &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)]; + + yy_current_state = yy_get_previous_state( ); + + yy_cp = (yy_c_buf_p); + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + goto yy_find_action; + } + break; + } + + default: + YY_FATAL_ERROR( + "fatal flex scanner internal error--no action found" ); + } /* end of action switch */ + } /* end of scanning one token */ + } /* end of user's declarations */ +} /* end of yylex */ + +/* yy_get_next_buffer - try to read in a new buffer + * + * Returns a code representing an action: + * EOB_ACT_LAST_MATCH - + * EOB_ACT_CONTINUE_SCAN - continue scanning from current position + * EOB_ACT_END_OF_FILE - end of file + */ +static int yy_get_next_buffer (void) +{ + char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; + char *source = (yytext_ptr); + int number_to_move, i; + int ret_val; + + if ( (yy_c_buf_p) > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] ) + YY_FATAL_ERROR( + "fatal flex scanner internal error--end of buffer missed" ); + + if ( YY_CURRENT_BUFFER_LVALUE->yy_fill_buffer == 0 ) + { /* Don't try to fill the buffer, so this is an EOF. */ + if ( (yy_c_buf_p) - (yytext_ptr) - YY_MORE_ADJ == 1 ) + { + /* We matched a single character, the EOB, so + * treat this as a final EOF. + */ + return EOB_ACT_END_OF_FILE; + } + + else + { + /* We matched some text prior to the EOB, first + * process it. + */ + return EOB_ACT_LAST_MATCH; + } + } + + /* Try to read more data. */ + + /* First move last chars to start of buffer. */ + number_to_move = (int) ((yy_c_buf_p) - (yytext_ptr) - 1); + + for ( i = 0; i < number_to_move; ++i ) + *(dest++) = *(source++); + + if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_EOF_PENDING ) + /* don't do the read, it's not guaranteed to return an EOF, + * just force an EOF + */ + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars) = 0; + + else + { + int num_to_read = + YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1; + + while ( num_to_read <= 0 ) + { /* Not enough room in the buffer - grow it. */ + + /* just a shorter name for the current buffer */ + YY_BUFFER_STATE b = YY_CURRENT_BUFFER_LVALUE; + + int yy_c_buf_p_offset = + (int) ((yy_c_buf_p) - b->yy_ch_buf); + + if ( b->yy_is_our_buffer ) + { + int new_size = b->yy_buf_size * 2; + + if ( new_size <= 0 ) + b->yy_buf_size += b->yy_buf_size / 8; + else + b->yy_buf_size *= 2; + + b->yy_ch_buf = (char *) + /* Include room in for 2 EOB chars. */ + yyrealloc( (void *) b->yy_ch_buf, + (yy_size_t) (b->yy_buf_size + 2) ); + } + else + /* Can't grow it, we don't own it. */ + b->yy_ch_buf = NULL; + + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( + "fatal error - scanner input buffer overflow" ); + + (yy_c_buf_p) = &b->yy_ch_buf[yy_c_buf_p_offset]; + + num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size - + number_to_move - 1; + + } + + if ( num_to_read > YY_READ_BUF_SIZE ) + num_to_read = YY_READ_BUF_SIZE; + + /* Read in more data. */ + YY_INPUT( (&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]), + (yy_n_chars), num_to_read ); + + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + if ( (yy_n_chars) == 0 ) + { + if ( number_to_move == YY_MORE_ADJ ) + { + ret_val = EOB_ACT_END_OF_FILE; + yyrestart( yyin ); + } + + else + { + ret_val = EOB_ACT_LAST_MATCH; + YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = + YY_BUFFER_EOF_PENDING; + } + } + + else + ret_val = EOB_ACT_CONTINUE_SCAN; + + if (((yy_n_chars) + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { + /* Extend the array by 50%, plus the number we really need. */ + int new_size = (yy_n_chars) + number_to_move + ((yy_n_chars) >> 1); + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc( + (void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, (yy_size_t) new_size ); + if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_get_next_buffer()" ); + /* "- 2" to take care of EOB's */ + YY_CURRENT_BUFFER_LVALUE->yy_buf_size = (int) (new_size - 2); + } + + (yy_n_chars) += number_to_move; + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] = YY_END_OF_BUFFER_CHAR; + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] = YY_END_OF_BUFFER_CHAR; + + (yytext_ptr) = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[0]; + + return ret_val; +} + +/* yy_get_previous_state - get the state just before the EOB char was reached */ + + static yy_state_type yy_get_previous_state (void) +{ + yy_state_type yy_current_state; + char *yy_cp; + + yy_current_state = (yy_start); + + for ( yy_cp = (yytext_ptr) + YY_MORE_ADJ; yy_cp < (yy_c_buf_p); ++yy_cp ) + { + YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 1103 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + } + + return yy_current_state; +} + +/* yy_try_NUL_trans - try to make a transition on the NUL character + * + * synopsis + * next_state = yy_try_NUL_trans( current_state ); + */ + static yy_state_type yy_try_NUL_trans (yy_state_type yy_current_state ) +{ + int yy_is_jam; + char *yy_cp = (yy_c_buf_p); + + YY_CHAR yy_c = 1; + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 1103 ) + yy_c = yy_meta[yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_is_jam = (yy_current_state == 1102); + + return yy_is_jam ? 0 : yy_current_state; +} + +#ifndef YY_NO_UNPUT + +#endif + +#ifndef YY_NO_INPUT +#ifdef __cplusplus + static int yyinput (void) +#else + static int input (void) +#endif + +{ + int c; + + *(yy_c_buf_p) = (yy_hold_char); + + if ( *(yy_c_buf_p) == YY_END_OF_BUFFER_CHAR ) + { + /* yy_c_buf_p now points to the character we want to return. + * If this occurs *before* the EOB characters, then it's a + * valid NUL; if not, then we've hit the end of the buffer. + */ + if ( (yy_c_buf_p) < &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] ) + /* This was really a NUL. */ + *(yy_c_buf_p) = '\0'; + + else + { /* need more input */ + int offset = (int) ((yy_c_buf_p) - (yytext_ptr)); + ++(yy_c_buf_p); + + switch ( yy_get_next_buffer( ) ) + { + case EOB_ACT_LAST_MATCH: + /* This happens because yy_g_n_b() + * sees that we've accumulated a + * token and flags that we need to + * try matching the token before + * proceeding. But for input(), + * there's no matching to consider. + * So convert the EOB_ACT_LAST_MATCH + * to EOB_ACT_END_OF_FILE. + */ + + /* Reset buffer status. */ + yyrestart( yyin ); + + /*FALLTHROUGH*/ + + case EOB_ACT_END_OF_FILE: + { + if ( yywrap( ) ) + return 0; + + if ( ! (yy_did_buffer_switch_on_eof) ) + YY_NEW_FILE; +#ifdef __cplusplus + return yyinput(); +#else + return input(); +#endif + } + + case EOB_ACT_CONTINUE_SCAN: + (yy_c_buf_p) = (yytext_ptr) + offset; + break; + } + } + } + + c = *(unsigned char *) (yy_c_buf_p); /* cast for 8-bit char's */ + *(yy_c_buf_p) = '\0'; /* preserve yytext */ + (yy_hold_char) = *++(yy_c_buf_p); + + return c; +} +#endif /* ifndef YY_NO_INPUT */ + +/** Immediately switch to a different input stream. + * @param input_file A readable stream. + * + * @note This function does not reset the start condition to @c INITIAL . + */ + void yyrestart (FILE * input_file ) +{ + + if ( ! YY_CURRENT_BUFFER ){ + yyensure_buffer_stack (); + YY_CURRENT_BUFFER_LVALUE = + yy_create_buffer( yyin, YY_BUF_SIZE ); + } + + yy_init_buffer( YY_CURRENT_BUFFER, input_file ); + yy_load_buffer_state( ); +} + +/** Switch to a different input buffer. + * @param new_buffer The new input buffer. + * + */ + void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ) +{ + + /* TODO. We should be able to replace this entire function body + * with + * yypop_buffer_state(); + * yypush_buffer_state(new_buffer); + */ + yyensure_buffer_stack (); + if ( YY_CURRENT_BUFFER == new_buffer ) + return; + + if ( YY_CURRENT_BUFFER ) + { + /* Flush out information for old buffer. */ + *(yy_c_buf_p) = (yy_hold_char); + YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p); + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + YY_CURRENT_BUFFER_LVALUE = new_buffer; + yy_load_buffer_state( ); + + /* We don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + (yy_did_buffer_switch_on_eof) = 1; +} + +static void yy_load_buffer_state (void) +{ + (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; + (yytext_ptr) = (yy_c_buf_p) = YY_CURRENT_BUFFER_LVALUE->yy_buf_pos; + yyin = YY_CURRENT_BUFFER_LVALUE->yy_input_file; + (yy_hold_char) = *(yy_c_buf_p); +} + +/** Allocate and initialize an input buffer state. + * @param file A readable stream. + * @param size The character buffer size in bytes. When in doubt, use @c YY_BUF_SIZE. + * + * @return the allocated buffer state. + */ + YY_BUFFER_STATE yy_create_buffer (FILE * file, int size ) +{ + YY_BUFFER_STATE b; + + b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_buf_size = size; + + /* yy_ch_buf has to be 2 characters longer than the size given because + * we need to put in 2 end-of-buffer characters. + */ + b->yy_ch_buf = (char *) yyalloc( (yy_size_t) (b->yy_buf_size + 2) ); + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_is_our_buffer = 1; + + yy_init_buffer( b, file ); + + return b; +} + +/** Destroy the buffer. + * @param b a buffer created with yy_create_buffer() + * + */ + void yy_delete_buffer (YY_BUFFER_STATE b ) +{ + + if ( ! b ) + return; + + if ( b == YY_CURRENT_BUFFER ) /* Not sure if we should pop here. */ + YY_CURRENT_BUFFER_LVALUE = (YY_BUFFER_STATE) 0; + + if ( b->yy_is_our_buffer ) + yyfree( (void *) b->yy_ch_buf ); + + yyfree( (void *) b ); +} + +/* Initializes or reinitializes a buffer. + * This function is sometimes called more than once on the same buffer, + * such as during a yyrestart() or at EOF. + */ + static void yy_init_buffer (YY_BUFFER_STATE b, FILE * file ) + +{ + int oerrno = errno; + + yy_flush_buffer( b ); + + b->yy_input_file = file; + b->yy_fill_buffer = 1; + + /* If b is the current buffer, then yy_init_buffer was _probably_ + * called from yyrestart() or through yy_get_next_buffer. + * In that case, we don't want to reset the lineno or column. + */ + if (b != YY_CURRENT_BUFFER){ + b->yy_bs_lineno = 1; + b->yy_bs_column = 0; + } + + b->yy_is_interactive = 0; + + errno = oerrno; +} + +/** Discard all buffered characters. On the next scan, YY_INPUT will be called. + * @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER. + * + */ + void yy_flush_buffer (YY_BUFFER_STATE b ) +{ + if ( ! b ) + return; + + b->yy_n_chars = 0; + + /* We always need two end-of-buffer characters. The first causes + * a transition to the end-of-buffer state. The second causes + * a jam in that state. + */ + b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; + b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; + + b->yy_buf_pos = &b->yy_ch_buf[0]; + + b->yy_at_bol = 1; + b->yy_buffer_status = YY_BUFFER_NEW; + + if ( b == YY_CURRENT_BUFFER ) + yy_load_buffer_state( ); +} + +/** Pushes the new state onto the stack. The new state becomes + * the current state. This function will allocate the stack + * if necessary. + * @param new_buffer The new state. + * + */ +void yypush_buffer_state (YY_BUFFER_STATE new_buffer ) +{ + if (new_buffer == NULL) + return; + + yyensure_buffer_stack(); + + /* This block is copied from yy_switch_to_buffer. */ + if ( YY_CURRENT_BUFFER ) + { + /* Flush out information for old buffer. */ + *(yy_c_buf_p) = (yy_hold_char); + YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p); + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + /* Only push if top exists. Otherwise, replace top. */ + if (YY_CURRENT_BUFFER) + (yy_buffer_stack_top)++; + YY_CURRENT_BUFFER_LVALUE = new_buffer; + + /* copied from yy_switch_to_buffer. */ + yy_load_buffer_state( ); + (yy_did_buffer_switch_on_eof) = 1; +} + +/** Removes and deletes the top of the stack, if present. + * The next element becomes the new top. + * + */ +void yypop_buffer_state (void) +{ + if (!YY_CURRENT_BUFFER) + return; + + yy_delete_buffer(YY_CURRENT_BUFFER ); + YY_CURRENT_BUFFER_LVALUE = NULL; + if ((yy_buffer_stack_top) > 0) + --(yy_buffer_stack_top); + + if (YY_CURRENT_BUFFER) { + yy_load_buffer_state( ); + (yy_did_buffer_switch_on_eof) = 1; + } +} + +/* Allocates the stack if it does not exist. + * Guarantees space for at least one push. + */ +static void yyensure_buffer_stack (void) +{ + yy_size_t num_to_alloc; + + if (!(yy_buffer_stack)) { + + /* First allocation is just for 2 elements, since we don't know if this + * scanner will even need a stack. We use 2 instead of 1 to avoid an + * immediate realloc on the next call. + */ + num_to_alloc = 1; /* After all that talk, this was set to 1 anyways... */ + (yy_buffer_stack) = (struct yy_buffer_state**)yyalloc + (num_to_alloc * sizeof(struct yy_buffer_state*) + ); + if ( ! (yy_buffer_stack) ) + YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + + memset((yy_buffer_stack), 0, num_to_alloc * sizeof(struct yy_buffer_state*)); + + (yy_buffer_stack_max) = num_to_alloc; + (yy_buffer_stack_top) = 0; + return; + } + + if ((yy_buffer_stack_top) >= ((yy_buffer_stack_max)) - 1){ + + /* Increase the buffer to prepare for a possible push. */ + yy_size_t grow_size = 8 /* arbitrary grow size */; + + num_to_alloc = (yy_buffer_stack_max) + grow_size; + (yy_buffer_stack) = (struct yy_buffer_state**)yyrealloc + ((yy_buffer_stack), + num_to_alloc * sizeof(struct yy_buffer_state*) + ); + if ( ! (yy_buffer_stack) ) + YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + + /* zero only the new slots.*/ + memset((yy_buffer_stack) + (yy_buffer_stack_max), 0, grow_size * sizeof(struct yy_buffer_state*)); + (yy_buffer_stack_max) = num_to_alloc; + } +} + +/** Setup the input buffer state to scan directly from a user-specified character buffer. + * @param base the character buffer + * @param size the size in bytes of the character buffer + * + * @return the newly allocated buffer state object. + */ +YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size ) +{ + YY_BUFFER_STATE b; + + if ( size < 2 || + base[size-2] != YY_END_OF_BUFFER_CHAR || + base[size-1] != YY_END_OF_BUFFER_CHAR ) + /* They forgot to leave room for the EOB's. */ + return NULL; + + b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); + + b->yy_buf_size = (int) (size - 2); /* "- 2" to take care of EOB's */ + b->yy_buf_pos = b->yy_ch_buf = base; + b->yy_is_our_buffer = 0; + b->yy_input_file = NULL; + b->yy_n_chars = b->yy_buf_size; + b->yy_is_interactive = 0; + b->yy_at_bol = 1; + b->yy_fill_buffer = 0; + b->yy_buffer_status = YY_BUFFER_NEW; + + yy_switch_to_buffer( b ); + + return b; +} + +/** Setup the input buffer state to scan a string. The next call to yylex() will + * scan from a @e copy of @a str. + * @param yystr a NUL-terminated string to scan + * + * @return the newly allocated buffer state object. + * @note If you want to scan bytes that may contain NUL values, then use + * yy_scan_bytes() instead. + */ +YY_BUFFER_STATE yy_scan_string (const char * yystr ) +{ + + return yy_scan_bytes( yystr, (int) strlen(yystr) ); +} + +/** Setup the input buffer state to scan the given bytes. The next call to yylex() will + * scan from a @e copy of @a bytes. + * @param yybytes the byte buffer to scan + * @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes. + * + * @return the newly allocated buffer state object. + */ +YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len ) +{ + YY_BUFFER_STATE b; + char *buf; + yy_size_t n; + int i; + + /* Get memory for full buffer, including space for trailing EOB's. */ + n = (yy_size_t) (_yybytes_len + 2); + buf = (char *) yyalloc( n ); + if ( ! buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); + + for ( i = 0; i < _yybytes_len; ++i ) + buf[i] = yybytes[i]; + + buf[_yybytes_len] = buf[_yybytes_len+1] = YY_END_OF_BUFFER_CHAR; + + b = yy_scan_buffer( buf, n ); + if ( ! b ) + YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); + + /* It's okay to grow etc. this buffer, and we should throw it + * away when we're done. + */ + b->yy_is_our_buffer = 1; + + return b; +} + +#ifndef YY_EXIT_FAILURE +#define YY_EXIT_FAILURE 2 +#endif + +static void yynoreturn yy_fatal_error (const char* msg ) +{ + fprintf( stderr, "%s\n", msg ); + exit( YY_EXIT_FAILURE ); +} + +/* Redefine yyless() so it works in section 3 code. */ + +#undef yyless +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + int yyless_macro_arg = (n); \ + YY_LESS_LINENO(yyless_macro_arg);\ + yytext[yyleng] = (yy_hold_char); \ + (yy_c_buf_p) = yytext + yyless_macro_arg; \ + (yy_hold_char) = *(yy_c_buf_p); \ + *(yy_c_buf_p) = '\0'; \ + yyleng = yyless_macro_arg; \ + } \ + while ( 0 ) + +/* Accessor methods (get/set functions) to struct members. */ + +/** Get the current line number. + * + */ +int yyget_lineno (void) +{ + + return yylineno; +} + +/** Get the input stream. + * + */ +FILE *yyget_in (void) +{ + return yyin; +} + +/** Get the output stream. + * + */ +FILE *yyget_out (void) +{ + return yyout; +} + +/** Get the length of the current token. + * + */ +int yyget_leng (void) +{ + return yyleng; +} + +/** Get the current token. + * + */ + +char *yyget_text (void) +{ + return yytext; +} + +/** Set the current line number. + * @param _line_number line number + * + */ +void yyset_lineno (int _line_number ) +{ + + yylineno = _line_number; +} + +/** Set the input stream. This does not discard the current + * input buffer. + * @param _in_str A readable stream. + * + * @see yy_switch_to_buffer + */ +void yyset_in (FILE * _in_str ) +{ + yyin = _in_str ; +} + +void yyset_out (FILE * _out_str ) +{ + yyout = _out_str ; +} + +int yyget_debug (void) +{ + return yy_flex_debug; +} + +void yyset_debug (int _bdebug ) +{ + yy_flex_debug = _bdebug ; +} + +static int yy_init_globals (void) +{ + /* Initialization is the same as for the non-reentrant scanner. + * This function is called from yylex_destroy(), so don't allocate here. + */ + + (yy_buffer_stack) = NULL; + (yy_buffer_stack_top) = 0; + (yy_buffer_stack_max) = 0; + (yy_c_buf_p) = NULL; + (yy_init) = 0; + (yy_start) = 0; + +/* Defined in main.c */ +#ifdef YY_STDINIT + yyin = stdin; + yyout = stdout; +#else + yyin = NULL; + yyout = NULL; +#endif + + /* For future reference: Set errno on error, since we are called by + * yylex_init() + */ + return 0; +} + +/* yylex_destroy is for both reentrant and non-reentrant scanners. */ +int yylex_destroy (void) +{ + + /* Pop the buffer stack, destroying each element. */ + while(YY_CURRENT_BUFFER){ + yy_delete_buffer( YY_CURRENT_BUFFER ); + YY_CURRENT_BUFFER_LVALUE = NULL; + yypop_buffer_state(); + } + + /* Destroy the stack itself. */ + yyfree((yy_buffer_stack) ); + (yy_buffer_stack) = NULL; + + /* Reset the globals. This is important in a non-reentrant scanner so the next time + * yylex() is called, initialization will occur. */ + yy_init_globals( ); + + return 0; +} + +/* + * Internal utility routines. + */ + +#ifndef yytext_ptr +static void yy_flex_strncpy (char* s1, const char * s2, int n ) +{ + + int i; + for ( i = 0; i < n; ++i ) + s1[i] = s2[i]; +} +#endif + +#ifdef YY_NEED_STRLEN +static int yy_flex_strlen (const char * s ) +{ + int n; + for ( n = 0; s[n]; ++n ) + ; + + return n; +} +#endif + +void *yyalloc (yy_size_t size ) +{ + return malloc(size); +} + +void *yyrealloc (void * ptr, yy_size_t size ) +{ + + /* The cast to (char *) in the following accommodates both + * implementations that use char* generic pointers, and those + * that use void* generic pointers. It works with the latter + * because both ANSI C and C++ allow castless assignment from + * any pointer type to void*, and deal with argument conversions + * as though doing an assignment. + */ + return realloc(ptr, size); +} + +void yyfree (void * ptr ) +{ + free( (char *) ptr ); /* see yyrealloc() for (char *) cast */ +} + +#define YYTABLES_NAME "yytables" + +#line 167 "css.l" + + diff --git a/src/exits.c b/src/exits.c new file mode 100644 index 0000000..56a7eb7 --- /dev/null +++ b/src/exits.c @@ -0,0 +1,93 @@ +/* Exit status handling. + Copyright (C) 2009-2012, 2015, 2018-2020 Free Software Foundation, + Inc. + + This file is part of GNU Wget. + + GNU Wget is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + GNU Wget is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Wget. If not, see <http://www.gnu.org/licenses/>. +*/ + +#include "wget.h" +#include "exits.h" + +static int final_exit_status = WGET_EXIT_SUCCESS; + +/* XXX: I don't like that newly-added uerr_t codes will doubtless fall + through the craccks, or the fact that we seem to have way more + codes than we know what to do with. Need to go through and sort + through the truly essential codes, and merge the rest with + those. Quite a few are never even used! + + Quite a few of the codes below would have no business being + returned to retrieve_url's caller, but since it's very difficult to + determine which do and which don't, I grab virtually all of them to + be safe. */ +static int +get_status_for_err (uerr_t err) +{ + switch (err) + { + case RETROK: + return WGET_EXIT_SUCCESS; + case FOPENERR: case FOPEN_EXCL_ERR: case FWRITEERR: case WRITEFAILED: + case UNLINKERR: case CLOSEFAILED: case FILEBADFILE: + return WGET_EXIT_IO_FAIL; + case NOCONERROR: case HOSTERR: case CONSOCKERR: case CONERROR: + case CONSSLERR: case CONIMPOSSIBLE: case FTPRERR: case FTPINVPASV: + case READERR: case TRYLIMEXC: + return WGET_EXIT_NETWORK_FAIL; + case VERIFCERTERR: + return WGET_EXIT_SSL_AUTH_FAIL; + case FTPLOGINC: case FTPLOGREFUSED: case AUTHFAILED: + return WGET_EXIT_SERVER_AUTH_FAIL; + case HEOF: case HERR: case ATTRMISSING: + return WGET_EXIT_PROTOCOL_ERROR; + case WRONGCODE: case FTPPORTERR: case FTPSYSERR: + case FTPNSFOD: case FTPUNKNOWNTYPE: case FTPSRVERR: + case FTPRETRINT: case FTPRESTFAIL: case FTPNOPASV: + case CONTNOTSUPPORTED: case RANGEERR: case RETRBADPATTERN: + case PROXERR: case GATEWAYTIMEOUT: + return WGET_EXIT_SERVER_ERROR; + case URLERROR: case QUOTEXC: case SSLINITFAILED: case UNKNOWNATTR: + default: + return WGET_EXIT_UNKNOWN; + } +} + +/* inform_exit_status + * + * Ensure that Wget's exit status will reflect the problem indicated + * by ERR, unless the exit status has already been set to reflect a more + * important problem. */ +void +inform_exit_status (uerr_t err) +{ + int new_status = get_status_for_err (err); + + if (new_status != WGET_EXIT_SUCCESS + && (final_exit_status == WGET_EXIT_SUCCESS + || new_status < final_exit_status)) + { + final_exit_status = new_status; + } +} + +int +get_exit_status (void) +{ + return + (final_exit_status == WGET_EXIT_UNKNOWN) + ? 1 + : final_exit_status; +} diff --git a/src/exits.h b/src/exits.h new file mode 100644 index 0000000..169ade1 --- /dev/null +++ b/src/exits.h @@ -0,0 +1,48 @@ +/* Exit status related declarations. + Copyright (C) 2009-2012, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. */ + +#ifndef WGET_EXITS_H +#define WGET_EXITS_H + +#include "wget.h" + +/* Final exit code possibilities. Exit codes 1 and 2 are reserved + * for situations that lead to direct exits from Wget, not using the + * value of final_exit_status. */ +enum + { + WGET_EXIT_SUCCESS = 0, + WGET_EXIT_GENERIC_ERROR = 1, + WGET_EXIT_PARSE_ERROR = 2, + WGET_EXIT_IO_FAIL = 3, + WGET_EXIT_NETWORK_FAIL = 4, + WGET_EXIT_SSL_AUTH_FAIL = 5, + WGET_EXIT_SERVER_AUTH_FAIL = 6, + WGET_EXIT_PROTOCOL_ERROR = 7, + WGET_EXIT_SERVER_ERROR = 8, + + WGET_EXIT_UNKNOWN + }; + +void inform_exit_status (uerr_t err); + +int get_exit_status (void); + + +#endif /* WGET_EXITS_H */ diff --git a/src/ftp-basic.c b/src/ftp-basic.c new file mode 100644 index 0000000..197cd83 --- /dev/null +++ b/src/ftp-basic.c @@ -0,0 +1,1342 @@ +/* Basic FTP routines. + Copyright (C) 1996-2011, 2014-2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <assert.h> +#include <stdio.h> +#include <stdlib.h> +#include <errno.h> + +#include <string.h> +#include <unistd.h> +#include "utils.h" +#include "connect.h" +#include "host.h" +#include "ftp.h" +#include "retr.h" +#include "c-strcase.h" + + +/* Get the response of FTP server and allocate enough room to handle + it. <CR> and <LF> characters are stripped from the line, and the + line is 0-terminated. All the response lines but the last one are + skipped. The last line is determined as described in RFC959. + + If the line is successfully read, FTPOK is returned, and *ret_line + is assigned a freshly allocated line. Otherwise, FTPRERR is + returned, and the value of *ret_line should be ignored. */ + +uerr_t +ftp_response (int fd, char **ret_line) +{ + for (;;) + { + char *p; + char *line = fd_read_line (fd); + if (!line) + return FTPRERR; + + /* Strip trailing CRLF before printing the line, so that + quoting doesn't include bogus \012 and \015. */ + if ((p = strpbrk(line , "\r\n"))) + *p = 0; + + if (opt.server_response) + logprintf (LOG_NOTQUIET, "%s\n", + quotearg_style (escape_quoting_style, line)); + else + DEBUGP (("%s\n", quotearg_style (escape_quoting_style, line))); + + /* The last line of output is the one that begins with "ddd ". */ + if (c_isdigit (line[0]) && c_isdigit (line[1]) && c_isdigit (line[2]) + && line[3] == ' ') + { + *ret_line = line; + return FTPOK; + } + xfree (line); + } +} + +/* Returns the malloc-ed FTP request, ending with <CR><LF>, printing + it if printing is required. If VALUE is NULL, just use + command<CR><LF>. */ +static char * +ftp_request (const char *command, const char *value) +{ + char *res; + + if (value) + { + char *defanged = NULL, buf[256]; + + /* Check for newlines in VALUE (possibly injected by the %0A URL + escape) making the callers inadvertently send multiple FTP + commands at once. Without this check an attacker could + intentionally redirect to ftp://server/fakedir%0Acommand.../ + and execute arbitrary FTP command on a remote FTP server. */ + if (strpbrk (value, "\r\n")) + { + /* Copy VALUE to the stack and modify CR/LF to space. */ + char *p; + size_t len = strlen(value); + + if (len < sizeof (buf)) + defanged = buf; + else + defanged = xmalloc (len + 1); + + memcpy (defanged, value, len + 1); + + for (p = defanged; *p; p++) + if (*p == '\r' || *p == '\n') + *p = ' '; + + DEBUGP (("\nDetected newlines in %s \"%s\"; changing to %s \"%s\"\n", + command, quotearg_style (escape_quoting_style, value), + command, quotearg_style (escape_quoting_style, defanged))); + + /* Make VALUE point to the defanged copy of the string. */ + value = defanged; + } + res = concat_strings (command, " ", value, "\r\n", (char *) 0); + + if (defanged != buf) + xfree (defanged); + } + else + res = concat_strings (command, "\r\n", (char *) 0); + if (opt.server_response) + { + /* Hack: don't print out password. */ + if (strncmp (res, "PASS", 4) != 0) + logprintf (LOG_ALWAYS, "--> %s\n", res); + else + logputs (LOG_ALWAYS, "--> PASS Turtle Power!\n\n"); + } + else + DEBUGP (("\n--> %s\n", res)); + return res; +} + +uerr_t +ftp_greeting (int csock) +{ + uerr_t err = FTPOK; + char *response = NULL; + + err = ftp_response (csock, &response); + if (err != FTPOK) + goto bail; + if (*response != '2') + err = FTPSRVERR; + +bail: + if (response) + xfree (response); + return err; +} +/* Sends the USER and PASS commands to the server, to control + connection socket csock. */ +uerr_t +ftp_login (int csock, const char *acc, const char *pass) +{ + uerr_t err; + char *request, *respline; + int nwritten; + + /* Send USER username. */ + request = ftp_request ("USER", acc); + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + return WRITEFAILED; + } + xfree (request); + /* Get appropriate response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + return err; + /* An unprobable possibility of logging without a password. */ + if (*respline == '2') + { + xfree (respline); + return FTPOK; + } + /* Else, only response 3 is appropriate. */ + if (*respline != '3') + { + xfree (respline); + return FTPLOGREFUSED; + } +#ifdef ENABLE_OPIE + { + static const char *skey_head[] = { + "331 s/key ", + "331 opiekey " + }; + size_t i; + const char *seed = NULL; + + for (i = 0; i < countof (skey_head); i++) + { + int l = strlen (skey_head[i]); + if (0 == c_strncasecmp (skey_head[i], respline, l)) + { + seed = respline + l; + break; + } + } + if (seed) + { + int skey_sequence = 0; + + /* Extract the sequence from SEED. */ + for (; c_isdigit (*seed); seed++) + skey_sequence = 10 * skey_sequence + *seed - '0'; + if (*seed == ' ') + ++seed; + else + { + xfree (respline); + return FTPLOGREFUSED; + } + /* Replace the password with the SKEY response to the + challenge. */ + pass = skey_response (skey_sequence, seed, pass); + } + } +#endif /* ENABLE_OPIE */ + xfree (respline); + /* Send PASS password. */ + request = ftp_request ("PASS", pass); + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + return WRITEFAILED; + } + xfree (request); + /* Get appropriate response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + return err; + if (*respline != '2') + { + xfree (respline); + return FTPLOGINC; + } + xfree (respline); + /* All OK. */ + return FTPOK; +} + +static void +ip_address_to_port_repr (const ip_address *addr, int port, char *buf, + size_t buflen) +{ + unsigned char *ptr; + + assert (addr->family == AF_INET); + /* buf must contain the argument of PORT (of the form a,b,c,d,e,f). */ + assert (buflen >= 6 * 4); + + ptr = IP_INADDR_DATA (addr); + snprintf (buf, buflen, "%d,%d,%d,%d,%d,%d", ptr[0], ptr[1], + ptr[2], ptr[3], (port & 0xff00) >> 8, port & 0xff); + buf[buflen - 1] = '\0'; +} + +/* Bind a port and send the appropriate PORT command to the FTP + server. Use acceptport after RETR, to get the socket of data + connection. */ +uerr_t +ftp_port (int csock, int *local_sock) +{ + uerr_t err; + char *request, *respline; + ip_address addr; + int nwritten; + int port; + /* Must contain the argument of PORT (of the form a,b,c,d,e,f). */ + char bytes[6 * 4 + 1]; + + /* Get the address of this side of the connection. */ + if (!socket_ip_address (csock, &addr, ENDPOINT_LOCAL)) + return FTPSYSERR; + + assert (addr.family == AF_INET); + + /* Setting port to 0 lets the system choose a free port. */ + port = 0; + + /* Bind the port. */ + *local_sock = bind_local (&addr, &port); + if (*local_sock < 0) + return FTPSYSERR; + + /* Construct the argument of PORT (of the form a,b,c,d,e,f). */ + ip_address_to_port_repr (&addr, port, bytes, sizeof (bytes)); + + /* Send PORT request. */ + request = ftp_request ("PORT", bytes); + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + fd_close (*local_sock); + return WRITEFAILED; + } + xfree (request); + + /* Get appropriate response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + { + fd_close (*local_sock); + return err; + } + if (*respline != '2') + { + xfree (respline); + fd_close (*local_sock); + return FTPPORTERR; + } + xfree (respline); + return FTPOK; +} + +#ifdef ENABLE_IPV6 +static void +ip_address_to_lprt_repr (const ip_address *addr, int port, char *buf, + size_t buflen) +{ + unsigned char *ptr = IP_INADDR_DATA (addr); + + /* buf must contain the argument of LPRT (of the form af,n,h1,h2,...,hn,p1,p2). */ + assert (buflen >= 21 * 4); + + /* Construct the argument of LPRT (of the form af,n,h1,h2,...,hn,p1,p2). */ + switch (addr->family) + { + case AF_INET: + snprintf (buf, buflen, "%d,%d,%d,%d,%d,%d,%d,%d,%d", 4, 4, + ptr[0], ptr[1], ptr[2], ptr[3], 2, + (port & 0xff00) >> 8, port & 0xff); + break; + case AF_INET6: + snprintf (buf, buflen, + "%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d", + 6, 16, + ptr[0], ptr[1], ptr[2], ptr[3], ptr[4], ptr[5], ptr[6], ptr[7], + ptr[8], ptr[9], ptr[10], ptr[11], ptr[12], ptr[13], ptr[14], ptr[15], + 2, (port & 0xff00) >> 8, port & 0xff); + break; + default: + abort (); + } +} + +/* Bind a port and send the appropriate PORT command to the FTP + server. Use acceptport after RETR, to get the socket of data + connection. */ +uerr_t +ftp_lprt (int csock, int *local_sock) +{ + uerr_t err; + char *request, *respline; + ip_address addr; + int nwritten; + int port; + /* Must contain the argument of LPRT (of the form af,n,h1,h2,...,hn,p1,p2). */ + char bytes[21 * 4 + 1]; + + /* Get the address of this side of the connection. */ + if (!socket_ip_address (csock, &addr, ENDPOINT_LOCAL)) + return FTPSYSERR; + + assert (addr.family == AF_INET || addr.family == AF_INET6); + + /* Setting port to 0 lets the system choose a free port. */ + port = 0; + + /* Bind the port. */ + *local_sock = bind_local (&addr, &port); + if (*local_sock < 0) + return FTPSYSERR; + + /* Construct the argument of LPRT (of the form af,n,h1,h2,...,hn,p1,p2). */ + ip_address_to_lprt_repr (&addr, port, bytes, sizeof (bytes)); + + /* Send PORT request. */ + request = ftp_request ("LPRT", bytes); + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + fd_close (*local_sock); + return WRITEFAILED; + } + xfree (request); + /* Get appropriate response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + { + fd_close (*local_sock); + return err; + } + if (*respline != '2') + { + xfree (respline); + fd_close (*local_sock); + return FTPPORTERR; + } + xfree (respline); + return FTPOK; +} + +static void +ip_address_to_eprt_repr (const ip_address *addr, int port, char *buf, + size_t buflen) +{ + int afnum; + + /* buf must contain the argument of EPRT (of the form |af|addr|port|). + * 4 chars for the | separators, INET6_ADDRSTRLEN chars for addr + * 1 char for af (1-2) and 5 chars for port (0-65535) */ + assert (buflen >= 4 + INET6_ADDRSTRLEN + 1 + 5); + + /* Construct the argument of EPRT (of the form |af|addr|port|). */ + afnum = (addr->family == AF_INET ? 1 : 2); + snprintf (buf, buflen, "|%d|%s|%d|", afnum, print_address (addr), port); + buf[buflen - 1] = '\0'; +} + +/* Bind a port and send the appropriate PORT command to the FTP + server. Use acceptport after RETR, to get the socket of data + connection. */ +uerr_t +ftp_eprt (int csock, int *local_sock) +{ + uerr_t err; + char *request, *respline; + ip_address addr; + int nwritten; + int port; + /* Must contain the argument of EPRT (of the form |af|addr|port|). + * 4 chars for the | separators, INET6_ADDRSTRLEN chars for addr + * 1 char for af (1-2) and 5 chars for port (0-65535) */ + char bytes[4 + INET6_ADDRSTRLEN + 1 + 5 + 1]; + + /* Get the address of this side of the connection. */ + if (!socket_ip_address (csock, &addr, ENDPOINT_LOCAL)) + return FTPSYSERR; + + /* Setting port to 0 lets the system choose a free port. */ + port = 0; + + /* Bind the port. */ + *local_sock = bind_local (&addr, &port); + if (*local_sock < 0) + return FTPSYSERR; + + /* Construct the argument of EPRT (of the form |af|addr|port|). */ + ip_address_to_eprt_repr (&addr, port, bytes, sizeof (bytes)); + + /* Send PORT request. */ + request = ftp_request ("EPRT", bytes); + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + fd_close (*local_sock); + return WRITEFAILED; + } + xfree (request); + /* Get appropriate response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + { + fd_close (*local_sock); + return err; + } + if (*respline != '2') + { + xfree (respline); + fd_close (*local_sock); + return FTPPORTERR; + } + xfree (respline); + return FTPOK; +} +#endif + +#ifdef HAVE_SSL +/* + * The following three functions defined into this #ifdef block + * wrap the extended FTP commands defined in RFC 2228 (FTP Security Extensions). + * Currently, only FTPS is supported, so these functions are only compiled when SSL + * support is available, because there's no point in using FTPS when there's no SSL. + * Shall someone add new secure FTP protocols in the future, feel free to remove this + * #ifdef, or add new constants to it. + */ + +/* + * Sends an AUTH command as defined by RFC 2228, + * deriving its argument from the scheme. For example, if the provided scheme + * is SCHEME_FTPS, the command sent will be "AUTH TLS". Currently, this is the only + * scheme supported, so this function will return FTPNOAUTH when supplied a different + * one. It will also return FTPNOAUTH if the target server does not support FTPS. + */ +uerr_t +ftp_auth (int csock, enum url_scheme scheme) +{ + uerr_t err = 0; + int written = 0; + char *request = NULL, *response = NULL; + + if (scheme == SCHEME_FTPS) + { + request = ftp_request ("AUTH", "TLS"); + written = fd_write (csock, request, strlen (request), -1); + if (written < 0) + { + err = WRITEFAILED; + goto bail; + } + err = ftp_response (csock, &response); + if (err != FTPOK) + goto bail; + if (*response != '2') + err = FTPNOAUTH; + } + else + err = FTPNOAUTH; + +bail: + xfree (request); + xfree (response); + + return err; +} + +uerr_t +ftp_pbsz (int csock, int pbsz) +{ + uerr_t err = 0; + int written = 0; + char spbsz[5]; + char *request = NULL, *response = NULL; + + snprintf (spbsz, 5, "%d", pbsz); + request = ftp_request ("PBSZ", spbsz); + written = fd_write (csock, request, strlen (request), -1); + if (written < 0) + { + err = WRITEFAILED; + goto bail; + } + + err = ftp_response (csock, &response); + if (err != FTPOK) + goto bail; + if (*response != '2') + err = FTPNOPBSZ; + +bail: + xfree (request); + xfree (response); + + return err; +} + +uerr_t +ftp_prot (int csock, enum prot_level prot) +{ + uerr_t err = 0; + int written = 0; + char *request = NULL, *response = NULL; + /* value must be a single character value */ + char value[2]; + + value[0] = prot; + value[1] = '\0'; + + request = ftp_request ("PROT", value); + written = fd_write (csock, request, strlen (request), -1); + if (written < 0) + { + err = WRITEFAILED; + goto bail; + } + + err = ftp_response (csock, &response); + if (err != FTPOK) + goto bail; + if (*response != '2') + err = FTPNOPROT; + +bail: + xfree (request); + xfree (response); + + return err; +} +#endif /* HAVE_SSL */ + +/* Similar to ftp_port, but uses `PASV' to initiate the passive FTP + transfer. Reads the response from server and parses it. Reads the + host and port addresses and returns them. */ +uerr_t +ftp_pasv (int csock, ip_address *addr, int *port) +{ + char *request, *respline, *s; + int nwritten, i; + uerr_t err; + unsigned char tmp[6]; + + assert (addr != NULL); + assert (port != NULL); + + xzero (*addr); + + /* Form the request. */ + request = ftp_request ("PASV", NULL); + /* And send it. */ + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + return WRITEFAILED; + } + xfree (request); + /* Get the server response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + return err; + if (*respline != '2') + { + xfree (respline); + return FTPNOPASV; + } + /* Parse the request. */ + s = respline; + for (s += 4; *s && !c_isdigit (*s); s++) + ; + if (!*s) + { + xfree (respline); + return FTPINVPASV; + } + for (i = 0; i < 6; i++) + { + tmp[i] = 0; + for (; c_isdigit (*s); s++) + tmp[i] = (*s - '0') + 10 * tmp[i]; + if (*s == ',') + s++; + else if (i < 5) + { + /* When on the last number, anything can be a terminator. */ + xfree (respline); + return FTPINVPASV; + } + } + xfree (respline); + + addr->family = AF_INET; + memcpy (IP_INADDR_DATA (addr), tmp, 4); + *port = ((tmp[4] << 8) & 0xff00) + tmp[5]; + + return FTPOK; +} + +#ifdef ENABLE_IPV6 +/* Similar to ftp_lprt, but uses `LPSV' to initiate the passive FTP + transfer. Reads the response from server and parses it. Reads the + host and port addresses and returns them. */ +uerr_t +ftp_lpsv (int csock, ip_address *addr, int *port) +{ + char *request, *respline, *s; + int nwritten, i, af, addrlen, portlen; + uerr_t err; + unsigned char tmp[16]; + unsigned char tmpprt[2]; + + assert (addr != NULL); + assert (port != NULL); + + xzero (*addr); + + /* Form the request. */ + request = ftp_request ("LPSV", NULL); + + /* And send it. */ + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + return WRITEFAILED; + } + xfree (request); + + /* Get the server response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + return err; + if (*respline != '2') + { + xfree (respline); + return FTPNOPASV; + } + + /* Parse the response. */ + s = respline; + for (s += 4; *s && !c_isdigit (*s); s++) + ; + if (!*s) + { + xfree (respline); + return FTPINVPASV; + } + + /* First, get the address family */ + af = 0; + for (; c_isdigit (*s); s++) + af = (*s - '0') + 10 * af; + + if (af != 4 && af != 6) + { + xfree (respline); + return FTPINVPASV; + } + + if (!*s || *s++ != ',') + { + xfree (respline); + return FTPINVPASV; + } + + /* Then, get the address length */ + addrlen = 0; + for (; c_isdigit (*s); s++) + addrlen = (*s - '0') + 10 * addrlen; + + if (!*s || *s++ != ',') + { + xfree (respline); + return FTPINVPASV; + } + + if (addrlen > 16) + { + xfree (respline); + return FTPINVPASV; + } + + if ((af == 4 && addrlen != 4) + || (af == 6 && addrlen != 16)) + { + xfree (respline); + return FTPINVPASV; + } + + /* Now, we get the actual address */ + for (i = 0; i < addrlen; i++) + { + tmp[i] = 0; + for (; c_isdigit (*s); s++) + tmp[i] = (*s - '0') + 10 * tmp[i]; + if (*s == ',') + s++; + else + { + xfree (respline); + return FTPINVPASV; + } + } + + /* Now, get the port length */ + portlen = 0; + for (; c_isdigit (*s); s++) + portlen = (*s - '0') + 10 * portlen; + + if (!*s || *s++ != ',') + { + xfree (respline); + return FTPINVPASV; + } + + if (portlen > 2) + { + xfree (respline); + return FTPINVPASV; + } + + /* Finally, we get the port number */ + tmpprt[0] = 0; + for (; c_isdigit (*s); s++) + tmpprt[0] = (*s - '0') + 10 * tmpprt[0]; + + if (!*s || *s++ != ',') + { + xfree (respline); + return FTPINVPASV; + } + + tmpprt[1] = 0; + for (; c_isdigit (*s); s++) + tmpprt[1] = (*s - '0') + 10 * tmpprt[1]; + + assert (s != NULL); + + if (af == 4) + { + addr->family = AF_INET; + memcpy (IP_INADDR_DATA (addr), tmp, 4); + *port = ((tmpprt[0] << 8) & 0xff00) + tmpprt[1]; + DEBUGP (("lpsv addr is: %s\n", print_address(addr))); + DEBUGP (("tmpprt[0] is: %d\n", tmpprt[0])); + DEBUGP (("tmpprt[1] is: %d\n", tmpprt[1])); + DEBUGP (("*port is: %d\n", *port)); + } + else + { + assert (af == 6); + addr->family = AF_INET6; + memcpy (IP_INADDR_DATA (addr), tmp, 16); + *port = ((tmpprt[0] << 8) & 0xff00) + tmpprt[1]; + DEBUGP (("lpsv addr is: %s\n", print_address(addr))); + DEBUGP (("tmpprt[0] is: %d\n", tmpprt[0])); + DEBUGP (("tmpprt[1] is: %d\n", tmpprt[1])); + DEBUGP (("*port is: %d\n", *port)); + } + + xfree (respline); + return FTPOK; +} + +/* Similar to ftp_eprt, but uses `EPSV' to initiate the passive FTP + transfer. Reads the response from server and parses it. Reads the + host and port addresses and returns them. */ +uerr_t +ftp_epsv (int csock, ip_address *ip, int *port) +{ + char *request, *respline, *start, delim, *s; + int nwritten, i; + uerr_t err; + int tport; + + assert (ip != NULL); + assert (port != NULL); + + /* IP already contains the IP address of the control connection's + peer, so we don't need to call socket_ip_address here. */ + + /* Form the request. */ + /* EPSV 1 means that we ask for IPv4 and EPSV 2 means that we ask for IPv6. */ + request = ftp_request ("EPSV", (ip->family == AF_INET ? "1" : "2")); + + /* And send it. */ + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + return WRITEFAILED; + } + xfree (request); + + /* Get the server response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + return err; + if (*respline != '2') + { + xfree (respline); + return FTPNOPASV; + } + + assert (respline != NULL); + + DEBUGP(("respline is %s\n", respline)); + + /* Skip the useless stuff and get what's inside the parentheses */ + start = strchr (respline, '('); + if (start == NULL) + { + xfree (respline); + return FTPINVPASV; + } + + /* Skip the first two void fields */ + s = start + 1; + delim = *s++; + if (delim < 33 || delim > 126) + { + xfree (respline); + return FTPINVPASV; + } + + for (i = 0; i < 2; i++) + { + if (*s++ != delim) + { + xfree (respline); + return FTPINVPASV; + } + } + + /* Finally, get the port number */ + for (tport = 0, i = 0; i < 5 && c_isdigit (*s); i++, s++) + tport = (*s - '0') + 10 * tport; + + /* Make sure that the response terminates correctly */ + if (*s++ != delim) + { + xfree (respline); + return FTPINVPASV; + } + + if (*s != ')') + { + xfree (respline); + return FTPINVPASV; + } + + *port = tport; + + xfree (respline); + return FTPOK; +} +#endif + +/* Sends the TYPE request to the server. */ +uerr_t +ftp_type (int csock, int type) +{ + char *request, *respline; + int nwritten; + uerr_t err; + char stype[2]; + + /* Construct argument. */ + stype[0] = type; + stype[1] = 0; + /* Send TYPE request. */ + request = ftp_request ("TYPE", stype); + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + return WRITEFAILED; + } + xfree (request); + /* Get appropriate response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + return err; + if (*respline != '2') + { + xfree (respline); + return FTPUNKNOWNTYPE; + } + xfree (respline); + /* All OK. */ + return FTPOK; +} + +/* Changes the working directory by issuing a CWD command to the + server. */ +uerr_t +ftp_cwd (int csock, const char *dir) +{ + char *request, *respline; + int nwritten; + uerr_t err; + + /* Send CWD request. */ + request = ftp_request ("CWD", dir); + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + return WRITEFAILED; + } + xfree (request); + /* Get appropriate response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + return err; + if (*respline == '5') + { + xfree (respline); + return FTPNSFOD; + } + if (*respline != '2') + { + xfree (respline); + return FTPRERR; + } + xfree (respline); + /* All OK. */ + return FTPOK; +} + +/* Sends REST command to the FTP server. */ +uerr_t +ftp_rest (int csock, wgint offset) +{ + char *request, *respline; + int nwritten; + uerr_t err; + + request = ftp_request ("REST", number_to_static_string (offset)); + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + return WRITEFAILED; + } + xfree (request); + /* Get appropriate response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + return err; + if (*respline != '3') + { + xfree (respline); + return FTPRESTFAIL; + } + xfree (respline); + /* All OK. */ + return FTPOK; +} + +/* Sends RETR command to the FTP server. */ +uerr_t +ftp_retr (int csock, const char *file) +{ + char *request, *respline; + int nwritten; + uerr_t err; + + /* Send RETR request. */ + request = ftp_request ("RETR", file); + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + return WRITEFAILED; + } + xfree (request); + /* Get appropriate response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + return err; + if (*respline == '5') + { + xfree (respline); + return FTPNSFOD; + } + if (*respline != '1') + { + xfree (respline); + return FTPRERR; + } + xfree (respline); + /* All OK. */ + return FTPOK; +} + +/* Sends the LIST command to the server. If FILE is NULL, send just + `LIST' (no space). */ +uerr_t +ftp_list (int csock, const char *file, bool avoid_list_a, bool avoid_list, + bool *list_a_used) +{ + char *request, *respline; + int nwritten; + uerr_t err; + bool ok = false; + size_t i = 0; + + /* 2013-10-12 Andrea Urbani (matfanjol) + For more information about LIST and "LIST -a" please look at ftp.c, + function getftp, text "__LIST_A_EXPLANATION__". + + If somebody changes the following commands, please, checks also the + later "i" variable. */ + static const char *list_commands[] = { + "LIST -a", + "LIST" + }; + + *list_a_used = false; + + if (avoid_list_a) + { + i = countof (list_commands)- 1; + DEBUGP (("(skipping \"LIST -a\")")); + } + + + do { + /* Send request. */ + request = ftp_request (list_commands[i], file); + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + return WRITEFAILED; + } + xfree (request); + /* Get appropriate response. */ + err = ftp_response (csock, &respline); + if (err == FTPOK) + { + if (*respline == '5') + { + err = FTPNSFOD; + } + else if (*respline == '1') + { + err = FTPOK; + ok = true; + /* Which list command was used? */ + *list_a_used = (i == 0); + } + else + { + err = FTPRERR; + } + xfree (respline); + } + ++i; + if ((avoid_list) && (i == 1)) + { + /* I skip LIST */ + ++i; + DEBUGP (("(skipping \"LIST\")")); + } + } while (i < countof (list_commands) && !ok); + + return err; +} + +/* Sends the SYST command to the server. */ +uerr_t +ftp_syst (int csock, enum stype *server_type, enum ustype *unix_type) +{ + char *request, *respline; + int nwritten; + uerr_t err; + char *ftp_last_respline; + + /* Send SYST request. */ + request = ftp_request ("SYST", NULL); + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + return WRITEFAILED; + } + xfree (request); + + /* Get appropriate response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + return err; + if (*respline == '5') + { + xfree (respline); + return FTPSRVERR; + } + + ftp_last_respline = strdup (respline); + + /* Skip the number (215, but 200 (!!!) in case of VMS) */ + strtok (respline, " "); + + /* Which system type has been reported (we are interested just in the + first word of the server response)? */ + request = strtok (NULL, " "); + + *unix_type = UST_OTHER; + + if (request == NULL) + *server_type = ST_OTHER; + else if (!c_strcasecmp (request, "VMS")) + *server_type = ST_VMS; + else if (!c_strcasecmp (request, "UNIX")) + { + *server_type = ST_UNIX; + /* 2013-10-17 Andrea Urbani (matfanjol) + I check more in depth the system type */ + if (!c_strncasecmp (ftp_last_respline, "215 UNIX Type: L8", 17)) + *unix_type = UST_TYPE_L8; + else if (!c_strncasecmp (ftp_last_respline, + "215 UNIX MultiNet Unix Emulation V5.3(93)", 41)) + *unix_type = UST_MULTINET; + } + else if (!c_strcasecmp (request, "WINDOWS_NT") + || !c_strcasecmp (request, "WINDOWS2000")) + *server_type = ST_WINNT; + else if (!c_strcasecmp (request, "MACOS")) + *server_type = ST_MACOS; + else if (!c_strcasecmp (request, "OS/400")) + *server_type = ST_OS400; + else + *server_type = ST_OTHER; + + xfree (ftp_last_respline); + xfree (respline); + /* All OK. */ + return FTPOK; +} + +/* Sends the PWD command to the server. */ +uerr_t +ftp_pwd (int csock, char **pwd) +{ + char *request, *respline; + int nwritten; + uerr_t err; + + /* Send PWD request. */ + request = ftp_request ("PWD", NULL); + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + return WRITEFAILED; + } + xfree (request); + /* Get appropriate response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + return err; + if (*respline == '5') + { + err: + xfree (respline); + return FTPSRVERR; + } + + /* Skip the number (257), leading citation mark, trailing citation mark + and everything following it. */ + strtok (respline, "\""); + request = strtok (NULL, "\""); + if (!request) + /* Treat the malformed response as an error, which the caller has + to handle gracefully anyway. */ + goto err; + + /* Has the `pwd' been already allocated? Free! */ + xfree (*pwd); + + *pwd = xstrdup (request); + + xfree (respline); + /* All OK. */ + return FTPOK; +} + +/* Sends the SIZE command to the server, and returns the value in 'size'. + * If an error occurs, size is set to zero. */ +uerr_t +ftp_size (int csock, const char *file, wgint *size) +{ + char *request, *respline; + int nwritten; + uerr_t err; + + /* Send PWD request. */ + request = ftp_request ("SIZE", file); + nwritten = fd_write (csock, request, strlen (request), -1); + if (nwritten < 0) + { + xfree (request); + *size = 0; + return WRITEFAILED; + } + xfree (request); + /* Get appropriate response. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + { + *size = 0; + return err; + } + if (*respline == '5') + { + /* + * Probably means SIZE isn't supported on this server. + * Error is nonfatal since SIZE isn't in RFC 959 + */ + xfree (respline); + *size = 0; + return FTPOK; + } + + errno = 0; + *size = str_to_wgint (respline + 4, NULL, 10); + if (errno) + { + /* + * Couldn't parse the response for some reason. On the (few) + * tests I've done, the response is 213 <SIZE> with nothing else - + * maybe something a bit more resilient is necessary. It's not a + * fatal error, however. + */ + xfree (respline); + *size = 0; + return FTPOK; + } + + xfree (respline); + /* All OK. */ + return FTPOK; +} + +/* If URL's params are of the form "type=X", return character X. + Otherwise, return 'I' (the default type). */ +char +ftp_process_type (const char *params) +{ + if (params + && 0 == strncasecmp (params, "type=", 5) + && params[5] != '\0') + return c_toupper (params[5]); + else + return 'I'; +} diff --git a/src/ftp-ls.c b/src/ftp-ls.c new file mode 100644 index 0000000..bb25754 --- /dev/null +++ b/src/ftp-ls.c @@ -0,0 +1,1175 @@ +/* Parsing FTP `ls' output. + Copyright (C) 1996-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> +#include <time.h> +#include "utils.h" +#include "ftp.h" +#include "url.h" +#include "convert.h" /* for html_quote_string prototype */ +#include "retr.h" /* for output_stream */ +#include "c-strcase.h" + +/* Converts symbolic permissions to number-style ones, e.g. string + rwxr-xr-x to 755. For now, it knows nothing of + setuid/setgid/sticky. ACLs are ignored. */ +static int +symperms (const char *s) +{ + int perms = 0, i; + + if (strlen (s) < 9) + return 0; + for (i = 0; i < 3; i++, s += 3) + { + perms <<= 3; + perms += (((s[0] == 'r') << 2) + ((s[1] == 'w') << 1) + + (s[2] == 'x' || s[2] == 's')); + } + return perms; +} + + +/* Cleans a line of text so that it can be consistently parsed. Destroys + <CR> and <LF> in case that they occur at the end of the line and + replaces all <TAB> character with <SPACE>. Returns the length of the + modified line. */ +static int +clean_line (char *line, int len) +{ + if (len <= 0) return 0; + + while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) + line[--len] = '\0'; + + if (!len) return 0; + + for ( ; *line ; line++ ) if (*line == '\t') *line = ' '; + + return len; +} + +/* Convert the Un*x-ish style directory listing stored in FILE to a + linked list of fileinfo (system-independent) entries. The contents + of FILE are considered to be produced by the standard Unix `ls -la' + output (whatever that might be). BSD (no group) and SYSV (with + group) listings are handled. + + The time stamps are stored in a separate variable, time_t + compatible (I hope). The timezones are ignored. */ +static struct fileinfo * +ftp_parse_unix_ls (FILE *fp, int ignore_perms) +{ + static const char *months[] = { + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" + }; + int next, len, i, error, ignore; + int year, month, day; /* for time analysis */ + int hour, min, sec, ptype; + struct tm timestruct, *tnow; + time_t timenow; + size_t bufsize = 0; + + char *line = NULL, *tok, *ptok; /* tokenizer */ + struct fileinfo *dir, *l, cur; /* list creation */ + + dir = l = NULL; + + /* Line loop to end of file: */ + while ((len = getline (&line, &bufsize, fp)) > 0) + { + len = clean_line (line, len); + /* Skip if total... */ + if (!c_strncasecmp (line, "total", 5)) + continue; + /* Get the first token (permissions). */ + tok = strtok (line, " "); + if (!tok) + continue; + + cur.name = NULL; + cur.linkto = NULL; + + /* Decide whether we deal with a file or a directory. */ + switch (*tok) + { + case '-': + cur.type = FT_PLAINFILE; + DEBUGP (("PLAINFILE; ")); + break; + case 'd': + cur.type = FT_DIRECTORY; + DEBUGP (("DIRECTORY; ")); + break; + case 'l': + cur.type = FT_SYMLINK; + DEBUGP (("SYMLINK; ")); + break; + default: + cur.type = FT_UNKNOWN; + DEBUGP (("UNKNOWN; ")); + break; + } + + if (ignore_perms) + { + switch (cur.type) + { + case FT_PLAINFILE: + cur.perms = 0644; + break; + case FT_DIRECTORY: + cur.perms = 0755; + break; + default: + /*cur.perms = 1023;*/ /* #### What is this? --hniksic */ + cur.perms = 0644; + } + DEBUGP (("implicit perms %0o; ", (unsigned) cur.perms)); + } + else + { + cur.perms = symperms (tok + 1); + DEBUGP (("perms %0o; ", (unsigned) cur.perms)); + } + + error = ignore = 0; /* Erroneous and ignoring entries are + treated equally for now. */ + year = hour = min = sec = 0; /* Silence the compiler. */ + month = day = 0; + ptype = TT_DAY; + next = -1; + /* While there are tokens on the line, parse them. Next is the + number of tokens left until the filename. + + Use the month-name token as the "anchor" (the place where the + position wrt the file name is "known"). When a month name is + encountered, `next' is set to 5. Also, the preceding + characters are parsed to get the file size. + + This tactic is quite dubious when it comes to + internationalization issues (non-English month names), but it + works for now. */ + tok = line; + while (ptok = tok, + (tok = strtok (NULL, " ")) != NULL) + { + --next; + if (next < 0) /* a month name was not encountered */ + { + for (i = 0; i < 12; i++) + if (!c_strcasecmp (tok, months[i])) + break; + /* If we got a month, it means the token before it is the + size, and the filename is three tokens away. */ + if (i != 12) + { + wgint size; + + /* Parse the previous token with str_to_wgint. */ + if (ptok == line) + { + /* Something has gone wrong during parsing. */ + error = 1; + break; + } + errno = 0; + size = str_to_wgint (ptok, NULL, 10); + if (size == WGINT_MAX && errno == ERANGE) + /* Out of range -- ignore the size. #### Should + we refuse to start the download. */ + cur.size = 0; + else + cur.size = size; + DEBUGP (("size: %s; ", number_to_static_string(cur.size))); + + month = i; + next = 5; + DEBUGP (("month: %s; ", months[month])); + } + } + else if (next == 4) /* days */ + { + if (tok[1]) /* two-digit... */ + day = 10 * (*tok - '0') + tok[1] - '0'; + else /* ...or one-digit */ + day = *tok - '0'; + DEBUGP (("day: %d; ", day)); + } + else if (next == 3) + { + /* This ought to be either the time, or the year. Let's + be flexible! + + If we have a number x, it's a year. If we have x:y, + it's hours and minutes. If we have x:y:z, z are + seconds. */ + year = 0; + min = hour = sec = 0; + /* We must deal with digits. */ + if (c_isdigit (*tok)) + { + /* Suppose it's year. Limit to year 99999 to avoid integer overflow. */ + for (; c_isdigit (*tok) && year <= 99999; tok++) + year = (*tok - '0') + 10 * year; + if (*tok == ':') + { + int n; + /* This means these were hours! */ + hour = year; + year = 0; + ptype = TT_HOUR_MIN; + ++tok; + /* Get the minutes... */ + for (n = 0; c_isdigit (*tok) && n < 2; tok++, n++) + min = (*tok - '0') + 10 * min; + if (*tok == ':') + { + /* ...and the seconds. */ + ++tok; + for (n = 0; c_isdigit (*tok) && n < 2; tok++, n++) + sec = (*tok - '0') + 10 * sec; + } + } + } + if (year) + DEBUGP (("year: %d (no tm); ", year)); + else + DEBUGP (("time: %02d:%02d:%02d (no yr); ", hour, min, sec)); + } + else if (next == 2) /* The file name */ + { + int fnlen; + char *p; + + /* Since the file name may contain a SPC, it is possible + for strtok to handle it wrong. */ + fnlen = strlen (tok); + if (fnlen < len - (tok - line)) + { + /* So we have a SPC in the file name. Restore the + original. */ + tok[fnlen] = ' '; + /* If the file is a symbolic link, it should have a + ` -> ' somewhere. */ + if (cur.type == FT_SYMLINK) + { + p = strstr (tok, " -> "); + if (!p) + { + error = 1; + break; + } + cur.linkto = xstrdup (p + 4); + DEBUGP (("link to: %s\n", cur.linkto)); + /* And separate it from the file name. */ + *p = '\0'; + } + } + /* If we have the filename, add it to the list of files or + directories. */ + /* "." and ".." are an exception! */ + if (!strcmp (tok, ".") || !strcmp (tok, "..")) + { + DEBUGP (("\nIgnoring `.' and `..'; ")); + ignore = 1; + break; + } + /* Some FTP sites choose to have ls -F as their default + LIST output, which marks the symlinks with a trailing + `@', directory names with a trailing `/' and + executables with a trailing `*'. This is no problem + unless encountering a symbolic link ending with `@', + or an executable ending with `*' on a server without + default -F output. I believe these cases are very + rare. */ + fnlen = strlen (tok); /* re-calculate `fnlen' */ + cur.name = xmalloc (fnlen + 1); + memcpy (cur.name, tok, fnlen + 1); + if (fnlen) + { + if (cur.type == FT_DIRECTORY && cur.name[fnlen - 1] == '/') + { + cur.name[fnlen - 1] = '\0'; + DEBUGP (("trailing `/' on dir.\n")); + } + else if (cur.type == FT_SYMLINK && cur.name[fnlen - 1] == '@') + { + cur.name[fnlen - 1] = '\0'; + DEBUGP (("trailing `@' on link.\n")); + } + else if (cur.type == FT_PLAINFILE + && (cur.perms & 0111) + && cur.name[fnlen - 1] == '*') + { + cur.name[fnlen - 1] = '\0'; + DEBUGP (("trailing `*' on exec.\n")); + } + } /* if (fnlen) */ + else + error = 1; + break; + } + else + abort (); + } /* while */ + + if (!cur.name || (cur.type == FT_SYMLINK && !cur.linkto)) + error = 1; + + DEBUGP (("%s\n", cur.name ? cur.name : "")); + + if (error || ignore) + { + DEBUGP (("Skipping.\n")); + xfree (cur.name); + xfree (cur.linkto); + continue; + } + + if (!dir) + { + l = dir = xnew (struct fileinfo); + memcpy (l, &cur, sizeof (cur)); + l->prev = l->next = NULL; + } + else + { + cur.prev = l; + l->next = xnew (struct fileinfo); + l = l->next; + memcpy (l, &cur, sizeof (cur)); + l->next = NULL; + } + /* Get the current time. */ + timenow = time (NULL); + tnow = localtime (&timenow); + /* Build the time-stamp (the idea by zaga@fly.cc.fer.hr). */ + timestruct.tm_sec = sec; + timestruct.tm_min = min; + timestruct.tm_hour = hour; + timestruct.tm_mday = day; + timestruct.tm_mon = month; + if (year == 0) + { + /* Some listings will not specify the year if it is "obvious" + that the file was from the previous year. E.g. if today + is 97-01-12, and you see a file of Dec 15th, its year is + 1996, not 1997. Thanks to Vladimir Volovich for + mentioning this! */ + if (month > tnow->tm_mon) + timestruct.tm_year = tnow->tm_year - 1; + else + timestruct.tm_year = tnow->tm_year; + } + else + timestruct.tm_year = year; + if (timestruct.tm_year >= 1900) + timestruct.tm_year -= 1900; + timestruct.tm_wday = 0; + timestruct.tm_yday = 0; + timestruct.tm_isdst = -1; + l->tstamp = mktime (×truct); /* store the time-stamp */ + l->ptype = ptype; + } + + xfree (line); + return dir; +} + +static struct fileinfo * +ftp_parse_winnt_ls (FILE *fp) +{ + int len; + int year, month, day; /* for time analysis */ + int hour, min; + size_t bufsize = 0; + struct tm timestruct; + + char *line = NULL, *tok; /* tokenizer */ + char *filename; + struct fileinfo *dir, *l, cur; /* list creation */ + + dir = l = NULL; + cur.name = NULL; + + /* Line loop to end of file: */ + while ((len = getline (&line, &bufsize, fp)) > 0) + { + len = clean_line (line, len); + + /* Name begins at 39 column of the listing if date presented in `mm-dd-yy' + format or at 41 column if date presented in `mm-dd-yyyy' format. Thus, + we cannot extract name before we parse date. Using this information we + also can recognize filenames that begin with a series of space + characters (but who really wants to use such filenames anyway?). */ + if (len < 40) continue; + filename = line + 39; + + /* First column: mm-dd-yy or mm-dd-yyyy. Should atoi() on the month fail, + january will be assumed. */ + tok = strtok(line, "-"); + if (tok == NULL) continue; + month = atoi(tok); + if (month < 0) month = 0; else month--; + tok = strtok(NULL, "-"); + if (tok == NULL) continue; + day = atoi(tok); + tok = strtok(NULL, " "); + if (tok == NULL) continue; + year = atoi(tok); + /* Assuming the epoch starting at 1.1.1970 */ + if (year <= 70) + { + year += 100; + } + else if (year >= 1900) + { + year -= 1900; + if (len < 42) continue; + filename += 2; + } + /* Now it is possible to determine the position of the first symbol in + filename. */ + xfree (cur.name); + memset(&cur, 0, sizeof (cur)); + cur.name = xstrdup(filename); + DEBUGP (("Name: '%s'\n", cur.name)); + + + /* Second column: hh:mm[AP]M, listing does not contain value for + seconds */ + tok = strtok(NULL, ":"); + if (tok == NULL) continue; + hour = atoi(tok); + tok = strtok(NULL, "M"); + if (tok == NULL) continue; + min = atoi(tok); + /* Adjust hour from AM/PM. Just for the record, the sequence goes + 11:00AM, 12:00PM, 01:00PM ... 11:00PM, 12:00AM, 01:00AM . */ + if (tok[0] && tok[1]) tok+=2; + if (hour >= 12 || hour < 0) hour = 0; + if (*tok == 'P') hour += 12; + + DEBUGP (("YYYY/MM/DD HH:MM - %d/%02d/%02d %02d:%02d\n", + year+1900, month, day, hour, min)); + + /* Build the time-stamp (copy & paste from above) */ + timestruct.tm_sec = 0; + timestruct.tm_min = min; + timestruct.tm_hour = hour; + timestruct.tm_mday = day; + timestruct.tm_mon = month; + timestruct.tm_year = year; + timestruct.tm_wday = 0; + timestruct.tm_yday = 0; + timestruct.tm_isdst = -1; + cur.tstamp = mktime (×truct); /* store the time-stamp */ + cur.ptype = TT_HOUR_MIN; + + DEBUGP (("Timestamp: %ld\n", cur.tstamp)); + + /* Third column: Either file length, or <DIR>. We also set the + permissions (guessed as 0644 for plain files and 0755 for + directories as the listing does not give us a clue) and filetype + here. */ + tok = strtok(NULL, " "); + if (tok == NULL) continue; + while ((tok != NULL) && (*tok == '\0')) tok = strtok(NULL, " "); + if (tok == NULL) continue; + if (*tok == '<') + { + cur.type = FT_DIRECTORY; + cur.size = 0; + cur.perms = 0755; + DEBUGP (("Directory\n")); + } + else + { + wgint size; + cur.type = FT_PLAINFILE; + errno = 0; + size = str_to_wgint (tok, NULL, 10); + if (size == WGINT_MAX && errno == ERANGE) + cur.size = 0; /* overflow */ + else + cur.size = size; + cur.perms = 0644; + DEBUGP (("File, size %s bytes\n", number_to_static_string (cur.size))); + } + + cur.linkto = NULL; + + /* And put everything into the linked list */ + if (!dir) + { + l = dir = xnew (struct fileinfo); + memcpy (l, &cur, sizeof (cur)); + l->prev = l->next = NULL; + } + else + { + cur.prev = l; + l->next = xnew (struct fileinfo); + l = l->next; + memcpy (l, &cur, sizeof (cur)); + l->next = NULL; + } + cur.name = NULL; + } + + xfree (cur.name); + xfree (line); + return dir; +} + + + +/* Convert the VMS-style directory listing stored in "file" to a + linked list of fileinfo (system-independent) entries. The contents + of FILE are considered to be produced by the standard VMS + "DIRECTORY [/SIZE [= ALL]] /DATE [/OWNER] [/PROTECTION]" command, + more or less. (Different VMS FTP servers may have different headers, + and may not supply the same data, but all should be subsets of this.) + + VMS normally provides local (server) time and date information. + Define the logical name or environment variable + "WGET_TIMEZONE_DIFFERENTIAL" (seconds) to adjust the receiving local + times if different from the remote local times. + + 2005-02-23 SMS. + Added code to eliminate "^" escape characters from ODS5 extended file + names. The TCPIP FTP server (V5.4) seems to prefer requests which do + not use the escaped names which it provides. +*/ + +#define VMS_DEFAULT_PROT_FILE 0644 +#define VMS_DEFAULT_PROT_DIR 0755 + +/* 2005-02-23 SMS. + eat_carets(). + + Delete ODS5 extended file name escape characters ("^") in the + original buffer. + Note that the current scheme does not handle all EFN cases, but it + could be made more complicated. +*/ + +static void eat_carets( char *str) +/* char *str; Source pointer. */ +{ + char *strd; /* Destination pointer. */ + char hdgt; + unsigned char uchr; + + /* Skip ahead to the first "^", if any. */ + while ((*str != '\0') && (*str != '^')) + str++; + + /* If no caret was found, quit early. */ + if (*str != '\0') + { + /* Shift characters leftward as carets are found. */ + strd = str; + while (*str != '\0') + { + uchr = *str; + if (uchr == '^') + { + /* Found a caret. Skip it, and check the next character. */ + if ((char_prop[(unsigned char) str[1]] & 64) && (char_prop[(unsigned char) str[2]] & 64)) + { + /* Hex digit. Get char code from this and next hex digit. */ + uchr = *(++str); + if (uchr <= '9') + { + hdgt = uchr - '0'; /* '0' - '9' -> 0 - 9. */ + } + else + { + hdgt = ((uchr - 'A') & 7) + 10; /* [Aa] - [Ff] -> 10 - 15. */ + } + hdgt <<= 4; /* X16. */ + uchr = *(++str); /* Next char must be hex digit. */ + if (uchr <= '9') + { + uchr = hdgt + uchr - '0'; + } + else + { + uchr = hdgt + ((uchr - 'A') & 15) + 10; + } + } + else if (uchr == '_') + { + /* Convert escaped "_" to " ". */ + uchr = ' '; + } + else if (uchr == '/') + { + /* Convert escaped "/" (invalid Zip) to "?" (invalid VMS). */ + /* Note that this is a left-over from Info-ZIP code, and is + probably of little value here, except perhaps to avoid + directory confusion which an unconverted slash might cause. + */ + uchr = '?'; + } + /* Else, not a hex digit. Must be a simple escaped character + (or Unicode, which is not yet handled here). + */ + } + /* Else, not a caret. Use as-is. */ + *strd = uchr; + + /* Advance destination and source pointers. */ + strd++; + str++; + } + /* Terminate the destination string. */ + *strd = '\0'; + } +} + + +static struct fileinfo * +ftp_parse_vms_ls (FILE *fp) +{ + int dt, i, j, len; + int perms; + size_t bufsize = 0; + time_t timenow; + struct tm *timestruct; + char date_str[32]; + + char *line = NULL, *tok; /* tokenizer */ + struct fileinfo *dir, *l, cur; /* list creation */ + + dir = l = NULL; + + /* Skip blank lines, Directory heading, and more blank lines. */ + + for (j = 0; (i = getline (&line, &bufsize, fp)) > 0; ) + { + i = clean_line (line, i); + if (i <= 0) + continue; /* Ignore blank line. */ + + if ((j == 0) && (line[i - 1] == ']')) + { + /* Found Directory heading line. Next non-blank line + is significant. */ + j = 1; + } + else if (!strncmp (line, "Total of ", 9)) + { + /* Found "Total of ..." footing line. No valid data + will follow (empty directory). */ + i = 0; /* Arrange for early exit. */ + break; + } + else + { + break; /* Must be significant data. */ + } + } + + /* Read remainder of file until the next blank line or EOF. */ + + cur.name = NULL; + while (i > 0) + { + char *p; + + /* The first token is the file name. After a long name, other + data may be on the following line. A valid directory name ends + in ".DIR;1" (any case), although some VMS FTP servers may omit + the version number (";1"). + */ + + tok = strtok(line, " "); + if (tok == NULL) tok = line; + DEBUGP (("file name: '%s'\n", tok)); + + /* Stripping the version number on a VMS system would be wrong. + It may be foolish on a non-VMS system, too, but that's someone + else's problem. (Define PRESERVE_VMS_VERSIONS for proper + operation on other operating systems.) + + 2005-02-23 SMS. + ODS5 extended file names may contain escaped semi-colons, so + the version number is identified as right-side decimal digits + led by a non-escaped semi-colon. It may be absent. + */ + +#if (!defined( __VMS) && !defined( PRESERVE_VMS_VERSIONS)) + for (p = tok + strlen (tok); (--p > tok) && c_isdigit(*p); ); + if (p > tok && (*p == ';') && (*(p - 1) != '^')) + { + *p = '\0'; + } +#endif /* (!defined( __VMS) && !defined( PRESERVE_VMS_VERSIONS)) */ + + /* 2005-02-23 SMS. + Eliminate "^" escape characters from ODS5 extended file name. + (A caret is invalid in an ODS2 name, so this is always safe.) + */ + eat_carets (tok); + DEBUGP (("file name-^: '%s'\n", tok)); + + /* Differentiate between a directory and any other file. A VMS + listing may not include file protections (permissions). Set a + default permissions value (according to the file type), which + may be overwritten later. Store directory names without the + ".DIR;1" file type and version number, as the plain name is + what will work in a CWD command. + */ + len = strlen (tok); + if (len >= 4 && !c_strncasecmp(tok + (len - 4), ".DIR", 4)) + { + *(tok + (len - 4)) = '\0'; /* Discard ".DIR". */ + cur.type = FT_DIRECTORY; + cur.perms = VMS_DEFAULT_PROT_DIR; + DEBUGP (("Directory (nv)\n")); + } + else if (len >= 6 && !c_strncasecmp (tok + len - 6, ".DIR;1", 6)) + { + *(tok + (len - 6)) = '\0'; /* Discard ".DIR;1". */ + cur.type = FT_DIRECTORY; + cur.perms = VMS_DEFAULT_PROT_DIR; + DEBUGP (("Directory (v)\n")); + } + else + { + cur.type = FT_PLAINFILE; + cur.perms = VMS_DEFAULT_PROT_FILE; + DEBUGP (("File\n")); + } + xfree (cur.name); + cur.name = xstrdup (tok); + DEBUGP (("Name: '%s'\n", cur.name)); + + /* Null the date and time string. */ + *date_str = '\0'; + + /* VMS lacks symbolic links. */ + cur.linkto = NULL; + + /* VMS reports file sizes in (512-byte) disk blocks, not bytes, + hence useless for an integrity check based on byte-count. + Set size to unknown. + */ + cur.size = 0; + + /* Get token 2, if any. A long name may force all other data onto + a second line. If needed, read the second line. + */ + + tok = strtok (NULL, " "); + if (tok == NULL) + { + DEBUGP (("Getting additional line.\n")); + i = getline (&line, &bufsize, fp); + if (i <= 0) + { + DEBUGP (("EOF. Leaving listing parser.\n")); + break; + } + + /* Second line must begin with " ". Otherwise, it's a first + line (and we may be confused). + */ + i = clean_line (line, i); + if (i <= 0) + { + /* Blank line. End of significant file listing. */ + DEBUGP (("Blank line. Leaving listing parser.\n")); + break; + } + else if (line[0] != ' ') + { + DEBUGP (("Non-blank in column 1. Must be a new file name?\n")); + continue; + } + else + { + tok = strtok (line, " "); + if (tok == NULL) + { + /* Unexpected non-empty but apparently blank line. */ + DEBUGP (("Null token. Leaving listing parser.\n")); + break; + } + } + } + + /* Analyze tokens. (Order is not significant, except date must + precede time.) + + Size: ddd or ddd/ddd (where "ddd" is a decimal number) + Date: DD-MMM-YYYY + Time: HH:MM or HH:MM:SS or HH:MM:SS.CC + Owner: [user] or [user,group] + Protection: (ppp,ppp,ppp,ppp) (where "ppp" is "RWED" or some + subset thereof, for System, Owner, Group, World. + + If permission is lacking, info may be replaced by the string: + "No privilege for attempted operation". + */ + while (tok != NULL) + { + DEBUGP (("Token: >%s<: ", tok)); + + if ((strlen (tok) < 12) && (strchr( tok, '-') != NULL)) + { + /* Date. */ + DEBUGP (("Date.\n")); + snprintf(date_str, sizeof(date_str), "%s ", tok); + } + else if ((strlen (tok) < 12) && (strchr( tok, ':') != NULL)) + { + /* Time. */ + DEBUGP (("Time. ")); + strncat( date_str, + tok, + (sizeof( date_str)- strlen (date_str) - 1)); + DEBUGP (("Date time: >%s<\n", date_str)); + } + else if (strchr (tok, '[') != NULL) + { + /* Owner. (Ignore.) */ + DEBUGP (("Owner.\n")); + } + else if (strchr (tok, '(') != NULL) + { + /* Protections (permissions). */ + perms = 0; + j = 0; + /*FIXME: Should not be using the variable like this. */ + for (i = 0; i < (int) strlen(tok); i++) + { + switch (tok[ i]) + { + case '(': + break; + case ')': + break; + case ',': + if (j == 0) + { + perms = 0; + } + else if (j < 4) + { + perms <<= 3; + } + j++; + break; + case 'R': + perms |= 4; + break; + case 'W': + perms |= 2; + break; + case 'E': + perms |= 1; + break; + case 'D': + perms |= 2; + break; + } + } + cur.perms = perms; + DEBUGP (("Prot. perms = %0o.\n", (unsigned) cur.perms)); + } + else + { + /* Nondescript. Probably size(s), probably in blocks. + Could be "No privilege ..." message. (Ignore.) + */ + DEBUGP (("Ignored (size?).\n")); + } + + tok = strtok (NULL, " "); + } + + /* Tokens exhausted. Interpret the data, and fill in the + structure. + */ + /* Fill tm timestruct according to date-time string. Fractional + seconds are ignored. Default to current time, if conversion + fails. + */ + timenow = time( NULL); + timestruct = localtime( &timenow ); + strptime( date_str, "%d-%b-%Y %H:%M:%S", timestruct); + + /* Convert struct tm local time to time_t local time. */ + timenow = mktime (timestruct); + /* Offset local time according to environment variable (seconds). */ + if ((tok = getenv ( "WGET_TIMEZONE_DIFFERENTIAL")) != NULL) + { + dt = atoi (tok); + DEBUGP (("Time differential = %d.\n", dt)); + } + else + dt = 0; + + if (dt >= 0) + timenow += dt; + else + timenow -= (-dt); + + cur.tstamp = timenow; /* Store the time-stamp. */ + DEBUGP (("Timestamp: %ld\n", cur.tstamp)); + cur.ptype = TT_HOUR_MIN; + + /* Add the data for this item to the linked list, */ + if (!dir) + { + l = dir = (struct fileinfo *)xmalloc (sizeof (struct fileinfo)); + cur.prev = cur.next = NULL; + memcpy (l, &cur, sizeof (cur)); + } + else + { + cur.prev = l; + cur.next = NULL; + l->next = (struct fileinfo *)xmalloc (sizeof (struct fileinfo)); + l = l->next; + memcpy (l, &cur, sizeof (cur)); + } + cur.name = NULL; + + i = getline (&line, &bufsize, fp); + if (i > 0) + { + i = clean_line (line, i); + if (i <= 0) + { + /* Blank line. End of significant file listing. */ + break; + } + } + } + + xfree (cur.name); + xfree (line); + return dir; +} + + +/* This function switches between the correct parsing routine depending on + the SYSTEM_TYPE. The system type should be based on the result of the + "SYST" response of the FTP server. According to this response we will + use on of the three different listing parsers that cover the most of FTP + servers used nowadays. */ + +struct fileinfo * +ftp_parse_ls (const char *file, const enum stype system_type) +{ + FILE *fp; + struct fileinfo *fi; + + fp = fopen (file, "rb"); + if (!fp) + { + logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno)); + return NULL; + } + + fi = ftp_parse_ls_fp (fp, system_type); + fclose(fp); + + return fi; +} + +struct fileinfo * +ftp_parse_ls_fp (FILE *fp, const enum stype system_type) +{ + switch (system_type) + { + case ST_UNIX: + return ftp_parse_unix_ls (fp, 0); + case ST_WINNT: + { + /* Detect whether the listing is simulating the UNIX format */ + int c = fgetc(fp); + rewind(fp); + + /* If the first character of the file is '0'-'9', it's WINNT + format. */ + if (c >= '0' && c <='9') + return ftp_parse_winnt_ls (fp); + else + return ftp_parse_unix_ls (fp, 1); + } + case ST_VMS: + return ftp_parse_vms_ls (fp); + case ST_MACOS: + return ftp_parse_unix_ls (fp, 1); + default: + logprintf (LOG_NOTQUIET, _("\ +Unsupported listing type, trying Unix listing parser.\n")); + return ftp_parse_unix_ls (fp, 0); + } +} + +/* Stuff for creating FTP index. */ + +/* The function creates an HTML index containing references to given + directories and files on the appropriate host. The references are + FTP. */ +uerr_t +ftp_index (const char *file, struct url *u, struct fileinfo *f) +{ + FILE *fp; + char *upwd; + char *htcldir; /* HTML-clean dir name */ + char *htclfile; /* HTML-clean file name */ + char *urlclfile; /* URL-clean file name */ + + if (!output_stream) + { + fp = fopen (file, "wb"); + if (!fp) + { + logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno)); + return FOPENERR; + } + } + else + fp = output_stream; + if (u->user) + { + char *tmpu, *tmpp; /* temporary, clean user and passwd */ + + tmpu = url_escape (u->user); + tmpp = u->passwd ? url_escape (u->passwd) : NULL; + if (tmpp) + upwd = concat_strings (tmpu, ":", tmpp, "@", (char *) 0); + else + upwd = concat_strings (tmpu, "@", (char *) 0); + xfree (tmpu); + xfree (tmpp); + } + else + upwd = xstrdup (""); + + htcldir = html_quote_string (u->dir); + + fprintf (fp, "<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML 2.0//EN\">\n"); + fprintf (fp, "<html>\n<head>\n<title>"); + fprintf (fp, _("Index of /%s on %s:%d"), htcldir, u->host, u->port); + fprintf (fp, "</title>\n</head>\n<body>\n<h1>"); + fprintf (fp, _("Index of /%s on %s:%d"), htcldir, u->host, u->port); + fprintf (fp, "</h1>\n<hr>\n<pre>\n"); + + while (f) + { + fprintf (fp, " "); + if (f->tstamp != -1) + { + /* #### Should we translate the months? Or, even better, use + ISO 8601 dates? */ + static const char *months[] = { + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" + }; + time_t tstamp = f->tstamp; + struct tm *ptm = localtime (&tstamp); + + fprintf (fp, "%d %s %02d ", ptm->tm_year + 1900, months[ptm->tm_mon], + ptm->tm_mday); + if (f->ptype == TT_HOUR_MIN) + fprintf (fp, "%02d:%02d ", ptm->tm_hour, ptm->tm_min); + else + fprintf (fp, " "); + } + else + fprintf (fp, _("time unknown ")); + switch (f->type) + { + case FT_PLAINFILE: + fprintf (fp, _("File ")); + break; + case FT_DIRECTORY: + fprintf (fp, _("Directory ")); + break; + case FT_SYMLINK: + fprintf (fp, _("Link ")); + break; + default: + fprintf (fp, _("Not sure ")); + break; + } + htclfile = html_quote_string (f->name); + urlclfile = url_escape_unsafe_and_reserved (f->name); + fprintf (fp, "<a href=\"ftp://%s%s:%d", upwd, u->host, u->port); + if (*u->dir != '/') + putc ('/', fp); + /* XXX: Should probably URL-escape dir components here, rather + * than just HTML-escape, for consistency with the next bit where + * we use urlclfile for the file component. Anyway, this is safer + * than what we had... */ + fprintf (fp, "%s", htcldir); + if (*u->dir) + putc ('/', fp); + fprintf (fp, "%s", urlclfile); + if (f->type == FT_DIRECTORY) + putc ('/', fp); + fprintf (fp, "\">%s", htclfile); + if (f->type == FT_DIRECTORY) + putc ('/', fp); + fprintf (fp, "</a> "); + if (f->type == FT_PLAINFILE) + fprintf (fp, _(" (%s bytes)"), number_to_static_string (f->size)); + else if (f->type == FT_SYMLINK) + fprintf (fp, "-> %s", f->linkto ? f->linkto : "(nil)"); + putc ('\n', fp); + xfree (htclfile); + xfree (urlclfile); + f = f->next; + } + fprintf (fp, "</pre>\n</body>\n</html>\n"); + xfree (htcldir); + xfree (upwd); + if (!output_stream) + fclose (fp); + else + fflush (fp); + return FTPOK; +} diff --git a/src/ftp-opie.c b/src/ftp-opie.c new file mode 100644 index 0000000..abe174b --- /dev/null +++ b/src/ftp-opie.c @@ -0,0 +1,2220 @@ +/* Opie (s/key) support for FTP. + Copyright (C) 1998-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "md5.h" +#include "ftp.h" + +/* Dictionary for integer-word translations. Available in appendix D + of rfc2289. */ + static char Wp[2048][4] = { + { 'A', '\0', '\0', '\0' }, + { 'A', 'B', 'E', '\0' }, + { 'A', 'C', 'E', '\0' }, + { 'A', 'C', 'T', '\0' }, + { 'A', 'D', '\0', '\0' }, + { 'A', 'D', 'A', '\0' }, + { 'A', 'D', 'D', '\0' }, + { 'A', 'G', 'O', '\0' }, + { 'A', 'I', 'D', '\0' }, + { 'A', 'I', 'M', '\0' }, + { 'A', 'I', 'R', '\0' }, + { 'A', 'L', 'L', '\0' }, + { 'A', 'L', 'P', '\0' }, + { 'A', 'M', '\0', '\0' }, + { 'A', 'M', 'Y', '\0' }, + { 'A', 'N', '\0', '\0' }, + { 'A', 'N', 'A', '\0' }, + { 'A', 'N', 'D', '\0' }, + { 'A', 'N', 'N', '\0' }, + { 'A', 'N', 'T', '\0' }, + { 'A', 'N', 'Y', '\0' }, + { 'A', 'P', 'E', '\0' }, + { 'A', 'P', 'S', '\0' }, + { 'A', 'P', 'T', '\0' }, + { 'A', 'R', 'C', '\0' }, + { 'A', 'R', 'E', '\0' }, + { 'A', 'R', 'K', '\0' }, + { 'A', 'R', 'M', '\0' }, + { 'A', 'R', 'T', '\0' }, + { 'A', 'S', '\0', '\0' }, + { 'A', 'S', 'H', '\0' }, + { 'A', 'S', 'K', '\0' }, + { 'A', 'T', '\0', '\0' }, + { 'A', 'T', 'E', '\0' }, + { 'A', 'U', 'G', '\0' }, + { 'A', 'U', 'K', '\0' }, + { 'A', 'V', 'E', '\0' }, + { 'A', 'W', 'E', '\0' }, + { 'A', 'W', 'K', '\0' }, + { 'A', 'W', 'L', '\0' }, + { 'A', 'W', 'N', '\0' }, + { 'A', 'X', '\0', '\0' }, + { 'A', 'Y', 'E', '\0' }, + { 'B', 'A', 'D', '\0' }, + { 'B', 'A', 'G', '\0' }, + { 'B', 'A', 'H', '\0' }, + { 'B', 'A', 'M', '\0' }, + { 'B', 'A', 'N', '\0' }, + { 'B', 'A', 'R', '\0' }, + { 'B', 'A', 'T', '\0' }, + { 'B', 'A', 'Y', '\0' }, + { 'B', 'E', '\0', '\0' }, + { 'B', 'E', 'D', '\0' }, + { 'B', 'E', 'E', '\0' }, + { 'B', 'E', 'G', '\0' }, + { 'B', 'E', 'N', '\0' }, + { 'B', 'E', 'T', '\0' }, + { 'B', 'E', 'Y', '\0' }, + { 'B', 'I', 'B', '\0' }, + { 'B', 'I', 'D', '\0' }, + { 'B', 'I', 'G', '\0' }, + { 'B', 'I', 'N', '\0' }, + { 'B', 'I', 'T', '\0' }, + { 'B', 'O', 'B', '\0' }, + { 'B', 'O', 'G', '\0' }, + { 'B', 'O', 'N', '\0' }, + { 'B', 'O', 'O', '\0' }, + { 'B', 'O', 'P', '\0' }, + { 'B', 'O', 'W', '\0' }, + { 'B', 'O', 'Y', '\0' }, + { 'B', 'U', 'B', '\0' }, + { 'B', 'U', 'D', '\0' }, + { 'B', 'U', 'G', '\0' }, + { 'B', 'U', 'M', '\0' }, + { 'B', 'U', 'N', '\0' }, + { 'B', 'U', 'S', '\0' }, + { 'B', 'U', 'T', '\0' }, + { 'B', 'U', 'Y', '\0' }, + { 'B', 'Y', '\0', '\0' }, + { 'B', 'Y', 'E', '\0' }, + { 'C', 'A', 'B', '\0' }, + { 'C', 'A', 'L', '\0' }, + { 'C', 'A', 'M', '\0' }, + { 'C', 'A', 'N', '\0' }, + { 'C', 'A', 'P', '\0' }, + { 'C', 'A', 'R', '\0' }, + { 'C', 'A', 'T', '\0' }, + { 'C', 'A', 'W', '\0' }, + { 'C', 'O', 'D', '\0' }, + { 'C', 'O', 'G', '\0' }, + { 'C', 'O', 'L', '\0' }, + { 'C', 'O', 'N', '\0' }, + { 'C', 'O', 'O', '\0' }, + { 'C', 'O', 'P', '\0' }, + { 'C', 'O', 'T', '\0' }, + { 'C', 'O', 'W', '\0' }, + { 'C', 'O', 'Y', '\0' }, + { 'C', 'R', 'Y', '\0' }, + { 'C', 'U', 'B', '\0' }, + { 'C', 'U', 'E', '\0' }, + { 'C', 'U', 'P', '\0' }, + { 'C', 'U', 'R', '\0' }, + { 'C', 'U', 'T', '\0' }, + { 'D', 'A', 'B', '\0' }, + { 'D', 'A', 'D', '\0' }, + { 'D', 'A', 'M', '\0' }, + { 'D', 'A', 'N', '\0' }, + { 'D', 'A', 'R', '\0' }, + { 'D', 'A', 'Y', '\0' }, + { 'D', 'E', 'E', '\0' }, + { 'D', 'E', 'L', '\0' }, + { 'D', 'E', 'N', '\0' }, + { 'D', 'E', 'S', '\0' }, + { 'D', 'E', 'W', '\0' }, + { 'D', 'I', 'D', '\0' }, + { 'D', 'I', 'E', '\0' }, + { 'D', 'I', 'G', '\0' }, + { 'D', 'I', 'N', '\0' }, + { 'D', 'I', 'P', '\0' }, + { 'D', 'O', '\0', '\0' }, + { 'D', 'O', 'E', '\0' }, + { 'D', 'O', 'G', '\0' }, + { 'D', 'O', 'N', '\0' }, + { 'D', 'O', 'T', '\0' }, + { 'D', 'O', 'W', '\0' }, + { 'D', 'R', 'Y', '\0' }, + { 'D', 'U', 'B', '\0' }, + { 'D', 'U', 'D', '\0' }, + { 'D', 'U', 'E', '\0' }, + { 'D', 'U', 'G', '\0' }, + { 'D', 'U', 'N', '\0' }, + { 'E', 'A', 'R', '\0' }, + { 'E', 'A', 'T', '\0' }, + { 'E', 'D', '\0', '\0' }, + { 'E', 'E', 'L', '\0' }, + { 'E', 'G', 'G', '\0' }, + { 'E', 'G', 'O', '\0' }, + { 'E', 'L', 'I', '\0' }, + { 'E', 'L', 'K', '\0' }, + { 'E', 'L', 'M', '\0' }, + { 'E', 'L', 'Y', '\0' }, + { 'E', 'M', '\0', '\0' }, + { 'E', 'N', 'D', '\0' }, + { 'E', 'S', 'T', '\0' }, + { 'E', 'T', 'C', '\0' }, + { 'E', 'V', 'A', '\0' }, + { 'E', 'V', 'E', '\0' }, + { 'E', 'W', 'E', '\0' }, + { 'E', 'Y', 'E', '\0' }, + { 'F', 'A', 'D', '\0' }, + { 'F', 'A', 'N', '\0' }, + { 'F', 'A', 'R', '\0' }, + { 'F', 'A', 'T', '\0' }, + { 'F', 'A', 'Y', '\0' }, + { 'F', 'E', 'D', '\0' }, + { 'F', 'E', 'E', '\0' }, + { 'F', 'E', 'W', '\0' }, + { 'F', 'I', 'B', '\0' }, + { 'F', 'I', 'G', '\0' }, + { 'F', 'I', 'N', '\0' }, + { 'F', 'I', 'R', '\0' }, + { 'F', 'I', 'T', '\0' }, + { 'F', 'L', 'O', '\0' }, + { 'F', 'L', 'Y', '\0' }, + { 'F', 'O', 'E', '\0' }, + { 'F', 'O', 'G', '\0' }, + { 'F', 'O', 'R', '\0' }, + { 'F', 'R', 'Y', '\0' }, + { 'F', 'U', 'M', '\0' }, + { 'F', 'U', 'N', '\0' }, + { 'F', 'U', 'R', '\0' }, + { 'G', 'A', 'B', '\0' }, + { 'G', 'A', 'D', '\0' }, + { 'G', 'A', 'G', '\0' }, + { 'G', 'A', 'L', '\0' }, + { 'G', 'A', 'M', '\0' }, + { 'G', 'A', 'P', '\0' }, + { 'G', 'A', 'S', '\0' }, + { 'G', 'A', 'Y', '\0' }, + { 'G', 'E', 'E', '\0' }, + { 'G', 'E', 'L', '\0' }, + { 'G', 'E', 'M', '\0' }, + { 'G', 'E', 'T', '\0' }, + { 'G', 'I', 'G', '\0' }, + { 'G', 'I', 'L', '\0' }, + { 'G', 'I', 'N', '\0' }, + { 'G', 'O', '\0', '\0' }, + { 'G', 'O', 'T', '\0' }, + { 'G', 'U', 'M', '\0' }, + { 'G', 'U', 'N', '\0' }, + { 'G', 'U', 'S', '\0' }, + { 'G', 'U', 'T', '\0' }, + { 'G', 'U', 'Y', '\0' }, + { 'G', 'Y', 'M', '\0' }, + { 'G', 'Y', 'P', '\0' }, + { 'H', 'A', '\0', '\0' }, + { 'H', 'A', 'D', '\0' }, + { 'H', 'A', 'L', '\0' }, + { 'H', 'A', 'M', '\0' }, + { 'H', 'A', 'N', '\0' }, + { 'H', 'A', 'P', '\0' }, + { 'H', 'A', 'S', '\0' }, + { 'H', 'A', 'T', '\0' }, + { 'H', 'A', 'W', '\0' }, + { 'H', 'A', 'Y', '\0' }, + { 'H', 'E', '\0', '\0' }, + { 'H', 'E', 'M', '\0' }, + { 'H', 'E', 'N', '\0' }, + { 'H', 'E', 'R', '\0' }, + { 'H', 'E', 'W', '\0' }, + { 'H', 'E', 'Y', '\0' }, + { 'H', 'I', '\0', '\0' }, + { 'H', 'I', 'D', '\0' }, + { 'H', 'I', 'M', '\0' }, + { 'H', 'I', 'P', '\0' }, + { 'H', 'I', 'S', '\0' }, + { 'H', 'I', 'T', '\0' }, + { 'H', 'O', '\0', '\0' }, + { 'H', 'O', 'B', '\0' }, + { 'H', 'O', 'C', '\0' }, + { 'H', 'O', 'E', '\0' }, + { 'H', 'O', 'G', '\0' }, + { 'H', 'O', 'P', '\0' }, + { 'H', 'O', 'T', '\0' }, + { 'H', 'O', 'W', '\0' }, + { 'H', 'U', 'B', '\0' }, + { 'H', 'U', 'E', '\0' }, + { 'H', 'U', 'G', '\0' }, + { 'H', 'U', 'H', '\0' }, + { 'H', 'U', 'M', '\0' }, + { 'H', 'U', 'T', '\0' }, + { 'I', '\0', '\0', '\0' }, + { 'I', 'C', 'Y', '\0' }, + { 'I', 'D', 'A', '\0' }, + { 'I', 'F', '\0', '\0' }, + { 'I', 'K', 'E', '\0' }, + { 'I', 'L', 'L', '\0' }, + { 'I', 'N', 'K', '\0' }, + { 'I', 'N', 'N', '\0' }, + { 'I', 'O', '\0', '\0' }, + { 'I', 'O', 'N', '\0' }, + { 'I', 'Q', '\0', '\0' }, + { 'I', 'R', 'A', '\0' }, + { 'I', 'R', 'E', '\0' }, + { 'I', 'R', 'K', '\0' }, + { 'I', 'S', '\0', '\0' }, + { 'I', 'T', '\0', '\0' }, + { 'I', 'T', 'S', '\0' }, + { 'I', 'V', 'Y', '\0' }, + { 'J', 'A', 'B', '\0' }, + { 'J', 'A', 'G', '\0' }, + { 'J', 'A', 'M', '\0' }, + { 'J', 'A', 'N', '\0' }, + { 'J', 'A', 'R', '\0' }, + { 'J', 'A', 'W', '\0' }, + { 'J', 'A', 'Y', '\0' }, + { 'J', 'E', 'T', '\0' }, + { 'J', 'I', 'G', '\0' }, + { 'J', 'I', 'M', '\0' }, + { 'J', 'O', '\0', '\0' }, + { 'J', 'O', 'B', '\0' }, + { 'J', 'O', 'E', '\0' }, + { 'J', 'O', 'G', '\0' }, + { 'J', 'O', 'T', '\0' }, + { 'J', 'O', 'Y', '\0' }, + { 'J', 'U', 'G', '\0' }, + { 'J', 'U', 'T', '\0' }, + { 'K', 'A', 'Y', '\0' }, + { 'K', 'E', 'G', '\0' }, + { 'K', 'E', 'N', '\0' }, + { 'K', 'E', 'Y', '\0' }, + { 'K', 'I', 'D', '\0' }, + { 'K', 'I', 'M', '\0' }, + { 'K', 'I', 'N', '\0' }, + { 'K', 'I', 'T', '\0' }, + { 'L', 'A', '\0', '\0' }, + { 'L', 'A', 'B', '\0' }, + { 'L', 'A', 'C', '\0' }, + { 'L', 'A', 'D', '\0' }, + { 'L', 'A', 'G', '\0' }, + { 'L', 'A', 'M', '\0' }, + { 'L', 'A', 'P', '\0' }, + { 'L', 'A', 'W', '\0' }, + { 'L', 'A', 'Y', '\0' }, + { 'L', 'E', 'A', '\0' }, + { 'L', 'E', 'D', '\0' }, + { 'L', 'E', 'E', '\0' }, + { 'L', 'E', 'G', '\0' }, + { 'L', 'E', 'N', '\0' }, + { 'L', 'E', 'O', '\0' }, + { 'L', 'E', 'T', '\0' }, + { 'L', 'E', 'W', '\0' }, + { 'L', 'I', 'D', '\0' }, + { 'L', 'I', 'E', '\0' }, + { 'L', 'I', 'N', '\0' }, + { 'L', 'I', 'P', '\0' }, + { 'L', 'I', 'T', '\0' }, + { 'L', 'O', '\0', '\0' }, + { 'L', 'O', 'B', '\0' }, + { 'L', 'O', 'G', '\0' }, + { 'L', 'O', 'P', '\0' }, + { 'L', 'O', 'S', '\0' }, + { 'L', 'O', 'T', '\0' }, + { 'L', 'O', 'U', '\0' }, + { 'L', 'O', 'W', '\0' }, + { 'L', 'O', 'Y', '\0' }, + { 'L', 'U', 'G', '\0' }, + { 'L', 'Y', 'E', '\0' }, + { 'M', 'A', '\0', '\0' }, + { 'M', 'A', 'C', '\0' }, + { 'M', 'A', 'D', '\0' }, + { 'M', 'A', 'E', '\0' }, + { 'M', 'A', 'N', '\0' }, + { 'M', 'A', 'O', '\0' }, + { 'M', 'A', 'P', '\0' }, + { 'M', 'A', 'T', '\0' }, + { 'M', 'A', 'W', '\0' }, + { 'M', 'A', 'Y', '\0' }, + { 'M', 'E', '\0', '\0' }, + { 'M', 'E', 'G', '\0' }, + { 'M', 'E', 'L', '\0' }, + { 'M', 'E', 'N', '\0' }, + { 'M', 'E', 'T', '\0' }, + { 'M', 'E', 'W', '\0' }, + { 'M', 'I', 'D', '\0' }, + { 'M', 'I', 'N', '\0' }, + { 'M', 'I', 'T', '\0' }, + { 'M', 'O', 'B', '\0' }, + { 'M', 'O', 'D', '\0' }, + { 'M', 'O', 'E', '\0' }, + { 'M', 'O', 'O', '\0' }, + { 'M', 'O', 'P', '\0' }, + { 'M', 'O', 'S', '\0' }, + { 'M', 'O', 'T', '\0' }, + { 'M', 'O', 'W', '\0' }, + { 'M', 'U', 'D', '\0' }, + { 'M', 'U', 'G', '\0' }, + { 'M', 'U', 'M', '\0' }, + { 'M', 'Y', '\0', '\0' }, + { 'N', 'A', 'B', '\0' }, + { 'N', 'A', 'G', '\0' }, + { 'N', 'A', 'N', '\0' }, + { 'N', 'A', 'P', '\0' }, + { 'N', 'A', 'T', '\0' }, + { 'N', 'A', 'Y', '\0' }, + { 'N', 'E', '\0', '\0' }, + { 'N', 'E', 'D', '\0' }, + { 'N', 'E', 'E', '\0' }, + { 'N', 'E', 'T', '\0' }, + { 'N', 'E', 'W', '\0' }, + { 'N', 'I', 'B', '\0' }, + { 'N', 'I', 'L', '\0' }, + { 'N', 'I', 'P', '\0' }, + { 'N', 'I', 'T', '\0' }, + { 'N', 'O', '\0', '\0' }, + { 'N', 'O', 'B', '\0' }, + { 'N', 'O', 'D', '\0' }, + { 'N', 'O', 'N', '\0' }, + { 'N', 'O', 'R', '\0' }, + { 'N', 'O', 'T', '\0' }, + { 'N', 'O', 'V', '\0' }, + { 'N', 'O', 'W', '\0' }, + { 'N', 'U', '\0', '\0' }, + { 'N', 'U', 'N', '\0' }, + { 'N', 'U', 'T', '\0' }, + { 'O', '\0', '\0', '\0' }, + { 'O', 'A', 'F', '\0' }, + { 'O', 'A', 'K', '\0' }, + { 'O', 'A', 'R', '\0' }, + { 'O', 'A', 'T', '\0' }, + { 'O', 'D', 'D', '\0' }, + { 'O', 'D', 'E', '\0' }, + { 'O', 'F', '\0', '\0' }, + { 'O', 'F', 'F', '\0' }, + { 'O', 'F', 'T', '\0' }, + { 'O', 'H', '\0', '\0' }, + { 'O', 'I', 'L', '\0' }, + { 'O', 'K', '\0', '\0' }, + { 'O', 'L', 'D', '\0' }, + { 'O', 'N', '\0', '\0' }, + { 'O', 'N', 'E', '\0' }, + { 'O', 'R', '\0', '\0' }, + { 'O', 'R', 'B', '\0' }, + { 'O', 'R', 'E', '\0' }, + { 'O', 'R', 'R', '\0' }, + { 'O', 'S', '\0', '\0' }, + { 'O', 'T', 'T', '\0' }, + { 'O', 'U', 'R', '\0' }, + { 'O', 'U', 'T', '\0' }, + { 'O', 'V', 'A', '\0' }, + { 'O', 'W', '\0', '\0' }, + { 'O', 'W', 'E', '\0' }, + { 'O', 'W', 'L', '\0' }, + { 'O', 'W', 'N', '\0' }, + { 'O', 'X', '\0', '\0' }, + { 'P', 'A', '\0', '\0' }, + { 'P', 'A', 'D', '\0' }, + { 'P', 'A', 'L', '\0' }, + { 'P', 'A', 'M', '\0' }, + { 'P', 'A', 'N', '\0' }, + { 'P', 'A', 'P', '\0' }, + { 'P', 'A', 'R', '\0' }, + { 'P', 'A', 'T', '\0' }, + { 'P', 'A', 'W', '\0' }, + { 'P', 'A', 'Y', '\0' }, + { 'P', 'E', 'A', '\0' }, + { 'P', 'E', 'G', '\0' }, + { 'P', 'E', 'N', '\0' }, + { 'P', 'E', 'P', '\0' }, + { 'P', 'E', 'R', '\0' }, + { 'P', 'E', 'T', '\0' }, + { 'P', 'E', 'W', '\0' }, + { 'P', 'H', 'I', '\0' }, + { 'P', 'I', '\0', '\0' }, + { 'P', 'I', 'E', '\0' }, + { 'P', 'I', 'N', '\0' }, + { 'P', 'I', 'T', '\0' }, + { 'P', 'L', 'Y', '\0' }, + { 'P', 'O', '\0', '\0' }, + { 'P', 'O', 'D', '\0' }, + { 'P', 'O', 'E', '\0' }, + { 'P', 'O', 'P', '\0' }, + { 'P', 'O', 'T', '\0' }, + { 'P', 'O', 'W', '\0' }, + { 'P', 'R', 'O', '\0' }, + { 'P', 'R', 'Y', '\0' }, + { 'P', 'U', 'B', '\0' }, + { 'P', 'U', 'G', '\0' }, + { 'P', 'U', 'N', '\0' }, + { 'P', 'U', 'P', '\0' }, + { 'P', 'U', 'T', '\0' }, + { 'Q', 'U', 'O', '\0' }, + { 'R', 'A', 'G', '\0' }, + { 'R', 'A', 'M', '\0' }, + { 'R', 'A', 'N', '\0' }, + { 'R', 'A', 'P', '\0' }, + { 'R', 'A', 'T', '\0' }, + { 'R', 'A', 'W', '\0' }, + { 'R', 'A', 'Y', '\0' }, + { 'R', 'E', 'B', '\0' }, + { 'R', 'E', 'D', '\0' }, + { 'R', 'E', 'P', '\0' }, + { 'R', 'E', 'T', '\0' }, + { 'R', 'I', 'B', '\0' }, + { 'R', 'I', 'D', '\0' }, + { 'R', 'I', 'G', '\0' }, + { 'R', 'I', 'M', '\0' }, + { 'R', 'I', 'O', '\0' }, + { 'R', 'I', 'P', '\0' }, + { 'R', 'O', 'B', '\0' }, + { 'R', 'O', 'D', '\0' }, + { 'R', 'O', 'E', '\0' }, + { 'R', 'O', 'N', '\0' }, + { 'R', 'O', 'T', '\0' }, + { 'R', 'O', 'W', '\0' }, + { 'R', 'O', 'Y', '\0' }, + { 'R', 'U', 'B', '\0' }, + { 'R', 'U', 'E', '\0' }, + { 'R', 'U', 'G', '\0' }, + { 'R', 'U', 'M', '\0' }, + { 'R', 'U', 'N', '\0' }, + { 'R', 'Y', 'E', '\0' }, + { 'S', 'A', 'C', '\0' }, + { 'S', 'A', 'D', '\0' }, + { 'S', 'A', 'G', '\0' }, + { 'S', 'A', 'L', '\0' }, + { 'S', 'A', 'M', '\0' }, + { 'S', 'A', 'N', '\0' }, + { 'S', 'A', 'P', '\0' }, + { 'S', 'A', 'T', '\0' }, + { 'S', 'A', 'W', '\0' }, + { 'S', 'A', 'Y', '\0' }, + { 'S', 'E', 'A', '\0' }, + { 'S', 'E', 'C', '\0' }, + { 'S', 'E', 'E', '\0' }, + { 'S', 'E', 'N', '\0' }, + { 'S', 'E', 'T', '\0' }, + { 'S', 'E', 'W', '\0' }, + { 'S', 'H', 'E', '\0' }, + { 'S', 'H', 'Y', '\0' }, + { 'S', 'I', 'N', '\0' }, + { 'S', 'I', 'P', '\0' }, + { 'S', 'I', 'R', '\0' }, + { 'S', 'I', 'S', '\0' }, + { 'S', 'I', 'T', '\0' }, + { 'S', 'K', 'I', '\0' }, + { 'S', 'K', 'Y', '\0' }, + { 'S', 'L', 'Y', '\0' }, + { 'S', 'O', '\0', '\0' }, + { 'S', 'O', 'B', '\0' }, + { 'S', 'O', 'D', '\0' }, + { 'S', 'O', 'N', '\0' }, + { 'S', 'O', 'P', '\0' }, + { 'S', 'O', 'W', '\0' }, + { 'S', 'O', 'Y', '\0' }, + { 'S', 'P', 'A', '\0' }, + { 'S', 'P', 'Y', '\0' }, + { 'S', 'U', 'B', '\0' }, + { 'S', 'U', 'D', '\0' }, + { 'S', 'U', 'E', '\0' }, + { 'S', 'U', 'M', '\0' }, + { 'S', 'U', 'N', '\0' }, + { 'S', 'U', 'P', '\0' }, + { 'T', 'A', 'B', '\0' }, + { 'T', 'A', 'D', '\0' }, + { 'T', 'A', 'G', '\0' }, + { 'T', 'A', 'N', '\0' }, + { 'T', 'A', 'P', '\0' }, + { 'T', 'A', 'R', '\0' }, + { 'T', 'E', 'A', '\0' }, + { 'T', 'E', 'D', '\0' }, + { 'T', 'E', 'E', '\0' }, + { 'T', 'E', 'N', '\0' }, + { 'T', 'H', 'E', '\0' }, + { 'T', 'H', 'Y', '\0' }, + { 'T', 'I', 'C', '\0' }, + { 'T', 'I', 'E', '\0' }, + { 'T', 'I', 'M', '\0' }, + { 'T', 'I', 'N', '\0' }, + { 'T', 'I', 'P', '\0' }, + { 'T', 'O', '\0', '\0' }, + { 'T', 'O', 'E', '\0' }, + { 'T', 'O', 'G', '\0' }, + { 'T', 'O', 'M', '\0' }, + { 'T', 'O', 'N', '\0' }, + { 'T', 'O', 'O', '\0' }, + { 'T', 'O', 'P', '\0' }, + { 'T', 'O', 'W', '\0' }, + { 'T', 'O', 'Y', '\0' }, + { 'T', 'R', 'Y', '\0' }, + { 'T', 'U', 'B', '\0' }, + { 'T', 'U', 'G', '\0' }, + { 'T', 'U', 'M', '\0' }, + { 'T', 'U', 'N', '\0' }, + { 'T', 'W', 'O', '\0' }, + { 'U', 'N', '\0', '\0' }, + { 'U', 'P', '\0', '\0' }, + { 'U', 'S', '\0', '\0' }, + { 'U', 'S', 'E', '\0' }, + { 'V', 'A', 'N', '\0' }, + { 'V', 'A', 'T', '\0' }, + { 'V', 'E', 'T', '\0' }, + { 'V', 'I', 'E', '\0' }, + { 'W', 'A', 'D', '\0' }, + { 'W', 'A', 'G', '\0' }, + { 'W', 'A', 'R', '\0' }, + { 'W', 'A', 'S', '\0' }, + { 'W', 'A', 'Y', '\0' }, + { 'W', 'E', '\0', '\0' }, + { 'W', 'E', 'B', '\0' }, + { 'W', 'E', 'D', '\0' }, + { 'W', 'E', 'E', '\0' }, + { 'W', 'E', 'T', '\0' }, + { 'W', 'H', 'O', '\0' }, + { 'W', 'H', 'Y', '\0' }, + { 'W', 'I', 'N', '\0' }, + { 'W', 'I', 'T', '\0' }, + { 'W', 'O', 'K', '\0' }, + { 'W', 'O', 'N', '\0' }, + { 'W', 'O', 'O', '\0' }, + { 'W', 'O', 'W', '\0' }, + { 'W', 'R', 'Y', '\0' }, + { 'W', 'U', '\0', '\0' }, + { 'Y', 'A', 'M', '\0' }, + { 'Y', 'A', 'P', '\0' }, + { 'Y', 'A', 'W', '\0' }, + { 'Y', 'E', '\0', '\0' }, + { 'Y', 'E', 'A', '\0' }, + { 'Y', 'E', 'S', '\0' }, + { 'Y', 'E', 'T', '\0' }, + { 'Y', 'O', 'U', '\0' }, + { 'A', 'B', 'E', 'D' }, + { 'A', 'B', 'E', 'L' }, + { 'A', 'B', 'E', 'T' }, + { 'A', 'B', 'L', 'E' }, + { 'A', 'B', 'U', 'T' }, + { 'A', 'C', 'H', 'E' }, + { 'A', 'C', 'I', 'D' }, + { 'A', 'C', 'M', 'E' }, + { 'A', 'C', 'R', 'E' }, + { 'A', 'C', 'T', 'A' }, + { 'A', 'C', 'T', 'S' }, + { 'A', 'D', 'A', 'M' }, + { 'A', 'D', 'D', 'S' }, + { 'A', 'D', 'E', 'N' }, + { 'A', 'F', 'A', 'R' }, + { 'A', 'F', 'R', 'O' }, + { 'A', 'G', 'E', 'E' }, + { 'A', 'H', 'E', 'M' }, + { 'A', 'H', 'O', 'Y' }, + { 'A', 'I', 'D', 'A' }, + { 'A', 'I', 'D', 'E' }, + { 'A', 'I', 'D', 'S' }, + { 'A', 'I', 'R', 'Y' }, + { 'A', 'J', 'A', 'R' }, + { 'A', 'K', 'I', 'N' }, + { 'A', 'L', 'A', 'N' }, + { 'A', 'L', 'E', 'C' }, + { 'A', 'L', 'G', 'A' }, + { 'A', 'L', 'I', 'A' }, + { 'A', 'L', 'L', 'Y' }, + { 'A', 'L', 'M', 'A' }, + { 'A', 'L', 'O', 'E' }, + { 'A', 'L', 'S', 'O' }, + { 'A', 'L', 'T', 'O' }, + { 'A', 'L', 'U', 'M' }, + { 'A', 'L', 'V', 'A' }, + { 'A', 'M', 'E', 'N' }, + { 'A', 'M', 'E', 'S' }, + { 'A', 'M', 'I', 'D' }, + { 'A', 'M', 'M', 'O' }, + { 'A', 'M', 'O', 'K' }, + { 'A', 'M', 'O', 'S' }, + { 'A', 'M', 'R', 'A' }, + { 'A', 'N', 'D', 'Y' }, + { 'A', 'N', 'E', 'W' }, + { 'A', 'N', 'N', 'A' }, + { 'A', 'N', 'N', 'E' }, + { 'A', 'N', 'T', 'E' }, + { 'A', 'N', 'T', 'I' }, + { 'A', 'Q', 'U', 'A' }, + { 'A', 'R', 'A', 'B' }, + { 'A', 'R', 'C', 'H' }, + { 'A', 'R', 'E', 'A' }, + { 'A', 'R', 'G', 'O' }, + { 'A', 'R', 'I', 'D' }, + { 'A', 'R', 'M', 'Y' }, + { 'A', 'R', 'T', 'S' }, + { 'A', 'R', 'T', 'Y' }, + { 'A', 'S', 'I', 'A' }, + { 'A', 'S', 'K', 'S' }, + { 'A', 'T', 'O', 'M' }, + { 'A', 'U', 'N', 'T' }, + { 'A', 'U', 'R', 'A' }, + { 'A', 'U', 'T', 'O' }, + { 'A', 'V', 'E', 'R' }, + { 'A', 'V', 'I', 'D' }, + { 'A', 'V', 'I', 'S' }, + { 'A', 'V', 'O', 'N' }, + { 'A', 'V', 'O', 'W' }, + { 'A', 'W', 'A', 'Y' }, + { 'A', 'W', 'R', 'Y' }, + { 'B', 'A', 'B', 'E' }, + { 'B', 'A', 'B', 'Y' }, + { 'B', 'A', 'C', 'H' }, + { 'B', 'A', 'C', 'K' }, + { 'B', 'A', 'D', 'E' }, + { 'B', 'A', 'I', 'L' }, + { 'B', 'A', 'I', 'T' }, + { 'B', 'A', 'K', 'E' }, + { 'B', 'A', 'L', 'D' }, + { 'B', 'A', 'L', 'E' }, + { 'B', 'A', 'L', 'I' }, + { 'B', 'A', 'L', 'K' }, + { 'B', 'A', 'L', 'L' }, + { 'B', 'A', 'L', 'M' }, + { 'B', 'A', 'N', 'D' }, + { 'B', 'A', 'N', 'E' }, + { 'B', 'A', 'N', 'G' }, + { 'B', 'A', 'N', 'K' }, + { 'B', 'A', 'R', 'B' }, + { 'B', 'A', 'R', 'D' }, + { 'B', 'A', 'R', 'E' }, + { 'B', 'A', 'R', 'K' }, + { 'B', 'A', 'R', 'N' }, + { 'B', 'A', 'R', 'R' }, + { 'B', 'A', 'S', 'E' }, + { 'B', 'A', 'S', 'H' }, + { 'B', 'A', 'S', 'K' }, + { 'B', 'A', 'S', 'S' }, + { 'B', 'A', 'T', 'E' }, + { 'B', 'A', 'T', 'H' }, + { 'B', 'A', 'W', 'D' }, + { 'B', 'A', 'W', 'L' }, + { 'B', 'E', 'A', 'D' }, + { 'B', 'E', 'A', 'K' }, + { 'B', 'E', 'A', 'M' }, + { 'B', 'E', 'A', 'N' }, + { 'B', 'E', 'A', 'R' }, + { 'B', 'E', 'A', 'T' }, + { 'B', 'E', 'A', 'U' }, + { 'B', 'E', 'C', 'K' }, + { 'B', 'E', 'E', 'F' }, + { 'B', 'E', 'E', 'N' }, + { 'B', 'E', 'E', 'R' }, + { 'B', 'E', 'E', 'T' }, + { 'B', 'E', 'L', 'A' }, + { 'B', 'E', 'L', 'L' }, + { 'B', 'E', 'L', 'T' }, + { 'B', 'E', 'N', 'D' }, + { 'B', 'E', 'N', 'T' }, + { 'B', 'E', 'R', 'G' }, + { 'B', 'E', 'R', 'N' }, + { 'B', 'E', 'R', 'T' }, + { 'B', 'E', 'S', 'S' }, + { 'B', 'E', 'S', 'T' }, + { 'B', 'E', 'T', 'A' }, + { 'B', 'E', 'T', 'H' }, + { 'B', 'H', 'O', 'Y' }, + { 'B', 'I', 'A', 'S' }, + { 'B', 'I', 'D', 'E' }, + { 'B', 'I', 'E', 'N' }, + { 'B', 'I', 'L', 'E' }, + { 'B', 'I', 'L', 'K' }, + { 'B', 'I', 'L', 'L' }, + { 'B', 'I', 'N', 'D' }, + { 'B', 'I', 'N', 'G' }, + { 'B', 'I', 'R', 'D' }, + { 'B', 'I', 'T', 'E' }, + { 'B', 'I', 'T', 'S' }, + { 'B', 'L', 'A', 'B' }, + { 'B', 'L', 'A', 'T' }, + { 'B', 'L', 'E', 'D' }, + { 'B', 'L', 'E', 'W' }, + { 'B', 'L', 'O', 'B' }, + { 'B', 'L', 'O', 'C' }, + { 'B', 'L', 'O', 'T' }, + { 'B', 'L', 'O', 'W' }, + { 'B', 'L', 'U', 'E' }, + { 'B', 'L', 'U', 'M' }, + { 'B', 'L', 'U', 'R' }, + { 'B', 'O', 'A', 'R' }, + { 'B', 'O', 'A', 'T' }, + { 'B', 'O', 'C', 'A' }, + { 'B', 'O', 'C', 'K' }, + { 'B', 'O', 'D', 'E' }, + { 'B', 'O', 'D', 'Y' }, + { 'B', 'O', 'G', 'Y' }, + { 'B', 'O', 'H', 'R' }, + { 'B', 'O', 'I', 'L' }, + { 'B', 'O', 'L', 'D' }, + { 'B', 'O', 'L', 'O' }, + { 'B', 'O', 'L', 'T' }, + { 'B', 'O', 'M', 'B' }, + { 'B', 'O', 'N', 'A' }, + { 'B', 'O', 'N', 'D' }, + { 'B', 'O', 'N', 'E' }, + { 'B', 'O', 'N', 'G' }, + { 'B', 'O', 'N', 'N' }, + { 'B', 'O', 'N', 'Y' }, + { 'B', 'O', 'O', 'K' }, + { 'B', 'O', 'O', 'M' }, + { 'B', 'O', 'O', 'N' }, + { 'B', 'O', 'O', 'T' }, + { 'B', 'O', 'R', 'E' }, + { 'B', 'O', 'R', 'G' }, + { 'B', 'O', 'R', 'N' }, + { 'B', 'O', 'S', 'E' }, + { 'B', 'O', 'S', 'S' }, + { 'B', 'O', 'T', 'H' }, + { 'B', 'O', 'U', 'T' }, + { 'B', 'O', 'W', 'L' }, + { 'B', 'O', 'Y', 'D' }, + { 'B', 'R', 'A', 'D' }, + { 'B', 'R', 'A', 'E' }, + { 'B', 'R', 'A', 'G' }, + { 'B', 'R', 'A', 'N' }, + { 'B', 'R', 'A', 'Y' }, + { 'B', 'R', 'E', 'D' }, + { 'B', 'R', 'E', 'W' }, + { 'B', 'R', 'I', 'G' }, + { 'B', 'R', 'I', 'M' }, + { 'B', 'R', 'O', 'W' }, + { 'B', 'U', 'C', 'K' }, + { 'B', 'U', 'D', 'D' }, + { 'B', 'U', 'F', 'F' }, + { 'B', 'U', 'L', 'B' }, + { 'B', 'U', 'L', 'K' }, + { 'B', 'U', 'L', 'L' }, + { 'B', 'U', 'N', 'K' }, + { 'B', 'U', 'N', 'T' }, + { 'B', 'U', 'O', 'Y' }, + { 'B', 'U', 'R', 'G' }, + { 'B', 'U', 'R', 'L' }, + { 'B', 'U', 'R', 'N' }, + { 'B', 'U', 'R', 'R' }, + { 'B', 'U', 'R', 'T' }, + { 'B', 'U', 'R', 'Y' }, + { 'B', 'U', 'S', 'H' }, + { 'B', 'U', 'S', 'S' }, + { 'B', 'U', 'S', 'T' }, + { 'B', 'U', 'S', 'Y' }, + { 'B', 'Y', 'T', 'E' }, + { 'C', 'A', 'D', 'Y' }, + { 'C', 'A', 'F', 'E' }, + { 'C', 'A', 'G', 'E' }, + { 'C', 'A', 'I', 'N' }, + { 'C', 'A', 'K', 'E' }, + { 'C', 'A', 'L', 'F' }, + { 'C', 'A', 'L', 'L' }, + { 'C', 'A', 'L', 'M' }, + { 'C', 'A', 'M', 'E' }, + { 'C', 'A', 'N', 'E' }, + { 'C', 'A', 'N', 'T' }, + { 'C', 'A', 'R', 'D' }, + { 'C', 'A', 'R', 'E' }, + { 'C', 'A', 'R', 'L' }, + { 'C', 'A', 'R', 'R' }, + { 'C', 'A', 'R', 'T' }, + { 'C', 'A', 'S', 'E' }, + { 'C', 'A', 'S', 'H' }, + { 'C', 'A', 'S', 'K' }, + { 'C', 'A', 'S', 'T' }, + { 'C', 'A', 'V', 'E' }, + { 'C', 'E', 'I', 'L' }, + { 'C', 'E', 'L', 'L' }, + { 'C', 'E', 'N', 'T' }, + { 'C', 'E', 'R', 'N' }, + { 'C', 'H', 'A', 'D' }, + { 'C', 'H', 'A', 'R' }, + { 'C', 'H', 'A', 'T' }, + { 'C', 'H', 'A', 'W' }, + { 'C', 'H', 'E', 'F' }, + { 'C', 'H', 'E', 'N' }, + { 'C', 'H', 'E', 'W' }, + { 'C', 'H', 'I', 'C' }, + { 'C', 'H', 'I', 'N' }, + { 'C', 'H', 'O', 'U' }, + { 'C', 'H', 'O', 'W' }, + { 'C', 'H', 'U', 'B' }, + { 'C', 'H', 'U', 'G' }, + { 'C', 'H', 'U', 'M' }, + { 'C', 'I', 'T', 'E' }, + { 'C', 'I', 'T', 'Y' }, + { 'C', 'L', 'A', 'D' }, + { 'C', 'L', 'A', 'M' }, + { 'C', 'L', 'A', 'N' }, + { 'C', 'L', 'A', 'W' }, + { 'C', 'L', 'A', 'Y' }, + { 'C', 'L', 'O', 'D' }, + { 'C', 'L', 'O', 'G' }, + { 'C', 'L', 'O', 'T' }, + { 'C', 'L', 'U', 'B' }, + { 'C', 'L', 'U', 'E' }, + { 'C', 'O', 'A', 'L' }, + { 'C', 'O', 'A', 'T' }, + { 'C', 'O', 'C', 'A' }, + { 'C', 'O', 'C', 'K' }, + { 'C', 'O', 'C', 'O' }, + { 'C', 'O', 'D', 'A' }, + { 'C', 'O', 'D', 'E' }, + { 'C', 'O', 'D', 'Y' }, + { 'C', 'O', 'E', 'D' }, + { 'C', 'O', 'I', 'L' }, + { 'C', 'O', 'I', 'N' }, + { 'C', 'O', 'K', 'E' }, + { 'C', 'O', 'L', 'A' }, + { 'C', 'O', 'L', 'D' }, + { 'C', 'O', 'L', 'T' }, + { 'C', 'O', 'M', 'A' }, + { 'C', 'O', 'M', 'B' }, + { 'C', 'O', 'M', 'E' }, + { 'C', 'O', 'O', 'K' }, + { 'C', 'O', 'O', 'L' }, + { 'C', 'O', 'O', 'N' }, + { 'C', 'O', 'O', 'T' }, + { 'C', 'O', 'R', 'D' }, + { 'C', 'O', 'R', 'E' }, + { 'C', 'O', 'R', 'K' }, + { 'C', 'O', 'R', 'N' }, + { 'C', 'O', 'S', 'T' }, + { 'C', 'O', 'V', 'E' }, + { 'C', 'O', 'W', 'L' }, + { 'C', 'R', 'A', 'B' }, + { 'C', 'R', 'A', 'G' }, + { 'C', 'R', 'A', 'M' }, + { 'C', 'R', 'A', 'Y' }, + { 'C', 'R', 'E', 'W' }, + { 'C', 'R', 'I', 'B' }, + { 'C', 'R', 'O', 'W' }, + { 'C', 'R', 'U', 'D' }, + { 'C', 'U', 'B', 'A' }, + { 'C', 'U', 'B', 'E' }, + { 'C', 'U', 'F', 'F' }, + { 'C', 'U', 'L', 'L' }, + { 'C', 'U', 'L', 'T' }, + { 'C', 'U', 'N', 'Y' }, + { 'C', 'U', 'R', 'B' }, + { 'C', 'U', 'R', 'D' }, + { 'C', 'U', 'R', 'E' }, + { 'C', 'U', 'R', 'L' }, + { 'C', 'U', 'R', 'T' }, + { 'C', 'U', 'T', 'S' }, + { 'D', 'A', 'D', 'E' }, + { 'D', 'A', 'L', 'E' }, + { 'D', 'A', 'M', 'E' }, + { 'D', 'A', 'N', 'A' }, + { 'D', 'A', 'N', 'E' }, + { 'D', 'A', 'N', 'G' }, + { 'D', 'A', 'N', 'K' }, + { 'D', 'A', 'R', 'E' }, + { 'D', 'A', 'R', 'K' }, + { 'D', 'A', 'R', 'N' }, + { 'D', 'A', 'R', 'T' }, + { 'D', 'A', 'S', 'H' }, + { 'D', 'A', 'T', 'A' }, + { 'D', 'A', 'T', 'E' }, + { 'D', 'A', 'V', 'E' }, + { 'D', 'A', 'V', 'Y' }, + { 'D', 'A', 'W', 'N' }, + { 'D', 'A', 'Y', 'S' }, + { 'D', 'E', 'A', 'D' }, + { 'D', 'E', 'A', 'F' }, + { 'D', 'E', 'A', 'L' }, + { 'D', 'E', 'A', 'N' }, + { 'D', 'E', 'A', 'R' }, + { 'D', 'E', 'B', 'T' }, + { 'D', 'E', 'C', 'K' }, + { 'D', 'E', 'E', 'D' }, + { 'D', 'E', 'E', 'M' }, + { 'D', 'E', 'E', 'R' }, + { 'D', 'E', 'F', 'T' }, + { 'D', 'E', 'F', 'Y' }, + { 'D', 'E', 'L', 'L' }, + { 'D', 'E', 'N', 'T' }, + { 'D', 'E', 'N', 'Y' }, + { 'D', 'E', 'S', 'K' }, + { 'D', 'I', 'A', 'L' }, + { 'D', 'I', 'C', 'E' }, + { 'D', 'I', 'E', 'D' }, + { 'D', 'I', 'E', 'T' }, + { 'D', 'I', 'M', 'E' }, + { 'D', 'I', 'N', 'E' }, + { 'D', 'I', 'N', 'G' }, + { 'D', 'I', 'N', 'T' }, + { 'D', 'I', 'R', 'E' }, + { 'D', 'I', 'R', 'T' }, + { 'D', 'I', 'S', 'C' }, + { 'D', 'I', 'S', 'H' }, + { 'D', 'I', 'S', 'K' }, + { 'D', 'I', 'V', 'E' }, + { 'D', 'O', 'C', 'K' }, + { 'D', 'O', 'E', 'S' }, + { 'D', 'O', 'L', 'E' }, + { 'D', 'O', 'L', 'L' }, + { 'D', 'O', 'L', 'T' }, + { 'D', 'O', 'M', 'E' }, + { 'D', 'O', 'N', 'E' }, + { 'D', 'O', 'O', 'M' }, + { 'D', 'O', 'O', 'R' }, + { 'D', 'O', 'R', 'A' }, + { 'D', 'O', 'S', 'E' }, + { 'D', 'O', 'T', 'E' }, + { 'D', 'O', 'U', 'G' }, + { 'D', 'O', 'U', 'R' }, + { 'D', 'O', 'V', 'E' }, + { 'D', 'O', 'W', 'N' }, + { 'D', 'R', 'A', 'B' }, + { 'D', 'R', 'A', 'G' }, + { 'D', 'R', 'A', 'M' }, + { 'D', 'R', 'A', 'W' }, + { 'D', 'R', 'E', 'W' }, + { 'D', 'R', 'U', 'B' }, + { 'D', 'R', 'U', 'G' }, + { 'D', 'R', 'U', 'M' }, + { 'D', 'U', 'A', 'L' }, + { 'D', 'U', 'C', 'K' }, + { 'D', 'U', 'C', 'T' }, + { 'D', 'U', 'E', 'L' }, + { 'D', 'U', 'E', 'T' }, + { 'D', 'U', 'K', 'E' }, + { 'D', 'U', 'L', 'L' }, + { 'D', 'U', 'M', 'B' }, + { 'D', 'U', 'N', 'E' }, + { 'D', 'U', 'N', 'K' }, + { 'D', 'U', 'S', 'K' }, + { 'D', 'U', 'S', 'T' }, + { 'D', 'U', 'T', 'Y' }, + { 'E', 'A', 'C', 'H' }, + { 'E', 'A', 'R', 'L' }, + { 'E', 'A', 'R', 'N' }, + { 'E', 'A', 'S', 'E' }, + { 'E', 'A', 'S', 'T' }, + { 'E', 'A', 'S', 'Y' }, + { 'E', 'B', 'E', 'N' }, + { 'E', 'C', 'H', 'O' }, + { 'E', 'D', 'D', 'Y' }, + { 'E', 'D', 'E', 'N' }, + { 'E', 'D', 'G', 'E' }, + { 'E', 'D', 'G', 'Y' }, + { 'E', 'D', 'I', 'T' }, + { 'E', 'D', 'N', 'A' }, + { 'E', 'G', 'A', 'N' }, + { 'E', 'L', 'A', 'N' }, + { 'E', 'L', 'B', 'A' }, + { 'E', 'L', 'L', 'A' }, + { 'E', 'L', 'S', 'E' }, + { 'E', 'M', 'I', 'L' }, + { 'E', 'M', 'I', 'T' }, + { 'E', 'M', 'M', 'A' }, + { 'E', 'N', 'D', 'S' }, + { 'E', 'R', 'I', 'C' }, + { 'E', 'R', 'O', 'S' }, + { 'E', 'V', 'E', 'N' }, + { 'E', 'V', 'E', 'R' }, + { 'E', 'V', 'I', 'L' }, + { 'E', 'Y', 'E', 'D' }, + { 'F', 'A', 'C', 'E' }, + { 'F', 'A', 'C', 'T' }, + { 'F', 'A', 'D', 'E' }, + { 'F', 'A', 'I', 'L' }, + { 'F', 'A', 'I', 'N' }, + { 'F', 'A', 'I', 'R' }, + { 'F', 'A', 'K', 'E' }, + { 'F', 'A', 'L', 'L' }, + { 'F', 'A', 'M', 'E' }, + { 'F', 'A', 'N', 'G' }, + { 'F', 'A', 'R', 'M' }, + { 'F', 'A', 'S', 'T' }, + { 'F', 'A', 'T', 'E' }, + { 'F', 'A', 'W', 'N' }, + { 'F', 'E', 'A', 'R' }, + { 'F', 'E', 'A', 'T' }, + { 'F', 'E', 'E', 'D' }, + { 'F', 'E', 'E', 'L' }, + { 'F', 'E', 'E', 'T' }, + { 'F', 'E', 'L', 'L' }, + { 'F', 'E', 'L', 'T' }, + { 'F', 'E', 'N', 'D' }, + { 'F', 'E', 'R', 'N' }, + { 'F', 'E', 'S', 'T' }, + { 'F', 'E', 'U', 'D' }, + { 'F', 'I', 'E', 'F' }, + { 'F', 'I', 'G', 'S' }, + { 'F', 'I', 'L', 'E' }, + { 'F', 'I', 'L', 'L' }, + { 'F', 'I', 'L', 'M' }, + { 'F', 'I', 'N', 'D' }, + { 'F', 'I', 'N', 'E' }, + { 'F', 'I', 'N', 'K' }, + { 'F', 'I', 'R', 'E' }, + { 'F', 'I', 'R', 'M' }, + { 'F', 'I', 'S', 'H' }, + { 'F', 'I', 'S', 'K' }, + { 'F', 'I', 'S', 'T' }, + { 'F', 'I', 'T', 'S' }, + { 'F', 'I', 'V', 'E' }, + { 'F', 'L', 'A', 'G' }, + { 'F', 'L', 'A', 'K' }, + { 'F', 'L', 'A', 'M' }, + { 'F', 'L', 'A', 'T' }, + { 'F', 'L', 'A', 'W' }, + { 'F', 'L', 'E', 'A' }, + { 'F', 'L', 'E', 'D' }, + { 'F', 'L', 'E', 'W' }, + { 'F', 'L', 'I', 'T' }, + { 'F', 'L', 'O', 'C' }, + { 'F', 'L', 'O', 'G' }, + { 'F', 'L', 'O', 'W' }, + { 'F', 'L', 'U', 'B' }, + { 'F', 'L', 'U', 'E' }, + { 'F', 'O', 'A', 'L' }, + { 'F', 'O', 'A', 'M' }, + { 'F', 'O', 'G', 'Y' }, + { 'F', 'O', 'I', 'L' }, + { 'F', 'O', 'L', 'D' }, + { 'F', 'O', 'L', 'K' }, + { 'F', 'O', 'N', 'D' }, + { 'F', 'O', 'N', 'T' }, + { 'F', 'O', 'O', 'D' }, + { 'F', 'O', 'O', 'L' }, + { 'F', 'O', 'O', 'T' }, + { 'F', 'O', 'R', 'D' }, + { 'F', 'O', 'R', 'E' }, + { 'F', 'O', 'R', 'K' }, + { 'F', 'O', 'R', 'M' }, + { 'F', 'O', 'R', 'T' }, + { 'F', 'O', 'S', 'S' }, + { 'F', 'O', 'U', 'L' }, + { 'F', 'O', 'U', 'R' }, + { 'F', 'O', 'W', 'L' }, + { 'F', 'R', 'A', 'U' }, + { 'F', 'R', 'A', 'Y' }, + { 'F', 'R', 'E', 'D' }, + { 'F', 'R', 'E', 'E' }, + { 'F', 'R', 'E', 'T' }, + { 'F', 'R', 'E', 'Y' }, + { 'F', 'R', 'O', 'G' }, + { 'F', 'R', 'O', 'M' }, + { 'F', 'U', 'E', 'L' }, + { 'F', 'U', 'L', 'L' }, + { 'F', 'U', 'M', 'E' }, + { 'F', 'U', 'N', 'D' }, + { 'F', 'U', 'N', 'K' }, + { 'F', 'U', 'R', 'Y' }, + { 'F', 'U', 'S', 'E' }, + { 'F', 'U', 'S', 'S' }, + { 'G', 'A', 'F', 'F' }, + { 'G', 'A', 'G', 'E' }, + { 'G', 'A', 'I', 'L' }, + { 'G', 'A', 'I', 'N' }, + { 'G', 'A', 'I', 'T' }, + { 'G', 'A', 'L', 'A' }, + { 'G', 'A', 'L', 'E' }, + { 'G', 'A', 'L', 'L' }, + { 'G', 'A', 'L', 'T' }, + { 'G', 'A', 'M', 'E' }, + { 'G', 'A', 'N', 'G' }, + { 'G', 'A', 'R', 'B' }, + { 'G', 'A', 'R', 'Y' }, + { 'G', 'A', 'S', 'H' }, + { 'G', 'A', 'T', 'E' }, + { 'G', 'A', 'U', 'L' }, + { 'G', 'A', 'U', 'R' }, + { 'G', 'A', 'V', 'E' }, + { 'G', 'A', 'W', 'K' }, + { 'G', 'E', 'A', 'R' }, + { 'G', 'E', 'L', 'D' }, + { 'G', 'E', 'N', 'E' }, + { 'G', 'E', 'N', 'T' }, + { 'G', 'E', 'R', 'M' }, + { 'G', 'E', 'T', 'S' }, + { 'G', 'I', 'B', 'E' }, + { 'G', 'I', 'F', 'T' }, + { 'G', 'I', 'L', 'D' }, + { 'G', 'I', 'L', 'L' }, + { 'G', 'I', 'L', 'T' }, + { 'G', 'I', 'N', 'A' }, + { 'G', 'I', 'R', 'D' }, + { 'G', 'I', 'R', 'L' }, + { 'G', 'I', 'S', 'T' }, + { 'G', 'I', 'V', 'E' }, + { 'G', 'L', 'A', 'D' }, + { 'G', 'L', 'E', 'E' }, + { 'G', 'L', 'E', 'N' }, + { 'G', 'L', 'I', 'B' }, + { 'G', 'L', 'O', 'B' }, + { 'G', 'L', 'O', 'M' }, + { 'G', 'L', 'O', 'W' }, + { 'G', 'L', 'U', 'E' }, + { 'G', 'L', 'U', 'M' }, + { 'G', 'L', 'U', 'T' }, + { 'G', 'O', 'A', 'D' }, + { 'G', 'O', 'A', 'L' }, + { 'G', 'O', 'A', 'T' }, + { 'G', 'O', 'E', 'R' }, + { 'G', 'O', 'E', 'S' }, + { 'G', 'O', 'L', 'D' }, + { 'G', 'O', 'L', 'F' }, + { 'G', 'O', 'N', 'E' }, + { 'G', 'O', 'N', 'G' }, + { 'G', 'O', 'O', 'D' }, + { 'G', 'O', 'O', 'F' }, + { 'G', 'O', 'R', 'E' }, + { 'G', 'O', 'R', 'Y' }, + { 'G', 'O', 'S', 'H' }, + { 'G', 'O', 'U', 'T' }, + { 'G', 'O', 'W', 'N' }, + { 'G', 'R', 'A', 'B' }, + { 'G', 'R', 'A', 'D' }, + { 'G', 'R', 'A', 'Y' }, + { 'G', 'R', 'E', 'G' }, + { 'G', 'R', 'E', 'W' }, + { 'G', 'R', 'E', 'Y' }, + { 'G', 'R', 'I', 'D' }, + { 'G', 'R', 'I', 'M' }, + { 'G', 'R', 'I', 'N' }, + { 'G', 'R', 'I', 'T' }, + { 'G', 'R', 'O', 'W' }, + { 'G', 'R', 'U', 'B' }, + { 'G', 'U', 'L', 'F' }, + { 'G', 'U', 'L', 'L' }, + { 'G', 'U', 'N', 'K' }, + { 'G', 'U', 'R', 'U' }, + { 'G', 'U', 'S', 'H' }, + { 'G', 'U', 'S', 'T' }, + { 'G', 'W', 'E', 'N' }, + { 'G', 'W', 'Y', 'N' }, + { 'H', 'A', 'A', 'G' }, + { 'H', 'A', 'A', 'S' }, + { 'H', 'A', 'C', 'K' }, + { 'H', 'A', 'I', 'L' }, + { 'H', 'A', 'I', 'R' }, + { 'H', 'A', 'L', 'E' }, + { 'H', 'A', 'L', 'F' }, + { 'H', 'A', 'L', 'L' }, + { 'H', 'A', 'L', 'O' }, + { 'H', 'A', 'L', 'T' }, + { 'H', 'A', 'N', 'D' }, + { 'H', 'A', 'N', 'G' }, + { 'H', 'A', 'N', 'K' }, + { 'H', 'A', 'N', 'S' }, + { 'H', 'A', 'R', 'D' }, + { 'H', 'A', 'R', 'K' }, + { 'H', 'A', 'R', 'M' }, + { 'H', 'A', 'R', 'T' }, + { 'H', 'A', 'S', 'H' }, + { 'H', 'A', 'S', 'T' }, + { 'H', 'A', 'T', 'E' }, + { 'H', 'A', 'T', 'H' }, + { 'H', 'A', 'U', 'L' }, + { 'H', 'A', 'V', 'E' }, + { 'H', 'A', 'W', 'K' }, + { 'H', 'A', 'Y', 'S' }, + { 'H', 'E', 'A', 'D' }, + { 'H', 'E', 'A', 'L' }, + { 'H', 'E', 'A', 'R' }, + { 'H', 'E', 'A', 'T' }, + { 'H', 'E', 'B', 'E' }, + { 'H', 'E', 'C', 'K' }, + { 'H', 'E', 'E', 'D' }, + { 'H', 'E', 'E', 'L' }, + { 'H', 'E', 'F', 'T' }, + { 'H', 'E', 'L', 'D' }, + { 'H', 'E', 'L', 'L' }, + { 'H', 'E', 'L', 'M' }, + { 'H', 'E', 'R', 'B' }, + { 'H', 'E', 'R', 'D' }, + { 'H', 'E', 'R', 'E' }, + { 'H', 'E', 'R', 'O' }, + { 'H', 'E', 'R', 'S' }, + { 'H', 'E', 'S', 'S' }, + { 'H', 'E', 'W', 'N' }, + { 'H', 'I', 'C', 'K' }, + { 'H', 'I', 'D', 'E' }, + { 'H', 'I', 'G', 'H' }, + { 'H', 'I', 'K', 'E' }, + { 'H', 'I', 'L', 'L' }, + { 'H', 'I', 'L', 'T' }, + { 'H', 'I', 'N', 'D' }, + { 'H', 'I', 'N', 'T' }, + { 'H', 'I', 'R', 'E' }, + { 'H', 'I', 'S', 'S' }, + { 'H', 'I', 'V', 'E' }, + { 'H', 'O', 'B', 'O' }, + { 'H', 'O', 'C', 'K' }, + { 'H', 'O', 'F', 'F' }, + { 'H', 'O', 'L', 'D' }, + { 'H', 'O', 'L', 'E' }, + { 'H', 'O', 'L', 'M' }, + { 'H', 'O', 'L', 'T' }, + { 'H', 'O', 'M', 'E' }, + { 'H', 'O', 'N', 'E' }, + { 'H', 'O', 'N', 'K' }, + { 'H', 'O', 'O', 'D' }, + { 'H', 'O', 'O', 'F' }, + { 'H', 'O', 'O', 'K' }, + { 'H', 'O', 'O', 'T' }, + { 'H', 'O', 'R', 'N' }, + { 'H', 'O', 'S', 'E' }, + { 'H', 'O', 'S', 'T' }, + { 'H', 'O', 'U', 'R' }, + { 'H', 'O', 'V', 'E' }, + { 'H', 'O', 'W', 'E' }, + { 'H', 'O', 'W', 'L' }, + { 'H', 'O', 'Y', 'T' }, + { 'H', 'U', 'C', 'K' }, + { 'H', 'U', 'E', 'D' }, + { 'H', 'U', 'F', 'F' }, + { 'H', 'U', 'G', 'E' }, + { 'H', 'U', 'G', 'H' }, + { 'H', 'U', 'G', 'O' }, + { 'H', 'U', 'L', 'K' }, + { 'H', 'U', 'L', 'L' }, + { 'H', 'U', 'N', 'K' }, + { 'H', 'U', 'N', 'T' }, + { 'H', 'U', 'R', 'D' }, + { 'H', 'U', 'R', 'L' }, + { 'H', 'U', 'R', 'T' }, + { 'H', 'U', 'S', 'H' }, + { 'H', 'Y', 'D', 'E' }, + { 'H', 'Y', 'M', 'N' }, + { 'I', 'B', 'I', 'S' }, + { 'I', 'C', 'O', 'N' }, + { 'I', 'D', 'E', 'A' }, + { 'I', 'D', 'L', 'E' }, + { 'I', 'F', 'F', 'Y' }, + { 'I', 'N', 'C', 'A' }, + { 'I', 'N', 'C', 'H' }, + { 'I', 'N', 'T', 'O' }, + { 'I', 'O', 'N', 'S' }, + { 'I', 'O', 'T', 'A' }, + { 'I', 'O', 'W', 'A' }, + { 'I', 'R', 'I', 'S' }, + { 'I', 'R', 'M', 'A' }, + { 'I', 'R', 'O', 'N' }, + { 'I', 'S', 'L', 'E' }, + { 'I', 'T', 'C', 'H' }, + { 'I', 'T', 'E', 'M' }, + { 'I', 'V', 'A', 'N' }, + { 'J', 'A', 'C', 'K' }, + { 'J', 'A', 'D', 'E' }, + { 'J', 'A', 'I', 'L' }, + { 'J', 'A', 'K', 'E' }, + { 'J', 'A', 'N', 'E' }, + { 'J', 'A', 'V', 'A' }, + { 'J', 'E', 'A', 'N' }, + { 'J', 'E', 'F', 'F' }, + { 'J', 'E', 'R', 'K' }, + { 'J', 'E', 'S', 'S' }, + { 'J', 'E', 'S', 'T' }, + { 'J', 'I', 'B', 'E' }, + { 'J', 'I', 'L', 'L' }, + { 'J', 'I', 'L', 'T' }, + { 'J', 'I', 'V', 'E' }, + { 'J', 'O', 'A', 'N' }, + { 'J', 'O', 'B', 'S' }, + { 'J', 'O', 'C', 'K' }, + { 'J', 'O', 'E', 'L' }, + { 'J', 'O', 'E', 'Y' }, + { 'J', 'O', 'H', 'N' }, + { 'J', 'O', 'I', 'N' }, + { 'J', 'O', 'K', 'E' }, + { 'J', 'O', 'L', 'T' }, + { 'J', 'O', 'V', 'E' }, + { 'J', 'U', 'D', 'D' }, + { 'J', 'U', 'D', 'E' }, + { 'J', 'U', 'D', 'O' }, + { 'J', 'U', 'D', 'Y' }, + { 'J', 'U', 'J', 'U' }, + { 'J', 'U', 'K', 'E' }, + { 'J', 'U', 'L', 'Y' }, + { 'J', 'U', 'N', 'E' }, + { 'J', 'U', 'N', 'K' }, + { 'J', 'U', 'N', 'O' }, + { 'J', 'U', 'R', 'Y' }, + { 'J', 'U', 'S', 'T' }, + { 'J', 'U', 'T', 'E' }, + { 'K', 'A', 'H', 'N' }, + { 'K', 'A', 'L', 'E' }, + { 'K', 'A', 'N', 'E' }, + { 'K', 'A', 'N', 'T' }, + { 'K', 'A', 'R', 'L' }, + { 'K', 'A', 'T', 'E' }, + { 'K', 'E', 'E', 'L' }, + { 'K', 'E', 'E', 'N' }, + { 'K', 'E', 'N', 'O' }, + { 'K', 'E', 'N', 'T' }, + { 'K', 'E', 'R', 'N' }, + { 'K', 'E', 'R', 'R' }, + { 'K', 'E', 'Y', 'S' }, + { 'K', 'I', 'C', 'K' }, + { 'K', 'I', 'L', 'L' }, + { 'K', 'I', 'N', 'D' }, + { 'K', 'I', 'N', 'G' }, + { 'K', 'I', 'R', 'K' }, + { 'K', 'I', 'S', 'S' }, + { 'K', 'I', 'T', 'E' }, + { 'K', 'L', 'A', 'N' }, + { 'K', 'N', 'E', 'E' }, + { 'K', 'N', 'E', 'W' }, + { 'K', 'N', 'I', 'T' }, + { 'K', 'N', 'O', 'B' }, + { 'K', 'N', 'O', 'T' }, + { 'K', 'N', 'O', 'W' }, + { 'K', 'O', 'C', 'H' }, + { 'K', 'O', 'N', 'G' }, + { 'K', 'U', 'D', 'O' }, + { 'K', 'U', 'R', 'D' }, + { 'K', 'U', 'R', 'T' }, + { 'K', 'Y', 'L', 'E' }, + { 'L', 'A', 'C', 'E' }, + { 'L', 'A', 'C', 'K' }, + { 'L', 'A', 'C', 'Y' }, + { 'L', 'A', 'D', 'Y' }, + { 'L', 'A', 'I', 'D' }, + { 'L', 'A', 'I', 'N' }, + { 'L', 'A', 'I', 'R' }, + { 'L', 'A', 'K', 'E' }, + { 'L', 'A', 'M', 'B' }, + { 'L', 'A', 'M', 'E' }, + { 'L', 'A', 'N', 'D' }, + { 'L', 'A', 'N', 'E' }, + { 'L', 'A', 'N', 'G' }, + { 'L', 'A', 'R', 'D' }, + { 'L', 'A', 'R', 'K' }, + { 'L', 'A', 'S', 'S' }, + { 'L', 'A', 'S', 'T' }, + { 'L', 'A', 'T', 'E' }, + { 'L', 'A', 'U', 'D' }, + { 'L', 'A', 'V', 'A' }, + { 'L', 'A', 'W', 'N' }, + { 'L', 'A', 'W', 'S' }, + { 'L', 'A', 'Y', 'S' }, + { 'L', 'E', 'A', 'D' }, + { 'L', 'E', 'A', 'F' }, + { 'L', 'E', 'A', 'K' }, + { 'L', 'E', 'A', 'N' }, + { 'L', 'E', 'A', 'R' }, + { 'L', 'E', 'E', 'K' }, + { 'L', 'E', 'E', 'R' }, + { 'L', 'E', 'F', 'T' }, + { 'L', 'E', 'N', 'D' }, + { 'L', 'E', 'N', 'S' }, + { 'L', 'E', 'N', 'T' }, + { 'L', 'E', 'O', 'N' }, + { 'L', 'E', 'S', 'K' }, + { 'L', 'E', 'S', 'S' }, + { 'L', 'E', 'S', 'T' }, + { 'L', 'E', 'T', 'S' }, + { 'L', 'I', 'A', 'R' }, + { 'L', 'I', 'C', 'E' }, + { 'L', 'I', 'C', 'K' }, + { 'L', 'I', 'E', 'D' }, + { 'L', 'I', 'E', 'N' }, + { 'L', 'I', 'E', 'S' }, + { 'L', 'I', 'E', 'U' }, + { 'L', 'I', 'F', 'E' }, + { 'L', 'I', 'F', 'T' }, + { 'L', 'I', 'K', 'E' }, + { 'L', 'I', 'L', 'A' }, + { 'L', 'I', 'L', 'T' }, + { 'L', 'I', 'L', 'Y' }, + { 'L', 'I', 'M', 'A' }, + { 'L', 'I', 'M', 'B' }, + { 'L', 'I', 'M', 'E' }, + { 'L', 'I', 'N', 'D' }, + { 'L', 'I', 'N', 'E' }, + { 'L', 'I', 'N', 'K' }, + { 'L', 'I', 'N', 'T' }, + { 'L', 'I', 'O', 'N' }, + { 'L', 'I', 'S', 'A' }, + { 'L', 'I', 'S', 'T' }, + { 'L', 'I', 'V', 'E' }, + { 'L', 'O', 'A', 'D' }, + { 'L', 'O', 'A', 'F' }, + { 'L', 'O', 'A', 'M' }, + { 'L', 'O', 'A', 'N' }, + { 'L', 'O', 'C', 'K' }, + { 'L', 'O', 'F', 'T' }, + { 'L', 'O', 'G', 'E' }, + { 'L', 'O', 'I', 'S' }, + { 'L', 'O', 'L', 'A' }, + { 'L', 'O', 'N', 'E' }, + { 'L', 'O', 'N', 'G' }, + { 'L', 'O', 'O', 'K' }, + { 'L', 'O', 'O', 'N' }, + { 'L', 'O', 'O', 'T' }, + { 'L', 'O', 'R', 'D' }, + { 'L', 'O', 'R', 'E' }, + { 'L', 'O', 'S', 'E' }, + { 'L', 'O', 'S', 'S' }, + { 'L', 'O', 'S', 'T' }, + { 'L', 'O', 'U', 'D' }, + { 'L', 'O', 'V', 'E' }, + { 'L', 'O', 'W', 'E' }, + { 'L', 'U', 'C', 'K' }, + { 'L', 'U', 'C', 'Y' }, + { 'L', 'U', 'G', 'E' }, + { 'L', 'U', 'K', 'E' }, + { 'L', 'U', 'L', 'U' }, + { 'L', 'U', 'N', 'D' }, + { 'L', 'U', 'N', 'G' }, + { 'L', 'U', 'R', 'A' }, + { 'L', 'U', 'R', 'E' }, + { 'L', 'U', 'R', 'K' }, + { 'L', 'U', 'S', 'H' }, + { 'L', 'U', 'S', 'T' }, + { 'L', 'Y', 'L', 'E' }, + { 'L', 'Y', 'N', 'N' }, + { 'L', 'Y', 'O', 'N' }, + { 'L', 'Y', 'R', 'A' }, + { 'M', 'A', 'C', 'E' }, + { 'M', 'A', 'D', 'E' }, + { 'M', 'A', 'G', 'I' }, + { 'M', 'A', 'I', 'D' }, + { 'M', 'A', 'I', 'L' }, + { 'M', 'A', 'I', 'N' }, + { 'M', 'A', 'K', 'E' }, + { 'M', 'A', 'L', 'E' }, + { 'M', 'A', 'L', 'I' }, + { 'M', 'A', 'L', 'L' }, + { 'M', 'A', 'L', 'T' }, + { 'M', 'A', 'N', 'A' }, + { 'M', 'A', 'N', 'N' }, + { 'M', 'A', 'N', 'Y' }, + { 'M', 'A', 'R', 'C' }, + { 'M', 'A', 'R', 'E' }, + { 'M', 'A', 'R', 'K' }, + { 'M', 'A', 'R', 'S' }, + { 'M', 'A', 'R', 'T' }, + { 'M', 'A', 'R', 'Y' }, + { 'M', 'A', 'S', 'H' }, + { 'M', 'A', 'S', 'K' }, + { 'M', 'A', 'S', 'S' }, + { 'M', 'A', 'S', 'T' }, + { 'M', 'A', 'T', 'E' }, + { 'M', 'A', 'T', 'H' }, + { 'M', 'A', 'U', 'L' }, + { 'M', 'A', 'Y', 'O' }, + { 'M', 'E', 'A', 'D' }, + { 'M', 'E', 'A', 'L' }, + { 'M', 'E', 'A', 'N' }, + { 'M', 'E', 'A', 'T' }, + { 'M', 'E', 'E', 'K' }, + { 'M', 'E', 'E', 'T' }, + { 'M', 'E', 'L', 'D' }, + { 'M', 'E', 'L', 'T' }, + { 'M', 'E', 'M', 'O' }, + { 'M', 'E', 'N', 'D' }, + { 'M', 'E', 'N', 'U' }, + { 'M', 'E', 'R', 'T' }, + { 'M', 'E', 'S', 'H' }, + { 'M', 'E', 'S', 'S' }, + { 'M', 'I', 'C', 'E' }, + { 'M', 'I', 'K', 'E' }, + { 'M', 'I', 'L', 'D' }, + { 'M', 'I', 'L', 'E' }, + { 'M', 'I', 'L', 'K' }, + { 'M', 'I', 'L', 'L' }, + { 'M', 'I', 'L', 'T' }, + { 'M', 'I', 'M', 'I' }, + { 'M', 'I', 'N', 'D' }, + { 'M', 'I', 'N', 'E' }, + { 'M', 'I', 'N', 'I' }, + { 'M', 'I', 'N', 'K' }, + { 'M', 'I', 'N', 'T' }, + { 'M', 'I', 'R', 'E' }, + { 'M', 'I', 'S', 'S' }, + { 'M', 'I', 'S', 'T' }, + { 'M', 'I', 'T', 'E' }, + { 'M', 'I', 'T', 'T' }, + { 'M', 'O', 'A', 'N' }, + { 'M', 'O', 'A', 'T' }, + { 'M', 'O', 'C', 'K' }, + { 'M', 'O', 'D', 'E' }, + { 'M', 'O', 'L', 'D' }, + { 'M', 'O', 'L', 'E' }, + { 'M', 'O', 'L', 'L' }, + { 'M', 'O', 'L', 'T' }, + { 'M', 'O', 'N', 'A' }, + { 'M', 'O', 'N', 'K' }, + { 'M', 'O', 'N', 'T' }, + { 'M', 'O', 'O', 'D' }, + { 'M', 'O', 'O', 'N' }, + { 'M', 'O', 'O', 'R' }, + { 'M', 'O', 'O', 'T' }, + { 'M', 'O', 'R', 'E' }, + { 'M', 'O', 'R', 'N' }, + { 'M', 'O', 'R', 'T' }, + { 'M', 'O', 'S', 'S' }, + { 'M', 'O', 'S', 'T' }, + { 'M', 'O', 'T', 'H' }, + { 'M', 'O', 'V', 'E' }, + { 'M', 'U', 'C', 'H' }, + { 'M', 'U', 'C', 'K' }, + { 'M', 'U', 'D', 'D' }, + { 'M', 'U', 'F', 'F' }, + { 'M', 'U', 'L', 'E' }, + { 'M', 'U', 'L', 'L' }, + { 'M', 'U', 'R', 'K' }, + { 'M', 'U', 'S', 'H' }, + { 'M', 'U', 'S', 'T' }, + { 'M', 'U', 'T', 'E' }, + { 'M', 'U', 'T', 'T' }, + { 'M', 'Y', 'R', 'A' }, + { 'M', 'Y', 'T', 'H' }, + { 'N', 'A', 'G', 'Y' }, + { 'N', 'A', 'I', 'L' }, + { 'N', 'A', 'I', 'R' }, + { 'N', 'A', 'M', 'E' }, + { 'N', 'A', 'R', 'Y' }, + { 'N', 'A', 'S', 'H' }, + { 'N', 'A', 'V', 'E' }, + { 'N', 'A', 'V', 'Y' }, + { 'N', 'E', 'A', 'L' }, + { 'N', 'E', 'A', 'R' }, + { 'N', 'E', 'A', 'T' }, + { 'N', 'E', 'C', 'K' }, + { 'N', 'E', 'E', 'D' }, + { 'N', 'E', 'I', 'L' }, + { 'N', 'E', 'L', 'L' }, + { 'N', 'E', 'O', 'N' }, + { 'N', 'E', 'R', 'O' }, + { 'N', 'E', 'S', 'S' }, + { 'N', 'E', 'S', 'T' }, + { 'N', 'E', 'W', 'S' }, + { 'N', 'E', 'W', 'T' }, + { 'N', 'I', 'B', 'S' }, + { 'N', 'I', 'C', 'E' }, + { 'N', 'I', 'C', 'K' }, + { 'N', 'I', 'L', 'E' }, + { 'N', 'I', 'N', 'A' }, + { 'N', 'I', 'N', 'E' }, + { 'N', 'O', 'A', 'H' }, + { 'N', 'O', 'D', 'E' }, + { 'N', 'O', 'E', 'L' }, + { 'N', 'O', 'L', 'L' }, + { 'N', 'O', 'N', 'E' }, + { 'N', 'O', 'O', 'K' }, + { 'N', 'O', 'O', 'N' }, + { 'N', 'O', 'R', 'M' }, + { 'N', 'O', 'S', 'E' }, + { 'N', 'O', 'T', 'E' }, + { 'N', 'O', 'U', 'N' }, + { 'N', 'O', 'V', 'A' }, + { 'N', 'U', 'D', 'E' }, + { 'N', 'U', 'L', 'L' }, + { 'N', 'U', 'M', 'B' }, + { 'O', 'A', 'T', 'H' }, + { 'O', 'B', 'E', 'Y' }, + { 'O', 'B', 'O', 'E' }, + { 'O', 'D', 'I', 'N' }, + { 'O', 'H', 'I', 'O' }, + { 'O', 'I', 'L', 'Y' }, + { 'O', 'I', 'N', 'T' }, + { 'O', 'K', 'A', 'Y' }, + { 'O', 'L', 'A', 'F' }, + { 'O', 'L', 'D', 'Y' }, + { 'O', 'L', 'G', 'A' }, + { 'O', 'L', 'I', 'N' }, + { 'O', 'M', 'A', 'N' }, + { 'O', 'M', 'E', 'N' }, + { 'O', 'M', 'I', 'T' }, + { 'O', 'N', 'C', 'E' }, + { 'O', 'N', 'E', 'S' }, + { 'O', 'N', 'L', 'Y' }, + { 'O', 'N', 'T', 'O' }, + { 'O', 'N', 'U', 'S' }, + { 'O', 'R', 'A', 'L' }, + { 'O', 'R', 'G', 'Y' }, + { 'O', 'S', 'L', 'O' }, + { 'O', 'T', 'I', 'S' }, + { 'O', 'T', 'T', 'O' }, + { 'O', 'U', 'C', 'H' }, + { 'O', 'U', 'S', 'T' }, + { 'O', 'U', 'T', 'S' }, + { 'O', 'V', 'A', 'L' }, + { 'O', 'V', 'E', 'N' }, + { 'O', 'V', 'E', 'R' }, + { 'O', 'W', 'L', 'Y' }, + { 'O', 'W', 'N', 'S' }, + { 'Q', 'U', 'A', 'D' }, + { 'Q', 'U', 'I', 'T' }, + { 'Q', 'U', 'O', 'D' }, + { 'R', 'A', 'C', 'E' }, + { 'R', 'A', 'C', 'K' }, + { 'R', 'A', 'C', 'Y' }, + { 'R', 'A', 'F', 'T' }, + { 'R', 'A', 'G', 'E' }, + { 'R', 'A', 'I', 'D' }, + { 'R', 'A', 'I', 'L' }, + { 'R', 'A', 'I', 'N' }, + { 'R', 'A', 'K', 'E' }, + { 'R', 'A', 'N', 'K' }, + { 'R', 'A', 'N', 'T' }, + { 'R', 'A', 'R', 'E' }, + { 'R', 'A', 'S', 'H' }, + { 'R', 'A', 'T', 'E' }, + { 'R', 'A', 'V', 'E' }, + { 'R', 'A', 'Y', 'S' }, + { 'R', 'E', 'A', 'D' }, + { 'R', 'E', 'A', 'L' }, + { 'R', 'E', 'A', 'M' }, + { 'R', 'E', 'A', 'R' }, + { 'R', 'E', 'C', 'K' }, + { 'R', 'E', 'E', 'D' }, + { 'R', 'E', 'E', 'F' }, + { 'R', 'E', 'E', 'K' }, + { 'R', 'E', 'E', 'L' }, + { 'R', 'E', 'I', 'D' }, + { 'R', 'E', 'I', 'N' }, + { 'R', 'E', 'N', 'A' }, + { 'R', 'E', 'N', 'D' }, + { 'R', 'E', 'N', 'T' }, + { 'R', 'E', 'S', 'T' }, + { 'R', 'I', 'C', 'E' }, + { 'R', 'I', 'C', 'H' }, + { 'R', 'I', 'C', 'K' }, + { 'R', 'I', 'D', 'E' }, + { 'R', 'I', 'F', 'T' }, + { 'R', 'I', 'L', 'L' }, + { 'R', 'I', 'M', 'E' }, + { 'R', 'I', 'N', 'G' }, + { 'R', 'I', 'N', 'K' }, + { 'R', 'I', 'S', 'E' }, + { 'R', 'I', 'S', 'K' }, + { 'R', 'I', 'T', 'E' }, + { 'R', 'O', 'A', 'D' }, + { 'R', 'O', 'A', 'M' }, + { 'R', 'O', 'A', 'R' }, + { 'R', 'O', 'B', 'E' }, + { 'R', 'O', 'C', 'K' }, + { 'R', 'O', 'D', 'E' }, + { 'R', 'O', 'I', 'L' }, + { 'R', 'O', 'L', 'L' }, + { 'R', 'O', 'M', 'E' }, + { 'R', 'O', 'O', 'D' }, + { 'R', 'O', 'O', 'F' }, + { 'R', 'O', 'O', 'K' }, + { 'R', 'O', 'O', 'M' }, + { 'R', 'O', 'O', 'T' }, + { 'R', 'O', 'S', 'A' }, + { 'R', 'O', 'S', 'E' }, + { 'R', 'O', 'S', 'S' }, + { 'R', 'O', 'S', 'Y' }, + { 'R', 'O', 'T', 'H' }, + { 'R', 'O', 'U', 'T' }, + { 'R', 'O', 'V', 'E' }, + { 'R', 'O', 'W', 'E' }, + { 'R', 'O', 'W', 'S' }, + { 'R', 'U', 'B', 'E' }, + { 'R', 'U', 'B', 'Y' }, + { 'R', 'U', 'D', 'E' }, + { 'R', 'U', 'D', 'Y' }, + { 'R', 'U', 'I', 'N' }, + { 'R', 'U', 'L', 'E' }, + { 'R', 'U', 'N', 'G' }, + { 'R', 'U', 'N', 'S' }, + { 'R', 'U', 'N', 'T' }, + { 'R', 'U', 'S', 'E' }, + { 'R', 'U', 'S', 'H' }, + { 'R', 'U', 'S', 'K' }, + { 'R', 'U', 'S', 'S' }, + { 'R', 'U', 'S', 'T' }, + { 'R', 'U', 'T', 'H' }, + { 'S', 'A', 'C', 'K' }, + { 'S', 'A', 'F', 'E' }, + { 'S', 'A', 'G', 'E' }, + { 'S', 'A', 'I', 'D' }, + { 'S', 'A', 'I', 'L' }, + { 'S', 'A', 'L', 'E' }, + { 'S', 'A', 'L', 'K' }, + { 'S', 'A', 'L', 'T' }, + { 'S', 'A', 'M', 'E' }, + { 'S', 'A', 'N', 'D' }, + { 'S', 'A', 'N', 'E' }, + { 'S', 'A', 'N', 'G' }, + { 'S', 'A', 'N', 'K' }, + { 'S', 'A', 'R', 'A' }, + { 'S', 'A', 'U', 'L' }, + { 'S', 'A', 'V', 'E' }, + { 'S', 'A', 'Y', 'S' }, + { 'S', 'C', 'A', 'N' }, + { 'S', 'C', 'A', 'R' }, + { 'S', 'C', 'A', 'T' }, + { 'S', 'C', 'O', 'T' }, + { 'S', 'E', 'A', 'L' }, + { 'S', 'E', 'A', 'M' }, + { 'S', 'E', 'A', 'R' }, + { 'S', 'E', 'A', 'T' }, + { 'S', 'E', 'E', 'D' }, + { 'S', 'E', 'E', 'K' }, + { 'S', 'E', 'E', 'M' }, + { 'S', 'E', 'E', 'N' }, + { 'S', 'E', 'E', 'S' }, + { 'S', 'E', 'L', 'F' }, + { 'S', 'E', 'L', 'L' }, + { 'S', 'E', 'N', 'D' }, + { 'S', 'E', 'N', 'T' }, + { 'S', 'E', 'T', 'S' }, + { 'S', 'E', 'W', 'N' }, + { 'S', 'H', 'A', 'G' }, + { 'S', 'H', 'A', 'M' }, + { 'S', 'H', 'A', 'W' }, + { 'S', 'H', 'A', 'Y' }, + { 'S', 'H', 'E', 'D' }, + { 'S', 'H', 'I', 'M' }, + { 'S', 'H', 'I', 'N' }, + { 'S', 'H', 'O', 'D' }, + { 'S', 'H', 'O', 'E' }, + { 'S', 'H', 'O', 'T' }, + { 'S', 'H', 'O', 'W' }, + { 'S', 'H', 'U', 'N' }, + { 'S', 'H', 'U', 'T' }, + { 'S', 'I', 'C', 'K' }, + { 'S', 'I', 'D', 'E' }, + { 'S', 'I', 'F', 'T' }, + { 'S', 'I', 'G', 'H' }, + { 'S', 'I', 'G', 'N' }, + { 'S', 'I', 'L', 'K' }, + { 'S', 'I', 'L', 'L' }, + { 'S', 'I', 'L', 'O' }, + { 'S', 'I', 'L', 'T' }, + { 'S', 'I', 'N', 'E' }, + { 'S', 'I', 'N', 'G' }, + { 'S', 'I', 'N', 'K' }, + { 'S', 'I', 'R', 'E' }, + { 'S', 'I', 'T', 'E' }, + { 'S', 'I', 'T', 'S' }, + { 'S', 'I', 'T', 'U' }, + { 'S', 'K', 'A', 'T' }, + { 'S', 'K', 'E', 'W' }, + { 'S', 'K', 'I', 'D' }, + { 'S', 'K', 'I', 'M' }, + { 'S', 'K', 'I', 'N' }, + { 'S', 'K', 'I', 'T' }, + { 'S', 'L', 'A', 'B' }, + { 'S', 'L', 'A', 'M' }, + { 'S', 'L', 'A', 'T' }, + { 'S', 'L', 'A', 'Y' }, + { 'S', 'L', 'E', 'D' }, + { 'S', 'L', 'E', 'W' }, + { 'S', 'L', 'I', 'D' }, + { 'S', 'L', 'I', 'M' }, + { 'S', 'L', 'I', 'T' }, + { 'S', 'L', 'O', 'B' }, + { 'S', 'L', 'O', 'G' }, + { 'S', 'L', 'O', 'T' }, + { 'S', 'L', 'O', 'W' }, + { 'S', 'L', 'U', 'G' }, + { 'S', 'L', 'U', 'M' }, + { 'S', 'L', 'U', 'R' }, + { 'S', 'M', 'O', 'G' }, + { 'S', 'M', 'U', 'G' }, + { 'S', 'N', 'A', 'G' }, + { 'S', 'N', 'O', 'B' }, + { 'S', 'N', 'O', 'W' }, + { 'S', 'N', 'U', 'B' }, + { 'S', 'N', 'U', 'G' }, + { 'S', 'O', 'A', 'K' }, + { 'S', 'O', 'A', 'R' }, + { 'S', 'O', 'C', 'K' }, + { 'S', 'O', 'D', 'A' }, + { 'S', 'O', 'F', 'A' }, + { 'S', 'O', 'F', 'T' }, + { 'S', 'O', 'I', 'L' }, + { 'S', 'O', 'L', 'D' }, + { 'S', 'O', 'M', 'E' }, + { 'S', 'O', 'N', 'G' }, + { 'S', 'O', 'O', 'N' }, + { 'S', 'O', 'O', 'T' }, + { 'S', 'O', 'R', 'E' }, + { 'S', 'O', 'R', 'T' }, + { 'S', 'O', 'U', 'L' }, + { 'S', 'O', 'U', 'R' }, + { 'S', 'O', 'W', 'N' }, + { 'S', 'T', 'A', 'B' }, + { 'S', 'T', 'A', 'G' }, + { 'S', 'T', 'A', 'N' }, + { 'S', 'T', 'A', 'R' }, + { 'S', 'T', 'A', 'Y' }, + { 'S', 'T', 'E', 'M' }, + { 'S', 'T', 'E', 'W' }, + { 'S', 'T', 'I', 'R' }, + { 'S', 'T', 'O', 'W' }, + { 'S', 'T', 'U', 'B' }, + { 'S', 'T', 'U', 'N' }, + { 'S', 'U', 'C', 'H' }, + { 'S', 'U', 'D', 'S' }, + { 'S', 'U', 'I', 'T' }, + { 'S', 'U', 'L', 'K' }, + { 'S', 'U', 'M', 'S' }, + { 'S', 'U', 'N', 'G' }, + { 'S', 'U', 'N', 'K' }, + { 'S', 'U', 'R', 'E' }, + { 'S', 'U', 'R', 'F' }, + { 'S', 'W', 'A', 'B' }, + { 'S', 'W', 'A', 'G' }, + { 'S', 'W', 'A', 'M' }, + { 'S', 'W', 'A', 'N' }, + { 'S', 'W', 'A', 'T' }, + { 'S', 'W', 'A', 'Y' }, + { 'S', 'W', 'I', 'M' }, + { 'S', 'W', 'U', 'M' }, + { 'T', 'A', 'C', 'K' }, + { 'T', 'A', 'C', 'T' }, + { 'T', 'A', 'I', 'L' }, + { 'T', 'A', 'K', 'E' }, + { 'T', 'A', 'L', 'E' }, + { 'T', 'A', 'L', 'K' }, + { 'T', 'A', 'L', 'L' }, + { 'T', 'A', 'N', 'K' }, + { 'T', 'A', 'S', 'K' }, + { 'T', 'A', 'T', 'E' }, + { 'T', 'A', 'U', 'T' }, + { 'T', 'E', 'A', 'L' }, + { 'T', 'E', 'A', 'M' }, + { 'T', 'E', 'A', 'R' }, + { 'T', 'E', 'C', 'H' }, + { 'T', 'E', 'E', 'M' }, + { 'T', 'E', 'E', 'N' }, + { 'T', 'E', 'E', 'T' }, + { 'T', 'E', 'L', 'L' }, + { 'T', 'E', 'N', 'D' }, + { 'T', 'E', 'N', 'T' }, + { 'T', 'E', 'R', 'M' }, + { 'T', 'E', 'R', 'N' }, + { 'T', 'E', 'S', 'S' }, + { 'T', 'E', 'S', 'T' }, + { 'T', 'H', 'A', 'N' }, + { 'T', 'H', 'A', 'T' }, + { 'T', 'H', 'E', 'E' }, + { 'T', 'H', 'E', 'M' }, + { 'T', 'H', 'E', 'N' }, + { 'T', 'H', 'E', 'Y' }, + { 'T', 'H', 'I', 'N' }, + { 'T', 'H', 'I', 'S' }, + { 'T', 'H', 'U', 'D' }, + { 'T', 'H', 'U', 'G' }, + { 'T', 'I', 'C', 'K' }, + { 'T', 'I', 'D', 'E' }, + { 'T', 'I', 'D', 'Y' }, + { 'T', 'I', 'E', 'D' }, + { 'T', 'I', 'E', 'R' }, + { 'T', 'I', 'L', 'E' }, + { 'T', 'I', 'L', 'L' }, + { 'T', 'I', 'L', 'T' }, + { 'T', 'I', 'M', 'E' }, + { 'T', 'I', 'N', 'A' }, + { 'T', 'I', 'N', 'E' }, + { 'T', 'I', 'N', 'T' }, + { 'T', 'I', 'N', 'Y' }, + { 'T', 'I', 'R', 'E' }, + { 'T', 'O', 'A', 'D' }, + { 'T', 'O', 'G', 'O' }, + { 'T', 'O', 'I', 'L' }, + { 'T', 'O', 'L', 'D' }, + { 'T', 'O', 'L', 'L' }, + { 'T', 'O', 'N', 'E' }, + { 'T', 'O', 'N', 'G' }, + { 'T', 'O', 'N', 'Y' }, + { 'T', 'O', 'O', 'K' }, + { 'T', 'O', 'O', 'L' }, + { 'T', 'O', 'O', 'T' }, + { 'T', 'O', 'R', 'E' }, + { 'T', 'O', 'R', 'N' }, + { 'T', 'O', 'T', 'E' }, + { 'T', 'O', 'U', 'R' }, + { 'T', 'O', 'U', 'T' }, + { 'T', 'O', 'W', 'N' }, + { 'T', 'R', 'A', 'G' }, + { 'T', 'R', 'A', 'M' }, + { 'T', 'R', 'A', 'Y' }, + { 'T', 'R', 'E', 'E' }, + { 'T', 'R', 'E', 'K' }, + { 'T', 'R', 'I', 'G' }, + { 'T', 'R', 'I', 'M' }, + { 'T', 'R', 'I', 'O' }, + { 'T', 'R', 'O', 'D' }, + { 'T', 'R', 'O', 'T' }, + { 'T', 'R', 'O', 'Y' }, + { 'T', 'R', 'U', 'E' }, + { 'T', 'U', 'B', 'A' }, + { 'T', 'U', 'B', 'E' }, + { 'T', 'U', 'C', 'K' }, + { 'T', 'U', 'F', 'T' }, + { 'T', 'U', 'N', 'A' }, + { 'T', 'U', 'N', 'E' }, + { 'T', 'U', 'N', 'G' }, + { 'T', 'U', 'R', 'F' }, + { 'T', 'U', 'R', 'N' }, + { 'T', 'U', 'S', 'K' }, + { 'T', 'W', 'I', 'G' }, + { 'T', 'W', 'I', 'N' }, + { 'T', 'W', 'I', 'T' }, + { 'U', 'L', 'A', 'N' }, + { 'U', 'N', 'I', 'T' }, + { 'U', 'R', 'G', 'E' }, + { 'U', 'S', 'E', 'D' }, + { 'U', 'S', 'E', 'R' }, + { 'U', 'S', 'E', 'S' }, + { 'U', 'T', 'A', 'H' }, + { 'V', 'A', 'I', 'L' }, + { 'V', 'A', 'I', 'N' }, + { 'V', 'A', 'L', 'E' }, + { 'V', 'A', 'R', 'Y' }, + { 'V', 'A', 'S', 'E' }, + { 'V', 'A', 'S', 'T' }, + { 'V', 'E', 'A', 'L' }, + { 'V', 'E', 'D', 'A' }, + { 'V', 'E', 'I', 'L' }, + { 'V', 'E', 'I', 'N' }, + { 'V', 'E', 'N', 'D' }, + { 'V', 'E', 'N', 'T' }, + { 'V', 'E', 'R', 'B' }, + { 'V', 'E', 'R', 'Y' }, + { 'V', 'E', 'T', 'O' }, + { 'V', 'I', 'C', 'E' }, + { 'V', 'I', 'E', 'W' }, + { 'V', 'I', 'N', 'E' }, + { 'V', 'I', 'S', 'E' }, + { 'V', 'O', 'I', 'D' }, + { 'V', 'O', 'L', 'T' }, + { 'V', 'O', 'T', 'E' }, + { 'W', 'A', 'C', 'K' }, + { 'W', 'A', 'D', 'E' }, + { 'W', 'A', 'G', 'E' }, + { 'W', 'A', 'I', 'L' }, + { 'W', 'A', 'I', 'T' }, + { 'W', 'A', 'K', 'E' }, + { 'W', 'A', 'L', 'E' }, + { 'W', 'A', 'L', 'K' }, + { 'W', 'A', 'L', 'L' }, + { 'W', 'A', 'L', 'T' }, + { 'W', 'A', 'N', 'D' }, + { 'W', 'A', 'N', 'E' }, + { 'W', 'A', 'N', 'G' }, + { 'W', 'A', 'N', 'T' }, + { 'W', 'A', 'R', 'D' }, + { 'W', 'A', 'R', 'M' }, + { 'W', 'A', 'R', 'N' }, + { 'W', 'A', 'R', 'T' }, + { 'W', 'A', 'S', 'H' }, + { 'W', 'A', 'S', 'T' }, + { 'W', 'A', 'T', 'S' }, + { 'W', 'A', 'T', 'T' }, + { 'W', 'A', 'V', 'E' }, + { 'W', 'A', 'V', 'Y' }, + { 'W', 'A', 'Y', 'S' }, + { 'W', 'E', 'A', 'K' }, + { 'W', 'E', 'A', 'L' }, + { 'W', 'E', 'A', 'N' }, + { 'W', 'E', 'A', 'R' }, + { 'W', 'E', 'E', 'D' }, + { 'W', 'E', 'E', 'K' }, + { 'W', 'E', 'I', 'R' }, + { 'W', 'E', 'L', 'D' }, + { 'W', 'E', 'L', 'L' }, + { 'W', 'E', 'L', 'T' }, + { 'W', 'E', 'N', 'T' }, + { 'W', 'E', 'R', 'E' }, + { 'W', 'E', 'R', 'T' }, + { 'W', 'E', 'S', 'T' }, + { 'W', 'H', 'A', 'M' }, + { 'W', 'H', 'A', 'T' }, + { 'W', 'H', 'E', 'E' }, + { 'W', 'H', 'E', 'N' }, + { 'W', 'H', 'E', 'T' }, + { 'W', 'H', 'O', 'A' }, + { 'W', 'H', 'O', 'M' }, + { 'W', 'I', 'C', 'K' }, + { 'W', 'I', 'F', 'E' }, + { 'W', 'I', 'L', 'D' }, + { 'W', 'I', 'L', 'L' }, + { 'W', 'I', 'N', 'D' }, + { 'W', 'I', 'N', 'E' }, + { 'W', 'I', 'N', 'G' }, + { 'W', 'I', 'N', 'K' }, + { 'W', 'I', 'N', 'O' }, + { 'W', 'I', 'R', 'E' }, + { 'W', 'I', 'S', 'E' }, + { 'W', 'I', 'S', 'H' }, + { 'W', 'I', 'T', 'H' }, + { 'W', 'O', 'L', 'F' }, + { 'W', 'O', 'N', 'T' }, + { 'W', 'O', 'O', 'D' }, + { 'W', 'O', 'O', 'L' }, + { 'W', 'O', 'R', 'D' }, + { 'W', 'O', 'R', 'E' }, + { 'W', 'O', 'R', 'K' }, + { 'W', 'O', 'R', 'M' }, + { 'W', 'O', 'R', 'N' }, + { 'W', 'O', 'V', 'E' }, + { 'W', 'R', 'I', 'T' }, + { 'W', 'Y', 'N', 'N' }, + { 'Y', 'A', 'L', 'E' }, + { 'Y', 'A', 'N', 'G' }, + { 'Y', 'A', 'N', 'K' }, + { 'Y', 'A', 'R', 'D' }, + { 'Y', 'A', 'R', 'N' }, + { 'Y', 'A', 'W', 'L' }, + { 'Y', 'A', 'W', 'N' }, + { 'Y', 'E', 'A', 'H' }, + { 'Y', 'E', 'A', 'R' }, + { 'Y', 'E', 'L', 'L' }, + { 'Y', 'O', 'G', 'A' }, + { 'Y', 'O', 'K', 'E' } +}; + +/* Extract LENGTH bits from the char array S starting with bit number + START. It always reads three consecutive octects, which means it + can read past end of data when START is at the edge of the region. */ + +static uint32_t +extract (const unsigned char *s, int start, int length) +{ + unsigned char cl = s[start / 8]; + unsigned char cc = s[start / 8 + 1]; + unsigned char cr = s[start / 8 + 2]; + uint32_t x; + x = (uint32_t)(cl << 8 | cc) << 8 | cr; + x >>= 24 - (length + (start % 8)); + x &= (0xffff >> (16 - length)); + return x; +} + +/* Length of a string known to be at least 1 and at most 4 chars + long. */ + +#define STRLEN_1_4(s) (!(s)[1] ? 1 : !(s)[2] ? 2 : !(s)[3] ? 3 : 4) + +/* Encode 8 bytes in C as a string of English words and store them to + STORE. Returns STORE. */ + +static char * +btoe (char *store, const unsigned char *c) +{ + unsigned char cp[10]; /* add in room for the parity 2 bits + + extract() slop. */ + int p, i; + char *store_beg = store; + + *store = '\0'; + + /* Workaround for extract() reads beyond end of data */ + xzero (cp); + memcpy (cp, c, 8); + + /* Compute parity and append it to CP. */ + for (p = 0, i = 0; i < 64; i += 2) + p += extract (cp, i, 2); + cp[8] = (char)p << 6; + + /* The 64 bits of input and the two parity bits comprise 66 bits of + data that are now in CP. We convert that information, 11 bits at + a time, to English words indexed from Wp. Since there are 2048 + (2^11) words in Wp, every 11-bit combination corresponds to a + distinct word. */ + memcpy (store, &Wp[extract (cp, 0, 11)][0], 4); + store += STRLEN_1_4 (store); + *store++ = ' '; + memcpy (store, &Wp[extract (cp, 11, 11)][0], 4); + store += STRLEN_1_4 (store); + *store++ = ' '; + memcpy (store, &Wp[extract (cp, 22, 11)][0], 4); + store += STRLEN_1_4 (store); + *store++ = ' '; + memcpy (store, &Wp[extract (cp, 33, 11)][0], 4); + store += STRLEN_1_4 (store); + *store++ = ' '; + memcpy (store, &Wp[extract (cp, 44, 11)][0], 4); + store += STRLEN_1_4 (store); + *store++ = ' '; + memcpy (store, &Wp[extract (cp, 55, 11)][0], 4); + store[4] = '\0'; /* make sure the string is terminated */ + + DEBUGP (("wrote %s to STORE\n", quote (store_beg))); + return store_beg; +} + +/* Calculate the SKEY response, based on the sequence, seed + (challenge), and the secret password. The calculated response is + used instead of the real password when logging in to SKEY-enabled + servers. + + The result is calculated like this: + + + Concatenate SEED and PASS and calculate the 16-byte MD5 checksum. + + + Shorten the checksum to eight bytes by folding the second eight + bytes onto the first eight using XOR. The resulting eight-byte + sequence is the key. + + + MD5-process the key, fold the checksum to eight bytes and store + it back to the key. Repeat this crunching SEQUENCE times. + (Sequence is a number that gets decremented every time the user + logs in to the server. Therefore an eavesdropper would have to + invert the hash function in order to guess the next one-time + password.) + + + Convert the resulting 64-bit key to 6 English words separated by + spaces (see btoe for details) and return the resulting ASCII + string. + + All this is described in section 6 of rfc2289 in more detail. */ + +const char * +skey_response (int sequence, const char *seed, const char *pass) +{ + unsigned char key[8]; + + /* Room to hold 6 four-letter words (heh), 5 space separators, and + the terminating \0. 24+5+1 == 30 */ + static char english[30]; + + struct md5_ctx ctx; + uint32_t checksum[4]; + + md5_init_ctx (&ctx); + md5_process_bytes ((const unsigned char *) seed, strlen (seed), &ctx); + md5_process_bytes ((const unsigned char *) pass, strlen (pass), &ctx); + md5_finish_ctx (&ctx, (unsigned char *) checksum); + checksum[0] ^= checksum[2]; + checksum[1] ^= checksum[3]; + memcpy (key, checksum, 8); + + while (sequence-- > 0) + { + md5_init_ctx (&ctx); + md5_process_bytes ((unsigned char *) key, 8, &ctx); + md5_finish_ctx (&ctx, (unsigned char *) checksum); + checksum[0] ^= checksum[2]; + checksum[1] ^= checksum[3]; + memcpy (key, checksum, 8); + } + return btoe (english, key); +} diff --git a/src/ftp.c b/src/ftp.c new file mode 100644 index 0000000..a44bf04 --- /dev/null +++ b/src/ftp.c @@ -0,0 +1,2898 @@ +/* File Transfer Protocol support. + Copyright (C) 1996-2011, 2014-2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <strings.h> +#include <unistd.h> +#include <assert.h> +#include <errno.h> +#include <time.h> + +#include "utils.h" +#include "url.h" +#include "retr.h" +#include "ftp.h" +#include "ssl.h" +#include "connect.h" +#include "host.h" +#include "netrc.h" +#include "convert.h" /* for downloaded_file */ +#include "recur.h" /* for INFINITE_RECURSION */ +#include "warc.h" +#include "c-strcase.h" +#ifdef ENABLE_XATTR +#include "xattr.h" +#endif + +#ifdef __VMS +# include "vms.h" +#endif /* def __VMS */ + + +/* File where the "ls -al" listing will be saved. */ +#ifdef MSDOS +#define LIST_FILENAME "_listing" +#else +#define LIST_FILENAME ".listing" +#endif + +typedef struct +{ + int st; /* connection status */ + int cmd; /* command code */ + int csock; /* control connection socket */ + double dltime; /* time of the download in msecs */ + enum stype rs; /* remote system reported by ftp server */ + enum ustype rsu; /* when rs is ST_UNIX, here there are more details */ + char *id; /* initial directory */ + char *target; /* target file name */ + struct url *proxy; /* FTWK-style proxy */ +} ccon; + + +/* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in + the string S, and return the number converted to wgint, if found, 0 + otherwise. */ +static wgint +ftp_expected_bytes (const char *s) +{ + wgint res; + + while (1) + { + while (*s && *s != '(') + ++s; + if (!*s) + return 0; + ++s; /* skip the '(' */ + res = str_to_wgint (s, (char **) &s, 10); + if (!*s) + return 0; + while (*s && c_isspace (*s)) + ++s; + if (!*s) + return 0; + if (c_tolower (*s) != 'b') + continue; + if (c_strncasecmp (s, "byte", 4)) + continue; + else + break; + } + return res; +} + +#ifdef ENABLE_IPV6 +/* + * This function sets up a passive data connection with the FTP server. + * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv. + */ +static uerr_t +ftp_do_pasv (int csock, ip_address *addr, int *port) +{ + uerr_t err; + + /* We need to determine the address family and need to call + getpeername, so while we're at it, store the address to ADDR. + ftp_pasv and ftp_lpsv can simply override it. */ + if (!socket_ip_address (csock, addr, ENDPOINT_PEER)) + abort (); + + /* If our control connection is over IPv6, then we first try EPSV and then + * LPSV if the former is not supported. If the control connection is over + * IPv4, we simply issue the good old PASV request. */ + switch (addr->family) + { + case AF_INET: + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> PASV ... "); + err = ftp_pasv (csock, addr, port); + break; + case AF_INET6: + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> EPSV ... "); + err = ftp_epsv (csock, addr, port); + + /* If EPSV is not supported try LPSV */ + if (err == FTPNOPASV) + { + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> LPSV ... "); + err = ftp_lpsv (csock, addr, port); + } + break; + default: + abort (); + } + + return err; +} + +/* + * This function sets up an active data connection with the FTP server. + * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port. + */ +static uerr_t +ftp_do_port (int csock, int *local_sock) +{ + uerr_t err; + ip_address cip; + + if (!socket_ip_address (csock, &cip, ENDPOINT_PEER)) + abort (); + + /* If our control connection is over IPv6, then we first try EPRT and then + * LPRT if the former is not supported. If the control connection is over + * IPv4, we simply issue the good old PORT request. */ + switch (cip.family) + { + case AF_INET: + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> PORT ... "); + err = ftp_port (csock, local_sock); + break; + case AF_INET6: + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> EPRT ... "); + err = ftp_eprt (csock, local_sock); + + /* If EPRT is not supported try LPRT */ + if (err == FTPPORTERR) + { + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> LPRT ... "); + err = ftp_lprt (csock, local_sock); + } + break; + default: + abort (); + } + return err; +} +#else + +static uerr_t +ftp_do_pasv (int csock, ip_address *addr, int *port) +{ + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> PASV ... "); + return ftp_pasv (csock, addr, port); +} + +static uerr_t +ftp_do_port (int csock, int *local_sock) +{ + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> PORT ... "); + return ftp_port (csock, local_sock); +} +#endif + +static void +print_length (wgint size, wgint start, bool authoritative) +{ + logprintf (LOG_VERBOSE, _("Length: %s"), number_to_static_string (size)); + if (size >= 1024) + logprintf (LOG_VERBOSE, " (%s)", human_readable (size, 10, 1)); + if (start > 0) + { + if (size - start >= 1024) + logprintf (LOG_VERBOSE, _(", %s (%s) remaining"), + number_to_static_string (size - start), + human_readable (size - start, 10, 1)); + else + logprintf (LOG_VERBOSE, _(", %s remaining"), + number_to_static_string (size - start)); + } + logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n"); +} + +static uerr_t ftp_get_listing (struct url *, struct url *, ccon *, struct fileinfo **); + +static uerr_t +get_ftp_greeting (int csock, ccon *con) +{ + uerr_t err = 0; + + /* Get the server's greeting */ + err = ftp_greeting (csock); + if (err != FTPOK) + { + logputs (LOG_NOTQUIET, "Error in server response. Closing.\n"); + fd_close (csock); + con->csock = -1; + } + + return err; +} + +#ifdef HAVE_SSL +static uerr_t +init_control_ssl_connection (int csock, struct url *u, bool *using_control_security) +{ + bool using_security = false; + + /* If '--ftps-implicit' was passed, perform the SSL handshake directly, + * and do not send an AUTH command. + * Otherwise send an AUTH sequence before login, + * and perform the SSL handshake if accepted by server. + */ + if (!opt.ftps_implicit && !opt.server_response) + logputs (LOG_VERBOSE, "==> AUTH TLS ... "); + if (opt.ftps_implicit || ftp_auth (csock, SCHEME_FTPS) == FTPOK) + { + if (!ssl_connect_wget (csock, u->host, NULL)) + { + fd_close (csock); + return CONSSLERR; + } + else if (!ssl_check_certificate (csock, u->host)) + { + fd_close (csock); + return VERIFCERTERR; + } + + if (!opt.ftps_implicit && !opt.server_response) + logputs (LOG_VERBOSE, " done.\n"); + + /* If implicit FTPS was requested, we act as "normal" FTP, but over SSL. + * We're not using RFC 2228 commands. + */ + using_security = true; + } + else + { + /* The server does not support 'AUTH TLS'. + * Check if --ftps-fallback-to-ftp was passed. */ + if (opt.ftps_fallback_to_ftp) + { + logputs (LOG_NOTQUIET, "Server does not support AUTH TLS. Falling back to FTP.\n"); + using_security = false; + } + else + { + fd_close (csock); + return FTPNOAUTH; + } + } + + *using_control_security = using_security; + return NOCONERROR; +} +#endif + +/* Retrieves a file with denoted parameters through opening an FTP + connection to the server. It always closes the data connection, + and closes the control connection in case of error. If warc_tmp + is non-NULL, the downloaded data will be written there as well. */ +static uerr_t +getftp (struct url *u, struct url *original_url, + wgint passed_expected_bytes, wgint *qtyread, + wgint restval, ccon *con, int count, wgint *last_expected_bytes, + FILE *warc_tmp) +{ + int csock, dtsock, local_sock, res; + uerr_t err = RETROK; /* appease the compiler */ + FILE *fp = NULL; + char *respline, *tms; + const char *user, *passwd, *tmrate; + int cmd = con->cmd; + wgint expected_bytes = 0; + bool got_expected_bytes = false; + bool rest_failed = false; + int flags; + wgint rd_size, previous_rd_size = 0; + char type_char; + bool try_again; + bool list_a_used = false; +#ifdef HAVE_SSL + enum prot_level prot = (opt.ftps_clear_data_connection ? PROT_CLEAR : PROT_PRIVATE); + /* these variables tell whether the target server + * accepts the security extensions (RFC 2228) or not, + * and whether we're actually using any of them + * (encryption at the control connection only, + * or both at control and data connections) */ + bool using_control_security = false, using_data_security = false; +#endif + + assert (con != NULL); + assert (con->target != NULL); + + /* Debug-check of the sanity of the request by making sure that LIST + and RETR are never both requested (since we can handle only one + at a time. */ + assert (!((cmd & DO_LIST) && (cmd & DO_RETR))); + /* Make sure that at least *something* is requested. */ + assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0); + + *qtyread = restval; + + /* Find the username with priority */ + if (u->user) + user = u->user; + else if (opt.user && (opt.use_askpass || opt.ask_passwd)) + user = opt.user; + else if (opt.ftp_user) + user = opt.ftp_user; + else if (opt.user) + user = opt.user; + else + user = NULL; + + /* Find the password with priority */ + if (u->passwd) + passwd = u->passwd; + else if (opt.passwd && (opt.use_askpass || opt.ask_passwd)) + passwd = opt.passwd; + else if (opt.ftp_passwd) + passwd = opt.ftp_passwd; + else if (opt.passwd) + passwd = opt.passwd; + else + passwd = NULL; + + /* Check for ~/.netrc if none of the above match */ + if (opt.netrc && (!user || !passwd)) + search_netrc (u->host, (const char **) &user, (const char **) &passwd, 1, NULL); + + if (!user) user = "anonymous"; + if (!passwd) passwd = "-wget@"; + + dtsock = -1; + local_sock = -1; + con->dltime = 0; + +#ifdef HAVE_SSL + if (u->scheme == SCHEME_FTPS) + { + /* Initialize SSL layer first */ + if (!ssl_init ()) + { + scheme_disable (SCHEME_FTPS); + logprintf (LOG_NOTQUIET, _("Could not initialize SSL. It will be disabled.\n")); + err = SSLINITFAILED; + return err; + } + + /* If we're using the default FTP port and implicit FTPS was requested, + * rewrite the port to the default *implicit* FTPS port. + */ + if (opt.ftps_implicit && u->port == DEFAULT_FTP_PORT) + { + DEBUGP (("Implicit FTPS was specified. Rewriting default port to %d.\n", DEFAULT_FTPS_IMPLICIT_PORT)); + u->port = DEFAULT_FTPS_IMPLICIT_PORT; + } + } +#endif + + if (!(cmd & DO_LOGIN)) + { + csock = con->csock; +#ifdef HAVE_SSL + using_data_security = con->st & DATA_CHANNEL_SECURITY; +#endif + } + else /* cmd & DO_LOGIN */ + { + char *host = con->proxy ? con->proxy->host : u->host; + int port = con->proxy ? con->proxy->port : u->port; + + /* Login to the server: */ + + /* First: Establish the control connection. */ + + csock = connect_to_host (host, port); + if (csock == E_HOST) + return HOSTERR; + else if (csock < 0) + return (retryable_socket_connect_error (errno) + ? CONERROR : CONIMPOSSIBLE); + + if (cmd & LEAVE_PENDING) + con->csock = csock; + else + con->csock = -1; + +#ifdef HAVE_SSL + if (u->scheme == SCHEME_FTPS) + { + /* If we're in implicit FTPS mode, we have to set up SSL/TLS before everything else. + * Otherwise we first read the server's greeting, and then send an "AUTH TLS". + */ + if (opt.ftps_implicit) + { + err = init_control_ssl_connection (csock, u, &using_control_security); + if (err != NOCONERROR) + return err; + err = get_ftp_greeting (csock, con); + if (err != FTPOK) + return err; + } + else + { + err = get_ftp_greeting (csock, con); + if (err != FTPOK) + return err; + err = init_control_ssl_connection (csock, u, &using_control_security); + if (err != NOCONERROR) + return err; + } + } + else + { + err = get_ftp_greeting (csock, con); + if (err != FTPOK) + return err; + } +#else + err = get_ftp_greeting (csock, con); + if (err != FTPOK) + return err; +#endif + + /* Second: Login with proper USER/PASS sequence. */ + logprintf (LOG_VERBOSE, _("Logging in as %s ... "), + quotearg_style (escape_quoting_style, user)); + if (opt.server_response) + logputs (LOG_ALWAYS, "\n"); + if (con->proxy) + { + /* If proxy is in use, log in as username@target-site. */ + char *logname = concat_strings (user, "@", u->host, (char *) 0); + err = ftp_login (csock, logname, passwd); + xfree (logname); + } + else + err = ftp_login (csock, user, passwd); + + /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */ + switch (err) + { + case FTPRERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + return err; + case FTPSRVERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("Error in server greeting.\n")); + fd_close (csock); + con->csock = -1; + return err; + case WRITEFAILED: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, + _("Write failed, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + return err; + case FTPLOGREFUSED: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("The server refuses login.\n")); + fd_close (csock); + con->csock = -1; + return FTPLOGREFUSED; + case FTPLOGINC: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("Login incorrect.\n")); + fd_close (csock); + con->csock = -1; + return FTPLOGINC; + case FTPOK: + if (!opt.server_response) + logputs (LOG_VERBOSE, _("Logged in!\n")); + break; + default: + abort (); + } + +#ifdef HAVE_SSL + if (using_control_security) + { + /* Send the PBSZ and PROT commands, in that order. + * If we are here it means that the server has already accepted + * some form of FTPS. Thus, these commands must work. + * If they don't work, that's an error. There's no sense in honoring + * --ftps-fallback-to-ftp or similar options. */ + if (u->scheme == SCHEME_FTPS) + { + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> PBSZ 0 ... "); + if ((err = ftp_pbsz (csock, 0)) == FTPNOPBSZ) + { + logputs (LOG_NOTQUIET, _("Server did not accept the 'PBSZ 0' command.\n")); + return err; + } + if (!opt.server_response) + logputs (LOG_VERBOSE, "done."); + + if (!opt.server_response) + logprintf (LOG_VERBOSE, " ==> PROT %c ... ", (int) prot); + if ((err = ftp_prot (csock, prot)) == FTPNOPROT) + { + logprintf (LOG_NOTQUIET, _("Server did not accept the 'PROT %c' command.\n"), (int) prot); + return err; + } + if (!opt.server_response) + logputs (LOG_VERBOSE, "done.\n"); + + if (prot != PROT_CLEAR) + { + using_data_security = true; + con->st |= DATA_CHANNEL_SECURITY; + } + } + } +#endif + + /* Third: Get the system type */ + if (!opt.server_response) + logprintf (LOG_VERBOSE, "==> SYST ... "); + err = ftp_syst (csock, &con->rs, &con->rsu); + /* FTPRERR */ + switch (err) + { + case FTPRERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + return err; + case FTPSRVERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, + _("Server error, can't determine system type.\n")); + break; + case FTPOK: + /* Everything is OK. */ + break; + default: + abort (); + } + if (!opt.server_response && err != FTPSRVERR) + logputs (LOG_VERBOSE, _("done. ")); + + /* 2013-10-17 Andrea Urbani (matfanjol) + According to the system type I choose which + list command will be used. + If I don't know that system, I will try, the + first time of each session, "LIST -a" and + "LIST". (see __LIST_A_EXPLANATION__ below) */ + switch (con->rs) + { + case ST_VMS: + /* About ST_VMS there is an old note: + 2008-01-29 SMS. For a VMS FTP server, where "LIST -a" may not + fail, but will never do what is desired here, + skip directly to the simple "LIST" command + (assumed to be the last one in the list). */ + DEBUGP (("\nVMS: I know it and I will use \"LIST\" as standard list command\n")); + con->st |= LIST_AFTER_LIST_A_CHECK_DONE; + con->st |= AVOID_LIST_A; + break; + case ST_UNIX: + if (con->rsu == UST_MULTINET) + { + DEBUGP (("\nUNIX MultiNet: I know it and I will use \"LIST\" " + "as standard list command\n")); + con->st |= LIST_AFTER_LIST_A_CHECK_DONE; + con->st |= AVOID_LIST_A; + } + else if (con->rsu == UST_TYPE_L8) + { + DEBUGP (("\nUNIX TYPE L8: I know it and I will use \"LIST -a\" " + "as standard list command\n")); + con->st |= LIST_AFTER_LIST_A_CHECK_DONE; + con->st |= AVOID_LIST; + } + break; + default: + break; + } + + /* Fourth: Find the initial ftp directory */ + + if (!opt.server_response) + logprintf (LOG_VERBOSE, "==> PWD ... "); + err = ftp_pwd (csock, &con->id); + /* FTPRERR */ + switch (err) + { + case FTPRERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + return err; + case FTPSRVERR : + /* PWD unsupported -- assume "/". */ + xfree (con->id); + con->id = xstrdup ("/"); + break; + case FTPOK: + /* Everything is OK. */ + break; + default: + abort (); + } + +#if 0 + /* 2004-09-17 SMS. + Don't help me out. Please. + A reasonably recent VMS FTP server will cope just fine with + UNIX file specifications. This code just spoils things. + Discarding the device name, for example, is not a wise move. + This code was disabled but left in as an example of what not + to do. + */ + + /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]". + Convert it to "/INITIAL/FOLDER" */ + if (con->rs == ST_VMS) + { + char *path = strchr (con->id, '['); + char *pathend = path ? strchr (path + 1, ']') : NULL; + if (!path || !pathend) + DEBUGP (("Initial VMS directory not in the form [...]!\n")); + else + { + char *idir = con->id; + DEBUGP (("Preprocessing the initial VMS directory\n")); + DEBUGP ((" old = '%s'\n", con->id)); + /* We do the conversion in-place by copying the stuff + between [ and ] to the beginning, and changing dots + to slashes at the same time. */ + *idir++ = '/'; + for (++path; path < pathend; path++, idir++) + *idir = *path == '.' ? '/' : *path; + *idir = '\0'; + DEBUGP ((" new = '%s'\n\n", con->id)); + } + } +#endif /* 0 */ + + if (!opt.server_response) + logputs (LOG_VERBOSE, _("done.\n")); + + /* Fifth: Set the FTP type. */ + type_char = ftp_process_type (u->params); + if (!opt.server_response) + logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char); + err = ftp_type (csock, type_char); + /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */ + switch (err) + { + case FTPRERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + return err; + case WRITEFAILED: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, + _("Write failed, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + return err; + case FTPUNKNOWNTYPE: + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, + _("Unknown type `%c', closing control connection.\n"), + type_char); + fd_close (csock); + con->csock = -1; + return err; + case FTPOK: + /* Everything is OK. */ + break; + default: + abort (); + } + if (!opt.server_response) + logputs (LOG_VERBOSE, _("done. ")); + } /* do login */ + + if (cmd & DO_CWD) + { + if (!*u->dir) + logputs (LOG_VERBOSE, _("==> CWD not needed.\n")); + else + { + const char *targ = NULL; + char *target = u->dir; + char targetbuf[1024]; + int cwd_count; + int cwd_end; + int cwd_start; + + DEBUGP (("changing working directory\n")); + + /* Change working directory. To change to a non-absolute + Unix directory, we need to prepend initial directory + (con->id) to it. Absolute directories "just work". + + A relative directory is one that does not begin with '/' + and, on non-Unix OS'es, one that doesn't begin with + "[a-z]:". + + This is not done for OS400, which doesn't use + "/"-delimited directories, nor does it support directory + hierarchies. "CWD foo" followed by "CWD bar" leaves us + in "bar", not in "foo/bar", as would be customary + elsewhere. */ + + /* 2004-09-20 SMS. + Why is this wise even on UNIX? It certainly fouls VMS. + See below for a more reliable, more universal method. + */ + + /* 2008-04-22 MJC. + I'm not crazy about it either. I'm informed it's useful + for misconfigured servers that have some dirs in the path + with +x but -r, but this method is not RFC-conformant. I + understand the need to deal with crappy server + configurations, but it's far better to use the canonical + method first, and fall back to kludges second. + */ + + if (target[0] != '/' + && !(con->rs != ST_UNIX + && c_isalpha (target[0]) + && target[1] == ':') + && (con->rs != ST_OS400) + && (con->rs != ST_VMS)) + { + char *ntarget, *p; + size_t idlen = strlen (con->id); + size_t len; + + /* Strip trailing slash(es) from con->id. */ + while (idlen > 0 && con->id[idlen - 1] == '/') + --idlen; + + len = idlen + 1 + strlen (target); + if (len < sizeof (targetbuf)) + p = ntarget = targetbuf; + else + p = ntarget = xmalloc (len + 1); + + memcpy (p, con->id, idlen); + p += idlen; + *p++ = '/'; + strcpy (p, target); + + DEBUGP (("Prepended initial PWD to relative path:\n")); + DEBUGP ((" pwd: '%s'\n old: '%s'\n new: '%s'\n", + con->id, target, ntarget)); + target = ntarget; + } + +#if 0 + /* 2004-09-17 SMS. + Don't help me out. Please. + A reasonably recent VMS FTP server will cope just fine with + UNIX file specifications. This code just spoils things. + Discarding the device name, for example, is not a wise + move. + This code was disabled but left in as an example of what + not to do. + */ + + /* If the FTP host runs VMS, we will have to convert the absolute + directory path in UNIX notation to absolute directory path in + VMS notation as VMS FTP servers do not like UNIX notation of + absolute paths. "VMS notation" is [dir.subdir.subsubdir]. */ + + if (con->rs == ST_VMS) + { + char *tmpp; + char *ntarget = (char *)alloca (strlen (target) + 2); + /* We use a converted initial dir, so directories in + TARGET will be separated with slashes, something like + "/INITIAL/FOLDER/DIR/SUBDIR". Convert that to + "[INITIAL.FOLDER.DIR.SUBDIR]". */ + strcpy (ntarget, target); + assert (*ntarget == '/'); + *ntarget = '['; + for (tmpp = ntarget + 1; *tmpp; tmpp++) + if (*tmpp == '/') + *tmpp = '.'; + *tmpp++ = ']'; + *tmpp = '\0'; + DEBUGP (("Changed file name to VMS syntax:\n")); + DEBUGP ((" Unix: '%s'\n VMS: '%s'\n", target, ntarget)); + target = ntarget; + } +#endif /* 0 */ + + /* 2004-09-20 SMS. + A relative directory is relative to the initial directory. + Thus, what _is_ useful on VMS (and probably elsewhere) is + to CWD to the initial directory (ideally, whatever the + server reports, _exactly_, NOT badly UNIX-ixed), and then + CWD to the (new) relative directory. This should probably + be restructured as a function, called once or twice, but + I'm lazy enough to take the badly indented loop short-cut + for now. + */ + + /* Decide on one pass (absolute) or two (relative). + The VMS restriction may be relaxed when the squirrely code + above is reformed. + */ + if ((con->rs == ST_VMS) && (target[0] != '/')) + { + cwd_start = 0; + DEBUGP (("Using two-step CWD for relative path.\n")); + } + else + { + /* Go straight to the target. */ + cwd_start = 1; + } + + /* At least one VMS FTP server (TCPware V5.6-2) can switch to + a UNIX emulation mode when given a UNIX-like directory + specification (like "a/b/c"). If allowed to continue this + way, LIST interpretation will be confused, because the + system type (SYST response) will not be re-checked, and + future UNIX-format directory listings (for multiple URLs or + "-r") will be horribly misinterpreted. + + The cheap and nasty work-around is to do a "CWD []" after a + UNIX-like directory specification is used. (A single-level + directory is harmless.) This puts the TCPware server back + into VMS mode, and does no harm on other servers. + + Unlike the rest of this block, this particular behavior + _is_ VMS-specific, so it gets its own VMS test. + */ + if ((con->rs == ST_VMS) && (strchr (target, '/') != NULL)) + { + cwd_end = 3; + DEBUGP (("Using extra \"CWD []\" step for VMS server.\n")); + } + else + { + cwd_end = 2; + } + + /* 2004-09-20 SMS. */ + /* Sorry about the deviant indenting. Laziness. */ + + for (cwd_count = cwd_start; cwd_count < cwd_end; cwd_count++) + { + switch (cwd_count) + { + case 0: + /* Step one (optional): Go to the initial directory, + exactly as reported by the server. + */ + targ = con->id; + break; + + case 1: + /* Step two: Go to the target directory. (Absolute or + relative will work now.) + */ + targ = target; + break; + + case 2: + /* Step three (optional): "CWD []" to restore server + VMS-ness. + */ + targ = "[]"; + break; + + default: + logprintf (LOG_ALWAYS, _("Logically impossible section reached in getftp()")); + logprintf (LOG_ALWAYS, _("cwd_count: %d\ncwd_start: %d\ncwd_end: %d\n"), + cwd_count, cwd_start, cwd_end); + abort (); + } + + if (!opt.server_response) + logprintf (LOG_VERBOSE, "==> CWD (%d) %s ... ", cwd_count, + quotearg_style (escape_quoting_style, target)); + + err = ftp_cwd (csock, targ); + + /* FTPRERR, WRITEFAILED, FTPNSFOD */ + switch (err) + { + case FTPRERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + return err; + case WRITEFAILED: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, + _("Write failed, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + return err; + case FTPNSFOD: + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("No such directory %s.\n\n"), + quote (u->dir)); + fd_close (csock); + con->csock = -1; + return err; + case FTPOK: + break; + default: + abort (); + } + + if (!opt.server_response) + logputs (LOG_VERBOSE, _("done.\n")); + + } /* for */ + + /* 2004-09-20 SMS. */ + + if (target != targetbuf) + xfree (target); + + } /* else */ + } + else /* do not CWD */ + logputs (LOG_VERBOSE, _("==> CWD not required.\n")); + + if ((cmd & DO_RETR) && passed_expected_bytes == 0) + { + if (opt.verbose) + { + if (!opt.server_response) + logprintf (LOG_VERBOSE, "==> SIZE %s ... ", + quotearg_style (escape_quoting_style, u->file)); + } + + err = ftp_size (csock, u->file, &expected_bytes); + /* FTPRERR */ + switch (err) + { + case FTPRERR: + case FTPSRVERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + return err; + case FTPOK: + got_expected_bytes = true; + /* Everything is OK. */ + break; + default: + abort (); + } + if (!opt.server_response) + { + logprintf (LOG_VERBOSE, "%s\n", + expected_bytes ? + number_to_static_string (expected_bytes) : + _("done.\n")); + } + } + + if (cmd & DO_RETR && restval > 0 && restval == expected_bytes) + { + /* Server confirms that file has length restval. We should stop now. + Some servers (f.e. NcFTPd) return error when receive REST 0 */ + logputs (LOG_VERBOSE, _("File has already been retrieved.\n")); + fd_close (csock); + con->csock = -1; + return RETRFINISHED; + } + + do + { + try_again = false; + /* If anything is to be retrieved, PORT (or PASV) must be sent. */ + if (cmd & (DO_LIST | DO_RETR)) + { + if (opt.ftp_pasv) + { + ip_address passive_addr; + int passive_port; + err = ftp_do_pasv (csock, &passive_addr, &passive_port); + /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */ + switch (err) + { + case FTPRERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + return err; + case WRITEFAILED: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, + _("Write failed, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + return err; + case FTPNOPASV: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n")); + break; + case FTPINVPASV: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n")); + break; + case FTPOK: + break; + default: + abort (); + } /* switch (err) */ + if (err==FTPOK) + { + DEBUGP (("trying to connect to %s port %d\n", + print_address (&passive_addr), passive_port)); + dtsock = connect_to_ip (&passive_addr, passive_port, NULL); + if (dtsock < 0) + { + int save_errno = errno; + fd_close (csock); + con->csock = -1; + logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"), + print_address (&passive_addr), passive_port, + strerror (save_errno)); + return (retryable_socket_connect_error (save_errno) + ? CONERROR : CONIMPOSSIBLE); + } + + if (!opt.server_response) + logputs (LOG_VERBOSE, _("done. ")); + } + else + return err; + + /* + * We do not want to fall back from PASSIVE mode to ACTIVE mode ! + * The reason is the PORT command exposes the client's real IP address + * to the server. Bad for someone who relies on privacy via a ftp proxy. + */ + } + else + { + err = ftp_do_port (csock, &local_sock); + /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR, + FTPPORTERR */ + switch (err) + { + case FTPRERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return err; + case WRITEFAILED: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, + _("Write failed, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return err; + case CONSOCKERR: + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno)); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return err; + case FTPSYSERR: + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"), + strerror (errno)); + fd_close (dtsock); + return err; + case FTPPORTERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("Invalid PORT.\n")); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return err; + case FTPOK: + break; + default: + abort (); + } /* port switch */ + if (!opt.server_response) + logputs (LOG_VERBOSE, _("done. ")); + } /* dtsock == -1 */ + } /* cmd & (DO_LIST | DO_RETR) */ + + /* Restart if needed. */ + if (restval && (cmd & DO_RETR)) + { + if (!opt.server_response) + logprintf (LOG_VERBOSE, "==> REST %s ... ", + number_to_static_string (restval)); + err = ftp_rest (csock, restval); + + /* FTPRERR, WRITEFAILED, FTPRESTFAIL */ + switch (err) + { + case FTPRERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return err; + case WRITEFAILED: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, + _("Write failed, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return err; + case FTPRESTFAIL: + logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n")); + rest_failed = true; + break; + case FTPOK: + break; + default: + abort (); + } + if (err != FTPRESTFAIL && !opt.server_response) + logputs (LOG_VERBOSE, _("done. ")); + } /* restval && cmd & DO_RETR */ + + if (cmd & DO_RETR) + { + /* If we're in spider mode, don't really retrieve anything except + the directory listing and verify whether the given "file" exists. */ + if (opt.spider) + { + bool exists = false; + bool all_exist = true; + struct fileinfo *f; + uerr_t _res = ftp_get_listing (u, original_url, con, &f); + /* Set the DO_RETR command flag again, because it gets unset when + calling ftp_get_listing() and would otherwise cause an assertion + failure earlier on when this function gets repeatedly called + (e.g., when recursing). */ + con->cmd |= DO_RETR; + if (_res == RETROK) + { + while (f) + { + if (!strcmp (f->name, u->file)) + { + exists = true; + break; + } else { + all_exist = false; + } + f = f->next; + } + if (exists) + { + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("File %s exists.\n"), + quote (u->file)); + } + else + { + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("No such file %s.\n"), + quote (u->file)); + } + } + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + if (all_exist) { + return RETRFINISHED; + } else { + return FTPNSFOD; + } + } + + if (opt.verbose) + { + if (!opt.server_response) + { + if (restval) + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_VERBOSE, "==> RETR %s ... ", + quotearg_style (escape_quoting_style, u->file)); + } + } + + err = ftp_retr (csock, u->file); + /* FTPRERR, WRITEFAILED, FTPNSFOD */ + switch (err) + { + case FTPRERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return err; + case WRITEFAILED: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, + _("Write failed, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return err; + case FTPNSFOD: + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("No such file %s.\n\n"), + quote (u->file)); + fd_close (dtsock); + fd_close (local_sock); + return err; + case FTPOK: + break; + default: + abort (); + } + + if (!opt.server_response) + logputs (LOG_VERBOSE, _("done.\n")); + + if (! got_expected_bytes) + expected_bytes = *last_expected_bytes; + } /* do retrieve */ + + if (cmd & DO_LIST) + { + if (!opt.server_response) + logputs (LOG_VERBOSE, "==> LIST ... "); + /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST' + without arguments is better than `LIST .'; confirmed by + RFC959. */ + err = ftp_list (csock, NULL, con->st&AVOID_LIST_A, con->st&AVOID_LIST, &list_a_used); + + /* FTPRERR, WRITEFAILED */ + switch (err) + { + case FTPRERR: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, _("\ +Error in server response, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return err; + case WRITEFAILED: + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_NOTQUIET, + _("Write failed, closing control connection.\n")); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return err; + case FTPNSFOD: + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("No such file or directory %s.\n\n"), + quote (".")); + fd_close (dtsock); + fd_close (local_sock); + return err; + case FTPOK: + break; + default: + abort (); + } + if (!opt.server_response) + logputs (LOG_VERBOSE, _("done.\n")); + + if (! got_expected_bytes) + expected_bytes = *last_expected_bytes; + } /* cmd & DO_LIST */ + + if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST))) + return RETRFINISHED; + + /* Some FTP servers return the total length of file after REST + command, others just return the remaining size. */ + if (passed_expected_bytes && restval && expected_bytes + && (expected_bytes == passed_expected_bytes - restval)) + { + DEBUGP (("Lying FTP server found, adjusting.\n")); + expected_bytes = passed_expected_bytes; + } + + /* If no transmission was required, then everything is OK. */ + if (!opt.ftp_pasv) /* we are not using passive mode so we need + to accept */ + { + /* Wait for the server to connect to the address we're waiting + at. */ + dtsock = accept_connection (local_sock); + if (dtsock < 0) + { + logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno)); + return CONERROR; + } + } + + /* Open the file -- if output_stream is set, use it instead. */ + + /* 2005-04-17 SMS. + Note that having the output_stream ("-O") file opened in main + (main.c) rather limits the ability in VMS to open the file + differently for ASCII versus binary FTP here. (Of course, doing it + there allows a open failure to be detected immediately, without first + connecting to the server.) + */ + if (!output_stream || con->cmd & DO_LIST) + { +/* On VMS, alter the name as required. */ +#ifdef __VMS + char *targ; + + targ = ods_conform (con->target); + if (targ != con->target) + { + xfree (con->target); + con->target = targ; + } +#endif /* def __VMS */ + + mkalldirs (con->target); + if (opt.backups) + rotate_backups (con->target); + +/* 2005-04-15 SMS. + For VMS, define common fopen() optional arguments, and a handy macro + for use as a variable "binary" flag. + Elsewhere, define a constant "binary" flag. + Isn't it nice to have distinct text and binary file types? +*/ +/* 2011-09-30 SMS. + Added listing files to the set of non-"binary" (text, Stream_LF) + files. (Wget works either way, but other programs, like, say, text + editors, work better on listing files which have text attributes.) + Now we use "binary" attributes for a binary ("IMAGE") transfer, + unless "--ftp-stmlf" was specified, and we always use non-"binary" + (text, Stream_LF) attributes for a listing file, or for an ASCII + transfer. + Tidied the VMS-specific BIN_TYPE_xxx macros, and changed the call to + fopen_excl() (restored?) to use BIN_TYPE_FILE instead of "true". +*/ +#ifdef __VMS +# define BIN_TYPE_TRANSFER (type_char != 'A') +# define BIN_TYPE_FILE \ + ((!(cmd & DO_LIST)) && BIN_TYPE_TRANSFER && (opt.ftp_stmlf == 0)) +# define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id +# define FOPEN_OPT_ARGS_BIN "ctx=bin,stm", "rfm=fix", "mrs=512" FOPEN_OPT_ARGS +#else /* def __VMS */ +# define BIN_TYPE_FILE true +#endif /* def __VMS [else] */ + + if (restval && !(con->cmd & DO_LIST)) + { +#ifdef __VMS + int open_id; + + if (BIN_TYPE_FILE) + { + open_id = 3; + fp = fopen (con->target, "ab", FOPEN_OPT_ARGS_BIN); + } + else + { + open_id = 4; + fp = fopen (con->target, "a", FOPEN_OPT_ARGS); + } +#else /* def __VMS */ + fp = fopen (con->target, "ab"); +#endif /* def __VMS [else] */ + } + else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct + || opt.output_document || count > 0) + { + if (opt.unlink_requested && file_exists_p (con->target, NULL)) + { + if (unlink (con->target) < 0) + { + logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, + strerror (errno)); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return UNLINKERR; + } + } + +#ifdef __VMS + int open_id; + + if (BIN_TYPE_FILE) + { + open_id = 5; + fp = fopen (con->target, "wb", FOPEN_OPT_ARGS_BIN); + } + else + { + open_id = 6; + fp = fopen (con->target, "w", FOPEN_OPT_ARGS); + } +#else /* def __VMS */ + fp = fopen (con->target, "wb"); +#endif /* def __VMS [else] */ + } + else + { + fp = fopen_excl (con->target, BIN_TYPE_FILE); + if (!fp && errno == EEXIST) + { + /* We cannot just invent a new name and use it (which is + what functions like unique_create typically do) + because we told the user we'd use this name. + Instead, return and retry the download. */ + logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"), + con->target); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return FOPEN_EXCL_ERR; + } + } + if (!fp) + { + logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno)); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + fd_close (local_sock); + return FOPENERR; + } + } + else + fp = output_stream; + + if (passed_expected_bytes) + { + print_length (passed_expected_bytes, restval, true); + expected_bytes = passed_expected_bytes; + /* for fd_read_body's progress bar */ + } + else if (expected_bytes) + print_length (expected_bytes, restval, false); + +#ifdef HAVE_SSL + if (u->scheme == SCHEME_FTPS && using_data_security) + { + /* We should try to restore the existing SSL session in the data connection + * and fall back to establishing a new session if the server doesn't want to restore it. + */ + if (!opt.ftps_resume_ssl || !ssl_connect_wget (dtsock, u->host, &csock)) + { + if (opt.ftps_resume_ssl) + logputs (LOG_NOTQUIET, "Server does not want to resume the SSL session. Trying with a new one.\n"); + if (!ssl_connect_wget (dtsock, u->host, NULL)) + { + fd_close (csock); + fd_close (dtsock); + err = CONERROR; + logputs (LOG_NOTQUIET, "Could not perform SSL handshake.\n"); + goto exit_error; + } + } + else + logputs (LOG_NOTQUIET, "Resuming SSL session in data connection.\n"); + + if (!ssl_check_certificate (dtsock, u->host)) + { + fd_close (csock); + fd_close (dtsock); + err = CONERROR; + goto exit_error; + } + } +#endif + + /* Get the contents of the document. */ + flags = 0; + if (restval && rest_failed) + flags |= rb_skip_startpos; + rd_size = 0; + res = fd_read_body (con->target, dtsock, fp, + expected_bytes ? expected_bytes - restval : 0, + restval, &rd_size, qtyread, &con->dltime, flags, warc_tmp); + + tms = datetime_str (time (NULL)); + tmrate = retr_rate (rd_size, con->dltime); + total_download_time += con->dltime; + +#ifdef ENABLE_XATTR + if (opt.enable_xattr) + set_file_metadata (u, NULL, fp); +#endif + + fd_close (local_sock); + /* Close the local file. */ + if (!output_stream || con->cmd & DO_LIST) + fclose (fp); + + /* If fd_read_body couldn't write to fp or warc_tmp, bail out. */ + if (res == -2 || (warc_tmp != NULL && res == -3)) + { + logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"), + con->target, strerror (errno)); + fd_close (csock); + con->csock = -1; + fd_close (dtsock); + if (res == -2) + return FWRITEERR; + else if (res == -3) + return WARC_TMP_FWRITEERR; + } + else if (res == -1) + { + logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "), + tms, tmrate, fd_errstr (dtsock)); + if (opt.server_response) + logputs (LOG_ALWAYS, "\n"); + } + fd_close (dtsock); + + /* Get the server to tell us if everything is retrieved. */ + err = ftp_response (csock, &respline); + if (err != FTPOK) + { + /* The control connection is decidedly closed. Print the time + only if it hasn't already been printed. */ + if (res != -1) + logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate); + logputs (LOG_NOTQUIET, _("Control connection closed.\n")); + /* If there is an error on the control connection, close it, but + return FTPRETRINT, since there is a possibility that the + whole file was retrieved nevertheless (but that is for + ftp_loop_internal to decide). */ + fd_close (csock); + con->csock = -1; + return FTPRETRINT; + } /* err != FTPOK */ + *last_expected_bytes = ftp_expected_bytes (respline); + /* If retrieval failed for any reason, return FTPRETRINT, but do not + close socket, since the control connection is still alive. If + there is something wrong with the control connection, it will + become apparent later. */ + if (*respline != '2') + { + if (res != -1) + logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate); + logputs (LOG_NOTQUIET, _("Data transfer aborted.\n")); +#ifdef HAVE_SSL + if (!c_strncasecmp (respline, "425", 3) && u->scheme == SCHEME_FTPS) + { + logputs (LOG_NOTQUIET, "FTPS server rejects new SSL sessions in the data connection.\n"); + xfree (respline); + return FTPRESTFAIL; + } +#endif + xfree (respline); + return FTPRETRINT; + } + xfree (respline); + + if (res == -1) + { + /* What now? The data connection was erroneous, whereas the + response says everything is OK. We shall play it safe. */ + return FTPRETRINT; + } + + if (!(cmd & LEAVE_PENDING)) + { + /* Closing the socket is faster than sending 'QUIT' and the + effect is the same. */ + fd_close (csock); + con->csock = -1; + } + /* If it was a listing, and opt.server_response is true, + print it out. */ + if (con->cmd & DO_LIST) + { + if (opt.server_response) + { +/* 2005-02-25 SMS. + Much of this work may already have been done, but repeating it should + do no damage beyond wasting time. +*/ +/* On VMS, alter the name as required. */ +#ifdef __VMS + char *targ; + + targ = ods_conform (con->target); + if (targ != con->target) + { + xfree (con->target); + con->target = targ; + } +#endif /* def __VMS */ + + mkalldirs (con->target); + fp = fopen (con->target, "r"); + if (!fp) + logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno)); + else + { + char *line = NULL; + size_t bufsize = 0; + ssize_t len; + + /* The lines are being read with getline because of + no-buffering on opt.lfile. */ + while ((len = getline (&line, &bufsize, fp)) > 0) + { + while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) + line[--len] = '\0'; + logprintf (LOG_ALWAYS, "%s\n", + quotearg_style (escape_quoting_style, line)); + } + xfree (line); + fclose (fp); + } + } /* server_response */ + + /* 2013-10-17 Andrea Urbani (matfanjol) + < __LIST_A_EXPLANATION__ > + After the SYST command, looks if it knows that system. + If yes, wget will force the use of "LIST" or "LIST -a". + If no, wget will try, only the first time of each session, before the + "LIST -a" command and after the "LIST". + If "LIST -a" works and returns more or equal data of the "LIST", + "LIST -a" will be the standard list command for all the session. + If "LIST -a" fails or returns less data than "LIST" (think on the case + of an existing file called "-a"), "LIST" will be the standard list + command for all the session. + ("LIST -a" is used to get also the hidden files) + + */ + if (!(con->st & LIST_AFTER_LIST_A_CHECK_DONE)) + { + /* We still have to check "LIST" after the first "LIST -a" to see + if with "LIST" we get more data than "LIST -a", that means + "LIST -a" returned files/folders with "-a" name. */ + if (con->st & AVOID_LIST_A) + { + /* LIST was used in this cycle. + Let's see the result. */ + if (rd_size > previous_rd_size) + { + /* LIST returns more data than "LIST -a". + "LIST" is the official command to use. */ + con->st |= LIST_AFTER_LIST_A_CHECK_DONE; + DEBUGP (("LIST returned more data than \"LIST -a\": " + "I will use \"LIST\" as standard list command\n")); + } + else if (previous_rd_size > rd_size) + { + /* "LIST -a" returned more data then LIST. + "LIST -a" is the official command to use. */ + con->st |= LIST_AFTER_LIST_A_CHECK_DONE; + con->st |= AVOID_LIST; + con->st &= ~AVOID_LIST_A; + /* Sorry, please, download again the "LIST -a"... */ + try_again = true; + DEBUGP (("LIST returned less data than \"LIST -a\": I will " + "use \"LIST -a\" as standard list command\n")); + } + else + { + /* LIST and "LIST -a" return the same data. */ + if (rd_size == 0) + { + /* Same empty data. We will check both again because + we cannot check if "LIST -a" has returned an empty + folder instead of a folder content. */ + con->st &= ~AVOID_LIST_A; + } + else + { + /* Same data, so, better to take "LIST -a" that + shows also hidden files/folders (when present) */ + con->st |= LIST_AFTER_LIST_A_CHECK_DONE; + con->st |= AVOID_LIST; + con->st &= ~AVOID_LIST_A; + DEBUGP (("LIST returned the same amount of data of " + "\"LIST -a\": I will use \"LIST -a\" as standard " + "list command\n")); + } + } + } + else + { + /* In this cycle "LIST -a" should being used. Is it true? */ + if (list_a_used) + { + /* Yes, it is. + OK, let's save the amount of data and try again + with LIST */ + previous_rd_size = rd_size; + try_again = true; + con->st |= AVOID_LIST_A; + } + else + { + /* No: something happens and LIST was used. + This means "LIST -a" raises an error. */ + con->st |= LIST_AFTER_LIST_A_CHECK_DONE; + con->st |= AVOID_LIST_A; + DEBUGP (("\"LIST -a\" failed: I will use \"LIST\" " + "as standard list command\n")); + } + } + } + } + } while (try_again); + return RETRFINISHED; + +exit_error: + + /* If fp is a regular file, close and try to remove it */ + if (fp && (!output_stream || con->cmd & DO_LIST)) + fclose (fp); + return err; +} + +/* A one-file FTP loop. This is the part where FTP retrieval is + retried, and retried, and retried, and... + + This loop either gets commands from con, or (if ON_YOUR_OWN is + set), makes them up to retrieve the file given by the URL. */ +static uerr_t +ftp_loop_internal (struct url *u, struct url *original_url, struct fileinfo *f, + ccon *con, char **local_file, bool force_full_retrieve) +{ + int count, orig_lp; + wgint restval, len = 0, qtyread = 0; + char *tms, *locf; + const char *tmrate = NULL; + uerr_t err; + struct stat st; + + /* Declare WARC variables. */ + bool warc_enabled = (opt.warc_filename != NULL); + FILE *warc_tmp = NULL; + ip_address warc_ip_buf, *warc_ip = NULL; + wgint last_expected_bytes = 0; + + /* Get the target, and set the name for the message accordingly. */ + if ((f == NULL) && (con->target)) + { + /* Explicit file (like ".listing"). */ + locf = con->target; + } + else + { + /* URL-derived file. Consider "-O file" name. */ + xfree (con->target); + con->target = url_file_name (opt.trustservernames || !original_url ? u : original_url, NULL); + if (!opt.output_document) + locf = con->target; + else + locf = opt.output_document; + } + + /* If the output_document was given, then this check was already done and + the file didn't exist. Hence the !opt.output_document */ + + /* If we receive .listing file it is necessary to determine system type of the ftp + server even if opn.noclobber is given. Thus we must ignore opt.noclobber in + order to establish connection with the server and get system type. */ + if (opt.noclobber && !opt.output_document && file_exists_p (con->target, NULL) + && !((con->cmd & DO_LIST) && !(con->cmd & DO_RETR))) + { + logprintf (LOG_VERBOSE, + _("File %s already there; not retrieving.\n"), quote (con->target)); + /* If the file is there, we suppose it's retrieved OK. */ + return RETROK; + } + + /* Remove it if it's a link. */ + remove_link (con->target); + + count = 0; + + if (con->st & ON_YOUR_OWN) + con->st = ON_YOUR_OWN; + + orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0; + + /* THE loop. */ + do + { + /* Increment the pass counter. */ + ++count; + sleep_between_retrievals (count); + if (con->st & ON_YOUR_OWN) + { + con->cmd = 0; + con->cmd |= (DO_RETR | LEAVE_PENDING); + if (con->csock != -1) + con->cmd &= ~ (DO_LOGIN | DO_CWD); + else + con->cmd |= (DO_LOGIN | DO_CWD); + } + else /* not on your own */ + { + if (con->csock != -1) + con->cmd &= ~DO_LOGIN; + else + con->cmd |= DO_LOGIN; + if (con->st & DONE_CWD) + con->cmd &= ~DO_CWD; + else + con->cmd |= DO_CWD; + } + + /* For file RETR requests, we can write a WARC record. + We record the file contents to a temporary file. */ + if (warc_enabled && (con->cmd & DO_RETR) && warc_tmp == NULL) + { + warc_tmp = warc_tempfile (); + if (warc_tmp == NULL) + return WARC_TMP_FOPENERR; + + if (!con->proxy && con->csock != -1) + { + warc_ip = &warc_ip_buf; + socket_ip_address (con->csock, warc_ip, ENDPOINT_PEER); + } + } + + /* Decide whether or not to restart. */ + if (con->cmd & DO_LIST) + restval = 0; + else if (force_full_retrieve) + restval = 0; + else if (opt.start_pos >= 0) + restval = opt.start_pos; + else if (opt.always_rest + && stat (locf, &st) == 0 + && S_ISREG (st.st_mode)) + /* When -c is used, continue from on-disk size. (Can't use + hstat.len even if count>1 because we don't want a failed + first attempt to clobber existing data.) */ + restval = st.st_size; + else if (count > 1) + restval = qtyread; /* start where the previous run left off */ + else + restval = 0; + + /* Get the current time string. */ + tms = datetime_str (time (NULL)); + /* Print fetch message, if opt.verbose. */ + if (opt.verbose) + { + char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD); + char tmp[256]; + strcpy (tmp, " "); + if (count > 1) + sprintf (tmp, _("(try:%2d)"), count); + logprintf (LOG_VERBOSE, "--%s-- %s\n %s => %s\n", + tms, hurl, tmp, quote (locf)); +#ifdef WINDOWS + ws_changetitle (hurl); +#endif + xfree (hurl); + } + /* Send getftp the proper length, if fileinfo was provided. */ + if (f && f->type != FT_SYMLINK) + len = f->size; + else + len = 0; + + /* If we are working on a WARC record, getftp should also write + to the warc_tmp file. */ + err = getftp (u, original_url, len, &qtyread, restval, con, count, + &last_expected_bytes, warc_tmp); + + if (con->csock == -1) + con->st &= ~DONE_CWD; + else + con->st |= DONE_CWD; + + switch (err) + { + case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR: + case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case FTPNOAUTH: case FTPNOPBSZ: case FTPNOPROT: + case UNLINKERR: case WARC_TMP_FWRITEERR: case CONSSLERR: case CONTNOTSUPPORTED: + case VERIFCERTERR: +#ifdef HAVE_SSL + if (err == FTPNOAUTH) + logputs (LOG_NOTQUIET, "Server does not support AUTH TLS.\n"); + if (opt.ftps_implicit) + logputs (LOG_NOTQUIET, "Server does not like implicit FTPS connections.\n"); +#endif + /* Fatal errors, give up. */ + if (warc_tmp != NULL) + { + fclose (warc_tmp); + warc_tmp = NULL; + } + return err; + case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR: + case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR: + case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV: + case FOPEN_EXCL_ERR: + printwhat (count, opt.ntry); + /* non-fatal errors */ + if (err == FOPEN_EXCL_ERR) + { + /* Re-determine the file name. */ + xfree (con->target); + con->target = url_file_name (u, NULL); + locf = con->target; + } + continue; + case FTPRETRINT: + /* If the control connection was closed, the retrieval + will be considered OK if f->size == len. */ + if (!f || qtyread != f->size) + { + printwhat (count, opt.ntry); + continue; + } + break; + case RETRFINISHED: + /* Great! */ + break; + default: + /* Not as great. */ + abort (); + } + tms = datetime_str (time (NULL)); + if (!opt.spider) + tmrate = retr_rate (qtyread - restval, con->dltime); + + /* If we get out of the switch above without continue'ing, we've + successfully downloaded a file. Remember this fact. */ + downloaded_file (FILE_DOWNLOADED_NORMALLY, locf); + + if (con->st & ON_YOUR_OWN) + { + fd_close (con->csock); + con->csock = -1; + } + if (!opt.spider) + { + bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document)); + + logprintf (LOG_VERBOSE, + write_to_stdout + ? _("%s (%s) - written to stdout %s[%s]\n\n") + : _("%s (%s) - %s saved [%s]\n\n"), + tms, tmrate, + write_to_stdout ? "" : quote (locf), + number_to_static_string (qtyread)); + } + if (!opt.verbose && !opt.quiet) + { + /* Need to hide the password from the URL. The `if' is here + so that we don't do the needless allocation every + time. */ + char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD); + logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n", + tms, hurl, number_to_static_string (qtyread), locf, count); + xfree (hurl); + } + + if (warc_enabled && (con->cmd & DO_RETR)) + { + /* Create and store a WARC resource record for the retrieved file. */ + bool warc_res; + + warc_res = warc_write_resource_record (NULL, u->url, NULL, NULL, + warc_ip, NULL, warc_tmp, -1); + + if (! warc_res) + return WARC_ERR; + + /* warc_write_resource_record has also closed warc_tmp. */ + warc_tmp = NULL; + } + + if (con->cmd & DO_LIST) + /* This is a directory listing file. */ + { + if (!opt.remove_listing) + /* --dont-remove-listing was specified, so do count this towards the + number of bytes and files downloaded. */ + { + total_downloaded_bytes += qtyread; + numurls++; + } + + /* Deletion of listing files is not controlled by --delete-after, but + by the more specific option --dont-remove-listing, and the code + to do this deletion is in another function. */ + } + else if (!opt.spider) + /* This is not a directory listing file. */ + { + /* Unlike directory listing files, don't pretend normal files weren't + downloaded if they're going to be deleted. People seeding proxies, + for instance, may want to know how many bytes and files they've + downloaded through it. */ + total_downloaded_bytes += qtyread; + numurls++; + + if (opt.delete_after && !input_file_url (opt.input_filename)) + { + DEBUGP (("\ +Removing file due to --delete-after in ftp_loop_internal():\n")); + logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf); + if (unlink (locf)) + logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); + } + } + + /* Restore the original leave-pendingness. */ + if (orig_lp) + con->cmd |= LEAVE_PENDING; + else + con->cmd &= ~LEAVE_PENDING; + + if (local_file) + *local_file = xstrdup (locf); + + if (warc_tmp != NULL) + { + fclose (warc_tmp); + warc_tmp = NULL; + } + + return RETROK; + } while (!opt.ntry || (count < opt.ntry)); + + if (con->csock != -1 && (con->st & ON_YOUR_OWN)) + { + fd_close (con->csock); + con->csock = -1; + } + + if (warc_tmp != NULL) + fclose (warc_tmp); + + return TRYLIMEXC; +} + +/* Return the directory listing in a reusable format. The directory + is specified in u->dir. */ +static uerr_t +ftp_get_listing (struct url *u, struct url *original_url, ccon *con, + struct fileinfo **f) +{ + uerr_t err; + char *uf; /* url file name */ + char *lf; /* list file name */ + char *old_target = con->target; + + con->st &= ~ON_YOUR_OWN; + con->cmd |= (DO_LIST | LEAVE_PENDING); + con->cmd &= ~DO_RETR; + + /* Find the listing file name. We do it by taking the file name of + the URL and replacing the last component with the listing file + name. */ + uf = url_file_name (u, NULL); + lf = file_merge (uf, LIST_FILENAME); + xfree (uf); + DEBUGP ((_("Using %s as listing tmp file.\n"), quote (lf))); + + con->target = xstrdup (lf); + xfree (lf); + err = ftp_loop_internal (u, original_url, NULL, con, NULL, false); + lf = xstrdup (con->target); + xfree (con->target); + con->target = old_target; + + if (err == RETROK) + { + *f = ftp_parse_ls (lf, con->rs); + if (opt.remove_listing) + { + if (unlink (lf)) + logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); + else + logprintf (LOG_VERBOSE, _("Removed %s.\n"), quote (lf)); + } + } + else + *f = NULL; + xfree (lf); + con->cmd &= ~DO_LIST; + return err; +} + +static uerr_t ftp_retrieve_dirs (struct url *, struct url *, + struct fileinfo *, ccon *); +static uerr_t ftp_retrieve_glob (struct url *, struct url *, ccon *, int); +static struct fileinfo *delelement (struct fileinfo **, struct fileinfo **); + +/* Retrieve a list of files given in struct fileinfo linked list. If + a file is a symbolic link, do not retrieve it, but rather try to + set up a similar link on the local disk, if the symlinks are + supported. + + If opt.recursive is set, after all files have been retrieved, + ftp_retrieve_dirs will be called to retrieve the directories. */ +static uerr_t +ftp_retrieve_list (struct url *u, struct url *original_url, + struct fileinfo *f, ccon *con) +{ + static int depth = 0; + uerr_t err; + struct fileinfo *orig; + wgint local_size; + time_t tml; + bool dlthis; /* Download this (file). */ + const char *actual_target = NULL; + bool force_full_retrieve = false; + + /* Increase the depth. */ + ++depth; + if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel) + { + DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"), + depth, opt.reclevel)); + --depth; + return RECLEVELEXC; + } + + assert (f != NULL); + orig = f; + + con->st &= ~ON_YOUR_OWN; + if (!(con->st & DONE_CWD)) + con->cmd |= DO_CWD; + else + con->cmd &= ~DO_CWD; + con->cmd |= (DO_RETR | LEAVE_PENDING); + + if (con->csock < 0) + con->cmd |= DO_LOGIN; + else + con->cmd &= ~DO_LOGIN; + + err = RETROK; /* in case it's not used */ + + while (f) + { + char *old_target, *ofile; + + if (opt.quota && total_downloaded_bytes > opt.quota) + { + --depth; + return QUOTEXC; + } + old_target = con->target; + + ofile = xstrdup (u->file); + url_set_file (u, f->name); + + con->target = url_file_name (u, NULL); + err = RETROK; + + dlthis = true; + if (opt.timestamping && f->type == FT_PLAINFILE) + { + struct stat st; + /* If conversion of HTML files retrieved via FTP is ever implemented, + we'll need to stat() <file>.orig here when -K has been specified. + I'm not implementing it now since files on an FTP server are much + more likely than files on an HTTP server to legitimately have a + .orig suffix. */ + if (!stat (con->target, &st)) + { + bool eq_size; + bool cor_val; + /* Else, get it from the file. */ + local_size = st.st_size; + tml = st.st_mtime; +#ifdef WINDOWS + /* Modification time granularity is 2 seconds for Windows, so + increase local time by 1 second for later comparison. */ + tml++; +#endif + /* Compare file sizes only for servers that tell us correct + values. Assume sizes being equal for servers that lie + about file size. */ + cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT); + eq_size = cor_val ? (local_size == f->size) : true; + if (f->tstamp <= tml && eq_size) + { + /* Remote file is older, file sizes can be compared and + are both equal. */ + logprintf (LOG_VERBOSE, _("\ +Remote file no newer than local file %s -- not retrieving.\n"), quote (con->target)); + dlthis = false; + } + else if (f->tstamp > tml) + { + /* Remote file is newer */ + force_full_retrieve = true; + logprintf (LOG_VERBOSE, _("\ +Remote file is newer than local file %s -- retrieving.\n\n"), + quote (con->target)); + } + else + { + /* Sizes do not match */ + logprintf (LOG_VERBOSE, _("\ +The sizes do not match (local %s) -- retrieving.\n\n"), + number_to_static_string (local_size)); + } + } + } /* opt.timestamping && f->type == FT_PLAINFILE */ + switch (f->type) + { + case FT_SYMLINK: + /* If opt.retr_symlinks is defined, we treat symlinks as + if they were normal files. There is currently no way + to distinguish whether they might be directories, and + follow them. */ + if (!opt.retr_symlinks) + { +#ifdef HAVE_SYMLINK + if (!f->linkto) + logputs (LOG_NOTQUIET, + _("Invalid name of the symlink, skipping.\n")); + else + { + struct stat st; + /* Check whether we already have the correct + symbolic link. */ + int rc = lstat (con->target, &st); + if (rc == 0) + { + size_t len = strlen (f->linkto) + 1; + if (S_ISLNK (st.st_mode)) + { + char buf[1024], *link_target; + size_t n; + bool res; + + if (len < sizeof (buf)) + link_target = buf; + else + link_target = xmalloc (len); + + n = readlink (con->target, link_target, len); + res = (n == len - 1) && (memcmp (link_target, f->linkto, n) == 0); + + if (link_target != buf) + xfree (link_target); + + if (res) + { + logprintf (LOG_VERBOSE, _("\ +Already have correct symlink %s -> %s\n\n"), + quote (con->target), + quote (f->linkto)); + dlthis = false; + break; + } + } + } + logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"), + quote (con->target), quote (f->linkto)); + /* Unlink before creating symlink! */ + unlink (con->target); + if (symlink (f->linkto, con->target) == -1) + logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno)); + logputs (LOG_VERBOSE, "\n"); + } /* have f->linkto */ +#else /* not HAVE_SYMLINK */ + logprintf (LOG_NOTQUIET, + _("Symlinks not supported, skipping symlink %s.\n"), + quote (con->target)); +#endif /* not HAVE_SYMLINK */ + } + else /* opt.retr_symlinks */ + { + if (dlthis) + { + err = ftp_loop_internal (u, original_url, f, con, NULL, + force_full_retrieve); + } + } /* opt.retr_symlinks */ + break; + case FT_DIRECTORY: + if (!opt.recursive) + logprintf (LOG_NOTQUIET, _("Skipping directory %s.\n"), + quote (f->name)); + break; + case FT_PLAINFILE: + /* Call the retrieve loop. */ + if (dlthis) + { + err = ftp_loop_internal (u, original_url, f, con, NULL, + force_full_retrieve); + } + break; + case FT_UNKNOWN: + default: + logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"), + quote (f->name)); + break; + } /* switch */ + + + /* 2004-12-15 SMS. + * Set permissions _before_ setting the times, as setting the + * permissions changes the modified-time, at least on VMS. + * Also, use the opt.output_document name here, too, as + * appropriate. (Do the test once, and save the result.) + */ + + set_local_file (&actual_target, con->target); + + /* If downloading a plain file, and the user requested it, then + set valid (non-zero) permissions. */ + if (dlthis && (actual_target != NULL) && + (f->type == FT_PLAINFILE) && opt.preserve_perm) + { + if (f->perms) + { + if (chmod (actual_target, f->perms)) + logprintf (LOG_NOTQUIET, + _("Failed to set permissions for %s.\n"), + actual_target); + } + else + DEBUGP (("Unrecognized permissions for %s.\n", actual_target)); + } + + /* Set the time-stamp information to the local file. Symlinks + are not to be stamped because it sets the stamp on the + original. :( */ + if (actual_target != NULL) + { + if (opt.useservertimestamps + && !(f->type == FT_SYMLINK && !opt.retr_symlinks) + && f->tstamp != -1 + && dlthis + && file_exists_p (con->target, NULL)) + { + touch (actual_target, f->tstamp); + } + else if (f->tstamp == -1) + logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), + actual_target); + } + + xfree (con->target); + con->target = old_target; + + url_set_file (u, ofile); + xfree (ofile); + + /* Break on fatals. */ + if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR + || err == WARC_ERR || err == WARC_TMP_FOPENERR + || err == WARC_TMP_FWRITEERR) + break; + con->cmd &= ~ (DO_CWD | DO_LOGIN); + f = f->next; + } + + /* We do not want to call ftp_retrieve_dirs here */ + if (opt.recursive && + !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel)) + err = ftp_retrieve_dirs (u, original_url, orig, con); + else if (opt.recursive) + DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"), + depth, opt.reclevel)); + --depth; + return err; +} + +/* Retrieve the directories given in a file list. This function works + by simply going through the linked list and calling + ftp_retrieve_glob on each directory entry. The function knows + about excluded directories. */ +static uerr_t +ftp_retrieve_dirs (struct url *u, struct url *original_url, + struct fileinfo *f, ccon *con) +{ + char buf[1024]; + char *container = buf; + int container_size = sizeof (buf); + + for (; f; f = f->next) + { + int size; + char *odir, *newdir; + + if (opt.quota && total_downloaded_bytes > opt.quota) + break; + if (f->type != FT_DIRECTORY) + continue; + + /* Allocate u->dir off stack, but reallocate only if a larger + string is needed. It's a pity there's no "realloca" for an + item on the bottom of the stack. */ + size = strlen (u->dir) + 1 + strlen (f->name) + 1; + if (size > container_size) + { + if (container == buf) + container = xmalloc (size); + else + container = xrealloc (container, size); + + container_size = size; + } + newdir = container; + + odir = u->dir; + if (*odir == '\0' + || (*odir == '/' && *(odir + 1) == '\0')) + /* If ODIR is empty or just "/", simply append f->name to + ODIR. (In the former case, to preserve u->dir being + relative; in the latter case, to avoid double slash.) */ + sprintf (newdir, "%s%s", odir, f->name); + else + /* Else, use a separator. */ + sprintf (newdir, "%s/%s", odir, f->name); + + DEBUGP (("Composing new CWD relative to the initial directory.\n")); + DEBUGP ((" odir = '%s'\n f->name = '%s'\n newdir = '%s'\n\n", + odir, f->name, newdir)); + if (!accdir (newdir)) + { + logprintf (LOG_VERBOSE, _("\ +Not descending to %s as it is excluded/not-included.\n"), + quote (newdir)); + continue; + } + + con->st &= ~DONE_CWD; + + odir = xstrdup (u->dir); /* because url_set_dir will free + u->dir. */ + url_set_dir (u, newdir); + ftp_retrieve_glob (u, original_url, con, GLOB_GETALL); + url_set_dir (u, odir); + xfree (odir); + + /* Set the time-stamp? */ + } + + if (container != buf) + xfree (container); + + if (opt.quota && total_downloaded_bytes > opt.quota) + return QUOTEXC; + else + return RETROK; +} + +/* Return true if S has a leading '/' or contains '../' */ +static bool +has_insecure_name_p (const char *s) +{ + if (*s == '/') + return true; + + if (strstr (s, "../") != 0) + return true; + + return false; +} + +/* Test if the file node is invalid. This can occur due to malformed or + * maliciously crafted listing files being returned by the server. + * + * Currently, this function only tests if there are multiple entries in the + * listing file by the same name. However this function can be expanded as more + * such illegal listing formats are discovered. */ +static bool +is_invalid_entry (struct fileinfo *f) +{ + struct fileinfo *cur = f; + char *f_name = f->name; + + /* If the node we're currently checking has a duplicate later, we eliminate + * the current node and leave the next one intact. */ + while (cur->next) + { + cur = cur->next; + if (strcmp (f_name, cur->name) == 0) + return true; + } + return false; +} + +/* A near-top-level function to retrieve the files in a directory. + The function calls ftp_get_listing, to get a linked list of files. + Then it weeds out the file names that do not match the pattern. + ftp_retrieve_list is called with this updated list as an argument. + + If the argument ACTION is GLOB_GETONE, just download the file (but + first get the listing, so that the time-stamp is heeded); if it's + GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole + directory. */ +static uerr_t +ftp_retrieve_glob (struct url *u, struct url *original_url, + ccon *con, int action) +{ + struct fileinfo *f, *start; + uerr_t res; + + con->cmd |= LEAVE_PENDING; + + res = ftp_get_listing (u, original_url, con, &start); + if (res != RETROK) + return res; + + // Set the function used for glob matching. + int (*matcher) (const char *, const char *, int) + = opt.ignore_case ? fnmatch_nocase : fnmatch; + + // Set the function used to compare strings +#ifdef __VMS + /* 2009-09-09 SMS. + * Odd-ball compiler ("HP C V7.3-009 on OpenVMS Alpha V7.3-2") + * bug causes spurious %CC-E-BADCONDIT complaint with this + * "?:" statement. (Different linkage attributes for strcmp() + * and strcasecmp().) Converting to "if" changes the + * complaint to %CC-W-PTRMISMATCH on "cmp = strcmp;". Adding + * the senseless type cast clears the complaint, and looks + * harmless. + */ + int (*cmp) (const char *, const char *) + = opt.ignore_case ? strcasecmp : (int (*)())strcmp; +#else /* def __VMS */ + int (*cmp) (const char *, const char *) + = opt.ignore_case ? strcasecmp : strcmp; +#endif /* def __VMS [else] */ + + f = start; + while (f) + { + + // Weed out files that do not confirm to the global rules given in + // opt.accepts and opt.rejects + if ((opt.accepts || opt.rejects) && + f->type != FT_DIRECTORY && !acceptable (f->name)) + { + logprintf (LOG_VERBOSE, _("Rejecting %s.\n"), + quote (f->name)); + f = delelement (&f, &start); + continue; + } + + + // Identify and eliminate possibly harmful names or invalid entries. + if (has_insecure_name_p (f->name) || is_invalid_entry (f)) + { + logprintf (LOG_VERBOSE, _("Rejecting %s (Invalid Entry).\n"), + quote (f->name)); + f = delelement (&f, &start); + continue; + } + + if (opt.acceptregex || opt.rejectregex) + { + // accept_url() takes the full URL. + char buf[1024]; + char *url = buf; + + if ((unsigned) snprintf(buf, sizeof(buf), "%s%s%s", + u->url, f->name, f->type == FT_DIRECTORY ? "/" : "") + >= sizeof(buf)) + { + url = aprintf("%s%s%s", u->url, f->name, f->type == FT_DIRECTORY ? "/" : ""); + } + + if (!accept_url (url)) + { + logprintf (LOG_VERBOSE, _ ("%s is excluded/not-included through regex.\n"), url); + f = delelement (&f, &start); + if (url != buf) + xfree(url); + continue; + } + + if (url != buf) + xfree(url); + } + + /* Now weed out the files that do not match our globbing pattern. + If we are dealing with a globbing pattern, that is. */ + if (*u->file) + { + if (action == GLOB_GLOBALL) + { + int matchres = matcher (u->file, f->name, 0); + if (matchres == -1) + { + logprintf (LOG_NOTQUIET, _("Error matching %s against %s: %s\n"), + u->file, quotearg_style (escape_quoting_style, f->name), + strerror (errno)); + freefileinfo (start); + return RETRBADPATTERN; + } + if (matchres == FNM_NOMATCH) + { + f = delelement (&f, &start); /* delete the element from the list */ + continue; + } + } + else if (action == GLOB_GETONE) + { + if (0 != cmp(u->file, f->name)) + { + f = delelement (&f, &start); + continue; + } + } + } + f = f->next; + } + + /* + * Now that preprocessing of the file listing is over, let's try to download + * all the remaining files in our listing. + */ + if (start) + { + /* Just get everything. */ + res = ftp_retrieve_list (u, original_url, start, con); + } + else + { + if (action == GLOB_GLOBALL) + { + /* No luck. */ + /* #### This message SUCKS. We should see what was the + reason that nothing was retrieved. */ + logprintf (LOG_VERBOSE, _("No matches on pattern %s.\n"), + quote (u->file)); + } + else if (action == GLOB_GETONE) /* GLOB_GETONE or GLOB_GETALL */ + { + /* Let's try retrieving it anyway. */ + con->st |= ON_YOUR_OWN; + res = ftp_loop_internal (u, original_url, NULL, con, NULL, false); + return res; + } + + /* If action == GLOB_GETALL, and the file list is empty, there's + no point in trying to download anything or in complaining about + it. (An empty directory should not cause complaints.) + */ + } + freefileinfo (start); + if (opt.quota && total_downloaded_bytes > opt.quota) + return QUOTEXC; + else + return res; +} + +/* The wrapper that calls an appropriate routine according to contents + of URL. Inherently, its capabilities are limited on what can be + encoded into a URL. */ +uerr_t +ftp_loop (struct url *u, struct url *original_url, char **local_file, int *dt, + struct url *proxy, bool recursive, bool glob) +{ + ccon con; /* FTP connection */ + uerr_t res; + + *dt = 0; + + xzero (con); + + con.csock = -1; + con.st = ON_YOUR_OWN; + con.rs = ST_UNIX; + con.id = NULL; + con.proxy = proxy; + + /* If the file name is empty, the user probably wants a directory + index. We'll provide one, properly HTML-ized. Unless + opt.htmlify is 0, of course. :-) */ + if (!*u->file && !recursive) + { + struct fileinfo *f; + res = ftp_get_listing (u, original_url, &con, &f); + + if (res == RETROK) + { + if (opt.htmlify && !opt.spider) + { + struct url *url_file = opt.trustservernames ? u : original_url; + char *filename = (opt.output_document + ? xstrdup (opt.output_document) + : (con.target ? xstrdup (con.target) + : url_file_name (url_file, NULL))); + res = ftp_index (filename, u, f); + if (res == FTPOK && opt.verbose) + { + if (!opt.output_document) + { + struct stat st; + wgint sz; + if (stat (filename, &st) == 0) + sz = st.st_size; + else + sz = -1; + logprintf (LOG_NOTQUIET, + _("Wrote HTML-ized index to %s [%s].\n"), + quote (filename), number_to_static_string (sz)); + } + else + logprintf (LOG_NOTQUIET, + _("Wrote HTML-ized index to %s.\n"), + quote (filename)); + } + xfree (filename); + } + freefileinfo (f); + } + } + else + { + bool ispattern = false; + if (glob) + { + /* Treat the URL as a pattern if the file name part of the + URL path contains wildcards. (Don't check for u->file + because it is unescaped and therefore doesn't leave users + the option to escape literal '*' as %2A.) */ + char *file_part = strrchr (u->path, '/'); + if (!file_part) + file_part = u->path; + ispattern = has_wildcards_p (file_part); + } + if (ispattern || recursive || opt.timestamping || opt.preserve_perm) + { + /* ftp_retrieve_glob is a catch-all function that gets called + if we need globbing, time-stamping, recursion or preserve + permissions. Its third argument is just what we really need. */ + res = ftp_retrieve_glob (u, original_url, &con, + ispattern ? GLOB_GLOBALL : GLOB_GETONE); + } + else + { + res = ftp_loop_internal (u, original_url, NULL, &con, local_file, false); + } + } + if (res == FTPOK) + res = RETROK; + if (res == RETROK) + *dt |= RETROKF; + /* If a connection was left, quench it. */ + if (con.csock != -1) + fd_close (con.csock); + xfree (con.id); + xfree (con.target); + return res; +} + +/* Delete an element from the fileinfo linked list. Returns the + address of the next element, or NULL if the list is exhausted. It + can modify the start of the list. */ +static struct fileinfo * +delelement (struct fileinfo **f, struct fileinfo **start) +{ + struct fileinfo *prev = (*f)->prev; + struct fileinfo *next = (*f)->next; + + xfree ((*f)->name); + xfree ((*f)->linkto); + xfree (*f); + *f = NULL; + + if (next) + next->prev = prev; + if (prev) + prev->next = next; + else + *start = next; + return next; +} + +/* Free the fileinfo linked list of files. */ +void +freefileinfo (struct fileinfo *f) +{ + while (f) + { + struct fileinfo *next = f->next; + xfree (f->name); + if (f->linkto) + xfree (f->linkto); + xfree (f); + f = next; + } +} diff --git a/src/ftp.h b/src/ftp.h new file mode 100644 index 0000000..9958853 --- /dev/null +++ b/src/ftp.h @@ -0,0 +1,184 @@ +/* Declarations for FTP support. + Copyright (C) 1996-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef FTP_H +#define FTP_H + +#include <stdio.h> +#include <stdbool.h> + +#include "host.h" +#include "url.h" + +/* System types. */ +enum stype +{ + ST_UNIX, + ST_VMS, + ST_WINNT, + ST_MACOS, + ST_OS400, + ST_OTHER +}; + +/* Extensions of the ST_UNIX */ +enum ustype +{ + UST_TYPE_L8, + UST_MULTINET, + UST_OTHER +}; + +#ifdef HAVE_SSL +/* Data channel protection levels (to be used with PBSZ) */ +enum prot_level +{ + PROT_CLEAR = 'C', + PROT_SAFE = 'S', + PROT_CONFIDENTIAL = 'E', + PROT_PRIVATE = 'P' +}; +#endif + +uerr_t ftp_response (int, char **); +uerr_t ftp_greeting (int); +uerr_t ftp_login (int, const char *, const char *); +uerr_t ftp_port (int, int *); +uerr_t ftp_pasv (int, ip_address *, int *); +#ifdef HAVE_SSL +uerr_t ftp_auth (int, enum url_scheme); +uerr_t ftp_pbsz (int, int); +uerr_t ftp_prot (int, enum prot_level); +#endif +#ifdef ENABLE_IPV6 +uerr_t ftp_lprt (int, int *); +uerr_t ftp_lpsv (int, ip_address *, int *); +uerr_t ftp_eprt (int, int *); +uerr_t ftp_epsv (int, ip_address *, int *); +#endif +uerr_t ftp_type (int, int); +uerr_t ftp_cwd (int, const char *); +uerr_t ftp_retr (int, const char *); +uerr_t ftp_rest (int, wgint); +uerr_t ftp_list (int, const char *, bool, bool, bool *); +uerr_t ftp_syst (int, enum stype *, enum ustype *); +uerr_t ftp_pwd (int, char **); +uerr_t ftp_size (int, const char *, wgint *); + +#ifdef ENABLE_OPIE +const char *skey_response (int, const char *, const char *); +#endif + +struct url; + +/* File types. */ +enum ftype +{ + FT_PLAINFILE, + FT_DIRECTORY, + FT_SYMLINK, + FT_UNKNOWN +}; + + +/* Globbing (used by ftp_retrieve_glob). */ +enum +{ + GLOB_GLOBALL, GLOB_GETALL, GLOB_GETONE +}; + +/* Used by to test if time parsed includes hours and minutes. */ +enum parsetype +{ + TT_HOUR_MIN, TT_DAY +}; + + +/* Information about one filename in a linked list. */ +struct fileinfo +{ + enum ftype type; /* file type */ + char *name; /* file name */ + wgint size; /* file size */ + long tstamp; /* time-stamp */ + enum parsetype ptype; /* time parsing */ + int perms; /* file permissions */ + char *linkto; /* link to which file points */ + struct fileinfo *prev; /* previous... */ + struct fileinfo *next; /* ...and next structure. */ +}; + +/* Commands for FTP functions. */ +enum wget_ftp_command +{ + DO_LOGIN = 0x0001, /* Connect and login to the server. */ + DO_CWD = 0x0002, /* Change current directory. */ + DO_RETR = 0x0004, /* Retrieve the file. */ + DO_LIST = 0x0008, /* Retrieve the directory list. */ + LEAVE_PENDING = 0x0010 /* Do not close the socket. */ +}; + +enum wget_ftp_fstatus +{ + NOTHING = 0x0000, /* Nothing done yet. */ + ON_YOUR_OWN = 0x0001, /* The ftp_loop_internal sets the + defaults. */ + DONE_CWD = 0x0002, /* The current working directory is + correct. */ + + /* 2013-10-17 Andrea Urbani (matfanjol) + For more information about the following entries, please, + look at ftp.c, function getftp, text "__LIST_A_EXPLANATION__". */ + AVOID_LIST_A = 0x0004, /* It tells us if during this + session we have to avoid the use + of "LIST -a".*/ + AVOID_LIST = 0x0008, /* It tells us if during this + session we have to avoid to use + "LIST". */ + LIST_AFTER_LIST_A_CHECK_DONE = 0x0010, + /* It tells us if we have already + checked "LIST" after the first + "LIST -a" to handle the case of + file/folders named "-a". */ + DATA_CHANNEL_SECURITY = 0x0020 /* Establish a secure data channel */ +}; + +struct fileinfo *ftp_parse_ls (const char *, const enum stype); +struct fileinfo *ftp_parse_ls_fp (FILE *, const enum stype); +void freefileinfo(struct fileinfo *); +uerr_t ftp_loop (struct url *, struct url *, char **, int *, struct url *, + bool, bool); + +uerr_t ftp_index (const char *, struct url *, struct fileinfo *); + +char ftp_process_type (const char *); + + +#endif /* FTP_H */ diff --git a/src/gnutls.c b/src/gnutls.c new file mode 100644 index 0000000..5706dbb --- /dev/null +++ b/src/gnutls.c @@ -0,0 +1,1116 @@ +/* SSL support via GnuTLS library. + Copyright (C) 2005-2012, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <errno.h> +#include <unistd.h> +#include <string.h> +#include <stdio.h> +#include <dirent.h> +#include <stdlib.h> +#include <xalloc.h> + +#include <gnutls/abstract.h> +#include <gnutls/gnutls.h> +#include <gnutls/x509.h> +#include <sys/ioctl.h> + +#include "utils.h" +#include "connect.h" +#include "url.h" +#include "ptimer.h" +#include "hash.h" +#include "ssl.h" + +#include <fcntl.h> + +#ifdef WIN32 +# include "w32sock.h" +#endif + +#include "host.h" + +struct st_read_timer +{ + double timeout; + double next_timeout; + struct ptimer *timer; + int timed_out; +}; + +static int +_do_handshake (gnutls_session_t session, int fd, struct st_read_timer *timeout); + +#if GNUTLS_VERSION_NUMBER >= 0x030604 +static int +_do_reauth (gnutls_session_t session, int fd, struct st_read_timer *timeout); +#endif + +static int +key_type_to_gnutls_type (enum keyfile_type type) +{ + switch (type) + { + case keyfile_pem: + return GNUTLS_X509_FMT_PEM; + case keyfile_asn1: + return GNUTLS_X509_FMT_DER; + default: + abort (); + } +} + +/* Note: some of the functions private to this file have names that + begin with "wgnutls_" (e.g. wgnutls_read) so that they wouldn't be + confused with actual gnutls functions -- such as the gnutls_read + preprocessor macro. */ + +static gnutls_certificate_credentials_t credentials; +bool +ssl_init (void) +{ + /* Becomes true if GnuTLS is initialized. */ + static bool ssl_initialized = false; + const char *ca_directory; + DIR *dir; + int ncerts = -1; + int rc; + + /* GnuTLS should be initialized only once. */ + if (ssl_initialized) + return true; + + gnutls_global_init (); + gnutls_certificate_allocate_credentials (&credentials); + gnutls_certificate_set_verify_flags (credentials, + GNUTLS_VERIFY_ALLOW_X509_V1_CA_CRT); + +#if GNUTLS_VERSION_MAJOR >= 3 + if (!opt.ca_directory) + ncerts = gnutls_certificate_set_x509_system_trust (credentials); +#endif + + /* If GnuTLS version is too old or CA loading failed, fallback to old behaviour. + * Also use old behaviour if the CA directory is user-provided. */ + if (ncerts <= 0) + { + ncerts = 0; + + ca_directory = opt.ca_directory ? opt.ca_directory : "/etc/ssl/certs"; + + if ((dir = opendir (ca_directory)) == NULL) + { + if (opt.ca_directory && *opt.ca_directory) + logprintf (LOG_NOTQUIET, _("ERROR: Cannot open directory %s.\n"), + opt.ca_directory); + } + else + { + struct hash_table *inode_map = hash_table_new (196, NULL, NULL); + struct dirent *dent; + + while ((dent = readdir (dir)) != NULL) + { + struct stat st; + char ca_file[1024]; + + if (((unsigned) snprintf (ca_file, sizeof (ca_file), "%s/%s", ca_directory, dent->d_name)) >= sizeof (ca_file)) + continue; // overflow + + if (stat (ca_file, &st) != 0) + continue; + + if (! S_ISREG (st.st_mode)) + continue; + + /* avoid loading the same file twice by checking the inode. */ + if (hash_table_contains (inode_map, (void *)(intptr_t) st.st_ino)) + continue; + + hash_table_put (inode_map, (void *)(intptr_t) st.st_ino, NULL); + if ((rc = gnutls_certificate_set_x509_trust_file (credentials, ca_file, + GNUTLS_X509_FMT_PEM)) <= 0) + DEBUGP (("WARNING: Failed to open cert %s: (%d).\n", ca_file, rc)); + else + ncerts += rc; + } + + hash_table_destroy (inode_map); + closedir (dir); + } + } + + if (opt.ca_cert) + { + if (ncerts < 0) + ncerts = 0; + + if ((rc = gnutls_certificate_set_x509_trust_file (credentials, opt.ca_cert, + GNUTLS_X509_FMT_PEM)) <= 0) + logprintf (LOG_NOTQUIET, _("ERROR: Failed to open cert %s: (%d).\n"), + opt.ca_cert, rc); + else + { + ncerts += rc; + logprintf (LOG_VERBOSE, _("Loaded CA certificate '%s'\n"), opt.ca_cert); + } + } + + if (opt.crl_file) + { + if ((rc = gnutls_certificate_set_x509_crl_file (credentials, opt.crl_file, GNUTLS_X509_FMT_PEM)) <= 0) + { + logprintf (LOG_NOTQUIET, _("ERROR: Failed to load CRL file '%s': (%d)\n"), opt.crl_file, rc); + return false; + } + + logprintf (LOG_VERBOSE, _("Loaded CRL file '%s'\n"), opt.crl_file); + } + + DEBUGP (("Certificates loaded: %d\n", ncerts)); + + /* Use the private key from the cert file unless otherwise specified. */ + if (opt.cert_file && !opt.private_key) + { + opt.private_key = xstrdup (opt.cert_file); + opt.private_key_type = opt.cert_type; + } + /* Use the cert from the private key file unless otherwise specified. */ + if (!opt.cert_file && opt.private_key) + { + opt.cert_file = xstrdup (opt.private_key); + opt.cert_type = opt.private_key_type; + } + + if (opt.cert_file && opt.private_key) + { + int type; + if (opt.private_key_type != opt.cert_type) + { + /* GnuTLS can't handle this */ + logprintf (LOG_NOTQUIET, _("ERROR: GnuTLS requires the key and the \ +cert to be of the same type.\n")); + } + + type = key_type_to_gnutls_type (opt.private_key_type); + + gnutls_certificate_set_x509_key_file (credentials, opt.cert_file, + opt.private_key, + type); + } + + ssl_initialized = true; + + return true; +} + +void +ssl_cleanup (void) +{ + if (credentials) + gnutls_certificate_free_credentials(credentials); + + gnutls_global_deinit(); +} + +struct wgnutls_transport_context +{ + gnutls_session_t session; /* GnuTLS session handle */ + gnutls_datum_t *session_data; + int last_error; /* last error returned by read/write/... */ + + /* Since GnuTLS doesn't support the equivalent to recv(..., + MSG_PEEK) or SSL_peek(), we have to do it ourselves. Peeked data + is stored to PEEKBUF, and wgnutls_read checks that buffer before + actually reading. */ + char peekbuf[512]; + int peeklen; +}; + +static int +wgnutls_read_timeout (int fd, char *buf, int bufsize, void *arg, double timeout) +{ +#ifdef F_GETFL + int flags = 0; +#endif + struct wgnutls_transport_context *ctx = arg; + int ret = gnutls_record_check_pending (ctx->session); + struct st_read_timer read_timer = {(timeout == -1 ? opt.read_timeout : timeout), 0, NULL, 0}; + + if (ret) + return gnutls_record_recv (ctx->session, buf, MIN (ret, bufsize)); + + if (read_timer.timeout) + { +#ifdef F_GETFL + flags = fcntl (fd, F_GETFL, 0); + if (flags < 0) + return flags; + if (fcntl (fd, F_SETFL, flags | O_NONBLOCK)) + return -1; +#else + /* XXX: Assume it was blocking before. */ + const int one = 1; + if (ioctl (fd, FIONBIO, &one) < 0) + return -1; +#endif + + read_timer.timer = ptimer_new (); + if (read_timer.timer == NULL) + { + ret = -1; + goto timer_err; + } + read_timer.next_timeout = read_timer.timeout; + } + + ret = ctx->last_error; + do + { + if (ret == GNUTLS_E_REHANDSHAKE) + { + int err; + DEBUGP (("GnuTLS: *** REHANDSHAKE while reading\n")); + if ((err = _do_handshake (ctx->session, fd, &read_timer)) != 0) + { + ret = err; + break; + } + } +#if GNUTLS_VERSION_NUMBER >= 0x030604 + else if (ret == GNUTLS_E_REAUTH_REQUEST) + { + int err; + DEBUGP (("GnuTLS: *** re-authentication while reading\n")); + if ((err = _do_reauth (ctx->session, fd, &read_timer)) != 0) + { + ret = err; + break; + } + } +#endif + do + { + ret = gnutls_record_recv (ctx->session, buf, bufsize); + if (ret == GNUTLS_E_AGAIN && read_timer.timer) + { + int err = select_fd_nb (fd, read_timer.next_timeout, WAIT_FOR_READ); + if (err <= 0) + { + if (err == 0) + read_timer.timed_out = 1; + goto break_all; + } + if ( (read_timer.next_timeout = read_timer.timeout - ptimer_measure (read_timer.timer)) <= 0 ) + { + read_timer.timed_out = 1; + goto break_all; + } + } + } + while (ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED); + } + while (ret == GNUTLS_E_REHANDSHAKE +#if GNUTLS_VERSION_NUMBER >= 0x030604 + || ret == GNUTLS_E_REAUTH_REQUEST +#endif + ); + +break_all: + if (read_timer.timer) + { + ptimer_destroy (read_timer.timer); +timer_err: ; +#ifdef F_GETFL + if (fcntl (fd, F_SETFL, flags) < 0) + return -1; +#else + { + const int zero = 0; + if (ioctl (fd, FIONBIO, &zero) < 0) + return -1; + } +#endif + if (read_timer.timed_out) + errno = ETIMEDOUT; + } + + return ret; +} + +static int +wgnutls_read (int fd, char *buf, int bufsize, void *arg, double timeout) +{ + int ret; + struct wgnutls_transport_context *ctx = arg; + + if (ctx->peeklen) + { + /* If we have any peek data, simply return that. */ + int copysize = MIN (bufsize, ctx->peeklen); + memcpy (buf, ctx->peekbuf, copysize); + ctx->peeklen -= copysize; + if (ctx->peeklen != 0) + memmove (ctx->peekbuf, ctx->peekbuf + copysize, ctx->peeklen); + + return copysize; + } + + ret = wgnutls_read_timeout (fd, buf, bufsize, arg, timeout); + ctx->last_error = ret; + return ret; +} + +static int +wgnutls_write (int fd _GL_UNUSED, char *buf, int bufsize, void *arg) +{ + struct wgnutls_transport_context *ctx = arg; + int ret = ctx->last_error; + + /* it should never happen, + placed here only for debug msg. */ + if (ret == GNUTLS_E_REHANDSHAKE) + { + DEBUGP (("GnuTLS: *** REHANDSHAKE while writing\n")); + if ((ret = _do_handshake (ctx->session, fd, NULL)) != 0) + goto ext; + } +#if GNUTLS_VERSION_NUMBER >= 0x030604 + else if (ret == GNUTLS_E_REAUTH_REQUEST) + { + DEBUGP (("GnuTLS: *** re-authentication while writing\n")); + if ((ret = _do_reauth (ctx->session, fd, NULL)) != 0) + goto ext; + } +#endif + + do + ret = gnutls_record_send (ctx->session, buf, bufsize); + while (ret == GNUTLS_E_INTERRUPTED || ret == GNUTLS_E_AGAIN); +ext: + ctx->last_error = ret; + return ret; +} + +static int +wgnutls_poll (int fd, double timeout, int wait_for, void *arg) +{ + struct wgnutls_transport_context *ctx = arg; + + if ((wait_for & WAIT_FOR_READ) + && (ctx->peeklen || gnutls_record_check_pending (ctx->session))) + return 1; + + if (timeout == -1) + timeout = opt.read_timeout; + return select_fd (fd, timeout, wait_for); +} + +static int +wgnutls_peek (int fd, char *buf, int bufsize, void *arg, double timeout) +{ + int read = 0; + struct wgnutls_transport_context *ctx = arg; + int offset = MIN (bufsize, ctx->peeklen); + + if (ctx->peeklen) + { + memcpy (buf, ctx->peekbuf, offset); + return offset; + } + + if (bufsize > (int) sizeof ctx->peekbuf) + bufsize = sizeof ctx->peekbuf; + + if (bufsize > offset) + { /* let wgnutls_read_timeout() take care about timeout */ + /*if (timeout && gnutls_record_check_pending (ctx->session) == 0 + && select_fd (fd, 0.0, WAIT_FOR_READ) <= 0) + read = 0; + else*/ + read = wgnutls_read_timeout (fd, buf + offset, bufsize - offset, + ctx, timeout); + ctx->last_error = read; + if (read < 0) + { + if (offset) + read = 0; + else + return read; + } + + if (read > 0) + { + memcpy (ctx->peekbuf + offset, buf + offset, + read); + ctx->peeklen += read; + } + } + + return offset + read; +} + +static const char * +wgnutls_errstr (int fd _GL_UNUSED, void *arg) +{ + struct wgnutls_transport_context *ctx = arg; + + if (ctx->last_error > 0 + || ((ctx->last_error == GNUTLS_E_AGAIN + || ctx->last_error == GNUTLS_E_REHANDSHAKE +#if GNUTLS_VERSION_NUMBER >= 0x030604 + || ctx->last_error == GNUTLS_E_REAUTH_REQUEST +#endif + ) && errno == ETIMEDOUT)) + return NULL; + + return gnutls_strerror (ctx->last_error); +} + +static void +wgnutls_close (int fd, void *arg) +{ + struct wgnutls_transport_context *ctx = arg; + /*gnutls_bye (ctx->session, GNUTLS_SHUT_RDWR);*/ + if (ctx->session_data) + { + gnutls_free (ctx->session_data->data); + gnutls_free (ctx->session_data); + } + gnutls_deinit (ctx->session); + xfree (ctx); + close (fd); +} + +/* gnutls_transport is the singleton that describes the SSL transport + methods provided by this file. */ + +static struct transport_implementation wgnutls_transport = +{ + wgnutls_read, wgnutls_write, wgnutls_poll, + wgnutls_peek, wgnutls_errstr, wgnutls_close +}; + +static int +_do_handshake (gnutls_session_t session, int fd, struct st_read_timer *read_timer) +{ +#ifdef F_GETFL + int flags = 0; +#endif + int err; + double next_timeout = (read_timer ? read_timer->next_timeout : opt.read_timeout); + + /* if (read_timer != NULL) - fd is already non blocking */ + if (!read_timer && next_timeout) + { +#ifdef F_GETFL + flags = fcntl (fd, F_GETFL, 0); + if (flags < 0) + return flags; + if (fcntl (fd, F_SETFL, flags | O_NONBLOCK)) + return -1; +#else + /* XXX: Assume it was blocking before. */ + const int one = 1; + if (ioctl (fd, FIONBIO, &one) < 0) + return -1; +#endif + } + + /* We don't stop the handshake process for non-fatal errors */ + do + { + err = gnutls_handshake (session); + + if (err == GNUTLS_E_AGAIN && next_timeout) + { + int sel; + if (gnutls_record_get_direction (session)) + { + /* wait for writeability */ + sel = WAIT_FOR_WRITE; + } + else + { + /* wait for readability */ + sel = WAIT_FOR_READ; + } + sel = select_fd_nb (fd, next_timeout, sel); + + if (sel <= 0) + { + if (sel == 0) + { + if (read_timer) + goto read_timedout; + else + { + errno = ETIMEDOUT; + err = -1; + } + } + break; + } + if (read_timer) + { + if ( (read_timer->next_timeout = read_timer->timeout - ptimer_measure (read_timer->timer)) <= 0 ) + { +read_timedout: /* return GNUTLS_E_REHANDSHAKE for gnutls_read */ + err = GNUTLS_E_REHANDSHAKE; + read_timer->timed_out = 1; + break; + } + next_timeout = read_timer->next_timeout; + } + } + else if (err < 0) + { + logprintf (LOG_NOTQUIET, "GnuTLS: %s\n", gnutls_strerror (err)); + if (err == GNUTLS_E_WARNING_ALERT_RECEIVED || + err == GNUTLS_E_FATAL_ALERT_RECEIVED) + { + gnutls_alert_description_t alert = gnutls_alert_get (session); + const char *str = gnutls_alert_get_name (alert); + logprintf (LOG_NOTQUIET, "GnuTLS: received alert [%u]: %s\n", + alert, str ? str : "(unknown)"); + } + } + } + while (err && gnutls_error_is_fatal (err) == 0); + + if (!read_timer && next_timeout) + { +#ifdef F_GETFL + if (fcntl (fd, F_SETFL, flags) < 0) + return -1; +#else + const int zero = 0; + if (ioctl (fd, FIONBIO, &zero) < 0) + return -1; +#endif + } + + return err; +} + +#if GNUTLS_VERSION_NUMBER >= 0x030604 +static int +_do_reauth (gnutls_session_t session, int fd, struct st_read_timer *read_timer) +{ +#ifdef F_GETFL + int flags = 0; +#endif + int err; + double next_timeout = (read_timer ? read_timer->next_timeout : opt.read_timeout); + + /* if (read_timer != NULL) - fd is already non blocking */ + if (!read_timer && next_timeout) + { +#ifdef F_GETFL + flags = fcntl (fd, F_GETFL, 0); + if (flags < 0) + return flags; + if (fcntl (fd, F_SETFL, flags | O_NONBLOCK)) + return -1; +#else + /* XXX: Assume it was blocking before. */ + const int one = 1; + if (ioctl (fd, FIONBIO, &one) < 0) + return -1; +#endif + } + + /* We don't stop the handshake process for non-fatal errors */ + do + { + err = gnutls_reauth (session, 0); + + if (err == GNUTLS_E_AGAIN && next_timeout) + { + int sel; + if (gnutls_record_get_direction (session)) + { + /* wait for writeability */ + sel = WAIT_FOR_WRITE; + } + else + { + /* wait for readability */ + sel = WAIT_FOR_READ; + } + sel = select_fd_nb (fd, next_timeout, sel); + + if (sel <= 0) + { + if (sel == 0) + { + if (read_timer) + goto read_timedout; + else + { + errno = ETIMEDOUT; + err = -1; + } + } + break; + } + if (read_timer) + { + if ( (read_timer->next_timeout = read_timer->timeout - ptimer_measure (read_timer->timer)) <= 0 ) + { +read_timedout: /* return GNUTLS_E_REAUTH_REQUEST for gnutls_read */ + err = GNUTLS_E_REAUTH_REQUEST; + read_timer->timed_out = 1; + break; + } + next_timeout = read_timer->next_timeout; + } + } + else if (err < 0) + { + logprintf (LOG_NOTQUIET, "GnuTLS: %s\n", gnutls_strerror (err)); + } + } + while (err && gnutls_error_is_fatal (err) == 0); + + if (!read_timer && next_timeout) + { +#ifdef F_GETFL + if (fcntl (fd, F_SETFL, flags) < 0) + return -1; +#else + const int zero = 0; + if (ioctl (fd, FIONBIO, &zero) < 0) + return -1; +#endif + } + + return err; +} +#endif + +static const char * +_sni_hostname(const char *hostname) +{ + size_t len = strlen(hostname); + + char *sni_hostname = xmemdup(hostname, len + 1); + + /* Remove trailing dot(s) to fix #47408. + * Regarding RFC 6066 (SNI): The hostname is represented as a byte + * string using ASCII encoding without a trailing dot. */ + while (len && sni_hostname[--len] == '.') + sni_hostname[len] = 0; + + return sni_hostname; +} + +static int +set_prio_default (gnutls_session_t session) +{ + int err = -1; + +#if HAVE_GNUTLS_PRIORITY_SET_DIRECT + switch (opt.secure_protocol) + { + case secure_protocol_auto: + err = gnutls_set_default_priority (session); + gnutls_session_enable_compatibility_mode(session); + break; + + case secure_protocol_sslv2: + case secure_protocol_sslv3: + err = gnutls_priority_set_direct (session, "NORMAL:-VERS-TLS-ALL:+VERS-SSL3.0", NULL); + break; + + case secure_protocol_tlsv1: + err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0", NULL); + break; + + case secure_protocol_tlsv1_1: + err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0:-VERS-TLS1.0", NULL); + break; + + case secure_protocol_tlsv1_2: + err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1", NULL); + break; + + case secure_protocol_tlsv1_3: +#if GNUTLS_VERSION_NUMBER >= 0x030603 + err = gnutls_priority_set_direct (session, "NORMAL:-VERS-SSL3.0:+VERS-TLS1.3:-VERS-TLS1.0:-VERS-TLS1.1:-VERS-TLS1.2", NULL); + break; +#else + logprintf (LOG_NOTQUIET, _("Your GnuTLS version is too old to support TLS 1.3\n")); + return -1; +#endif + + case secure_protocol_pfs: + err = gnutls_priority_set_direct (session, "PFS:-VERS-SSL3.0", NULL); + if (err != GNUTLS_E_SUCCESS) + /* fallback if PFS is not available */ + err = gnutls_priority_set_direct (session, "NORMAL:-RSA:-VERS-SSL3.0", NULL); + break; + + default: + logprintf (LOG_NOTQUIET, _("GnuTLS: unimplemented 'secure-protocol' option value %u\n"), + (unsigned) opt.secure_protocol); + logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n")); + abort (); + } +#else + int allowed_protocols[4] = {0, 0, 0, 0}; + switch (opt.secure_protocol) + { + case secure_protocol_auto: + err = gnutls_set_default_priority (session); + break; + + case secure_protocol_sslv2: + case secure_protocol_sslv3: + allowed_protocols[0] = GNUTLS_SSL3; + err = gnutls_protocol_set_priority (session, allowed_protocols); + break; + + case secure_protocol_tlsv1: + allowed_protocols[0] = GNUTLS_TLS1_0; + allowed_protocols[1] = GNUTLS_TLS1_1; + allowed_protocols[2] = GNUTLS_TLS1_2; +#if GNUTLS_VERSION_NUMBER >= 0x030603 + allowed_protocols[3] = GNUTLS_TLS1_3; +#endif + err = gnutls_protocol_set_priority (session, allowed_protocols); + break; + + case secure_protocol_tlsv1_1: + allowed_protocols[0] = GNUTLS_TLS1_1; + allowed_protocols[1] = GNUTLS_TLS1_2; +#if GNUTLS_VERSION_NUMBER >= 0x030603 + allowed_protocols[2] = GNUTLS_TLS1_3; +#endif + err = gnutls_protocol_set_priority (session, allowed_protocols); + break; + + case secure_protocol_tlsv1_2: + allowed_protocols[0] = GNUTLS_TLS1_2; +#if GNUTLS_VERSION_NUMBER >= 0x030603 + allowed_protocols[1] = GNUTLS_TLS1_3; +#endif + err = gnutls_protocol_set_priority (session, allowed_protocols); + break; + + case secure_protocol_tlsv1_3: +#if GNUTLS_VERSION_NUMBER >= 0x030603 + allowed_protocols[0] = GNUTLS_TLS1_3; + err = gnutls_protocol_set_priority (session, allowed_protocols); + break; +#else + logprintf (LOG_NOTQUIET, _("Your GnuTLS version is too old to support TLS 1.3\n")); + return -1; +#endif + + default: + logprintf (LOG_NOTQUIET, _("GnuTLS: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol); + logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n")); + abort (); + } +#endif + + return err; +} + +bool +ssl_connect_wget (int fd, const char *hostname, int *continue_session) +{ + struct wgnutls_transport_context *ctx; + gnutls_session_t session; + int err; + +#if GNUTLS_VERSION_NUMBER >= 0x030604 + // enable support of TLS1.3 post-handshake authentication + gnutls_init (&session, GNUTLS_CLIENT | GNUTLS_POST_HANDSHAKE_AUTH); +#else + gnutls_init (&session, GNUTLS_CLIENT); +#endif + + /* We set the server name but only if it's not an IP address. */ + if (! is_valid_ip_address (hostname)) + { + /* GnuTLS 3.4.x (x<=10) disrespects the length parameter, we have to construct a new string */ + /* see https://gitlab.com/gnutls/gnutls/issues/78 */ + const char *sni_hostname = _sni_hostname(hostname); + + gnutls_server_name_set (session, GNUTLS_NAME_DNS, sni_hostname, strlen(sni_hostname)); + xfree(sni_hostname); + } + + gnutls_credentials_set (session, GNUTLS_CRD_CERTIFICATE, credentials); +#ifndef FD_TO_SOCKET +# define FD_TO_SOCKET(X) (X) +#endif +#ifdef HAVE_INTPTR_T + gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) (intptr_t) FD_TO_SOCKET (fd)); +#else + gnutls_transport_set_ptr (session, (gnutls_transport_ptr_t) FD_TO_SOCKET (fd)); +#endif + + if (!opt.tls_ciphers_string) + { + err = set_prio_default (session); + } + else + { +#if HAVE_GNUTLS_PRIORITY_SET_DIRECT + err = gnutls_priority_set_direct (session, opt.tls_ciphers_string, NULL); +#else + logprintf (LOG_NOTQUIET, _("GnuTLS: Cannot set prio string directly. Falling back to default priority.\n")); + err = gnutls_set_default_priority (session); +#endif + } + + if (err < 0) + { + logprintf (LOG_NOTQUIET, "GnuTLS: %s\n", gnutls_strerror (err)); + gnutls_deinit (session); + return false; + } + + if (continue_session) + { + ctx = (struct wgnutls_transport_context *) fd_transport_context (*continue_session); + if (!gnutls_session_is_resumed (session)) + { + if (!ctx || !ctx->session_data || gnutls_session_set_data (session, ctx->session_data->data, ctx->session_data->size)) + { + if (ctx && ctx->session_data) + { + /* server does not want to continue the session */ + if (ctx->session_data->data) + gnutls_free (ctx->session_data->data); + gnutls_free (ctx->session_data); + } + gnutls_deinit (session); + return false; + } + } + else + { + logputs (LOG_ALWAYS, "SSL session has already been resumed. Continuing.\n"); + continue_session = NULL; + } + } + + err = _do_handshake (session, fd, NULL); + + if (err < 0) + { + gnutls_deinit (session); + return false; + } + + ctx = xnew0 (struct wgnutls_transport_context); + ctx->session_data = xnew0 (gnutls_datum_t); + ctx->session = session; + if (gnutls_session_get_data2 (session, ctx->session_data)) + { + xfree (ctx->session_data); + logprintf (LOG_NOTQUIET, "WARNING: Could not save SSL session data for socket %d\n", fd); + } + fd_register_transport (fd, &wgnutls_transport, ctx); + return true; +} + +static bool +pkp_pin_peer_pubkey (gnutls_x509_crt_t cert, const char *pinnedpubkey) +{ + /* Scratch */ + size_t len1 = 0, len2 = 0; + char *buff1 = NULL; + + gnutls_pubkey_t key = NULL; + + /* Result is returned to caller */ + int ret = 0; + bool result = false; + + /* if a path wasn't specified, don't pin */ + if (NULL == pinnedpubkey) + return true; + + if (NULL == cert) + return result; + + /* Begin Gyrations to get the public key */ + gnutls_pubkey_init (&key); + + ret = gnutls_pubkey_import_x509 (key, cert, 0); + if (ret < 0) + goto cleanup; /* failed */ + + ret = gnutls_pubkey_export (key, GNUTLS_X509_FMT_DER, NULL, &len1); + if (ret != GNUTLS_E_SHORT_MEMORY_BUFFER || len1 == 0) + goto cleanup; /* failed */ + + buff1 = xmalloc (len1); + + len2 = len1; + + ret = gnutls_pubkey_export (key, GNUTLS_X509_FMT_DER, buff1, &len2); + if (ret < 0 || len1 != len2) + goto cleanup; /* failed */ + + /* End Gyrations */ + + /* The one good exit point */ + result = wg_pin_peer_pubkey (pinnedpubkey, buff1, len1); + + cleanup: + if (NULL != key) + gnutls_pubkey_deinit (key); + + xfree (buff1); + + return result; +} + +#define _CHECK_CERT(flag,msg) \ + if (status & (flag))\ + {\ + logprintf (LOG_NOTQUIET, (msg),\ + severity, quote (host));\ + success = false;\ + } + +bool +ssl_check_certificate (int fd, const char *host) +{ + struct wgnutls_transport_context *ctx = fd_transport_context (fd); + + unsigned int status; + int err; + + /* If the user has specified --no-check-cert, we still want to warn + him about problems with the server's certificate. */ + const char *severity = opt.check_cert ? _("ERROR") : _("WARNING"); + bool success = true; + bool pinsuccess = opt.pinnedpubkey == NULL; + + /* The user explicitly said to not check for the certificate. */ + if (opt.check_cert == CHECK_CERT_QUIET && pinsuccess) + return success; + + err = gnutls_certificate_verify_peers2 (ctx->session, &status); + if (err < 0) + { + logprintf (LOG_NOTQUIET, _("%s: No certificate presented by %s.\n"), + severity, quotearg_style (escape_quoting_style, host)); + success = false; + goto out; + } + + _CHECK_CERT (GNUTLS_CERT_INVALID, _("%s: The certificate of %s is not trusted.\n")); + _CHECK_CERT (GNUTLS_CERT_SIGNER_NOT_FOUND, _("%s: The certificate of %s doesn't have a known issuer.\n")); + _CHECK_CERT (GNUTLS_CERT_REVOKED, _("%s: The certificate of %s has been revoked.\n")); + _CHECK_CERT (GNUTLS_CERT_SIGNER_NOT_CA, _("%s: The certificate signer of %s was not a CA.\n")); + _CHECK_CERT (GNUTLS_CERT_INSECURE_ALGORITHM, _("%s: The certificate of %s was signed using an insecure algorithm.\n")); + _CHECK_CERT (GNUTLS_CERT_NOT_ACTIVATED, _("%s: The certificate of %s is not yet activated.\n")); + _CHECK_CERT (GNUTLS_CERT_EXPIRED, _("%s: The certificate of %s has expired.\n")); + + if (gnutls_certificate_type_get (ctx->session) == GNUTLS_CRT_X509) + { + time_t now = time (NULL); + gnutls_x509_crt_t cert; + const gnutls_datum_t *cert_list; + unsigned int cert_list_size; + const char *sni_hostname; + + if ((err = gnutls_x509_crt_init (&cert)) < 0) + { + logprintf (LOG_NOTQUIET, _("Error initializing X509 certificate: %s\n"), + gnutls_strerror (err)); + success = false; + goto out; + } + + cert_list = gnutls_certificate_get_peers (ctx->session, &cert_list_size); + if (!cert_list) + { + logprintf (LOG_NOTQUIET, _("No certificate found\n")); + success = false; + goto crt_deinit; + } + err = gnutls_x509_crt_import (cert, cert_list, GNUTLS_X509_FMT_DER); + if (err < 0) + { + logprintf (LOG_NOTQUIET, _("Error parsing certificate: %s\n"), + gnutls_strerror (err)); + success = false; + goto crt_deinit; + } + if (now < gnutls_x509_crt_get_activation_time (cert)) + { + logprintf (LOG_NOTQUIET, _("The certificate has not yet been activated\n")); + success = false; + } + if (now >= gnutls_x509_crt_get_expiration_time (cert)) + { + logprintf (LOG_NOTQUIET, _("The certificate has expired\n")); + success = false; + } + sni_hostname = _sni_hostname(host); + if (!gnutls_x509_crt_check_hostname (cert, sni_hostname)) + { + logprintf (LOG_NOTQUIET, + _("The certificate's owner does not match hostname %s\n"), + quote (sni_hostname)); + success = false; + } + xfree(sni_hostname); + + pinsuccess = pkp_pin_peer_pubkey (cert, opt.pinnedpubkey); + if (!pinsuccess) + { + logprintf (LOG_ALWAYS, _("The public key does not match pinned public key!\n")); + success = false; + } + + crt_deinit: + gnutls_x509_crt_deinit (cert); + } + else + { + logprintf (LOG_NOTQUIET, _("Certificate must be X.509\n")); + success = false; + } + + out: + /* never return true if pinsuccess fails */ + return !pinsuccess ? false : (opt.check_cert == CHECK_CERT_ON ? success : true); +} diff --git a/src/hash.c b/src/hash.c new file mode 100644 index 0000000..6609528 --- /dev/null +++ b/src/hash.c @@ -0,0 +1,813 @@ +/* Hash tables. + Copyright (C) 2000-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +/* With -DSTANDALONE, this file can be compiled outside Wget source + tree. To test, also use -DTEST. */ + +#ifndef STANDALONE +# include "wget.h" +#endif + +#include <stdio.h> +#include <stdlib.h> +#include <assert.h> +#include <string.h> +#include <limits.h> + +#ifndef STANDALONE +/* Get Wget's utility headers. */ +# include "utils.h" +#else +/* Make do without them. */ +# define xnew(type) (xmalloc (sizeof (type))) +# define xnew0(type) (xcalloc (1, sizeof (type))) +# define xnew_array(type, len) (xmalloc ((len) * sizeof (type))) +# define xfree(p) do { free ((void *) (p)); p = NULL; } while (0) + +# ifndef countof +# define countof(x) (sizeof (x) / sizeof ((x)[0])) +# endif +# include <ctype.h> +# define c_tolower(x) tolower ((unsigned char) (x)) +# include <stdint.h> +#endif + +#include "hash.h" + +/* INTERFACE: + + Hash tables are a technique used to implement mapping between + objects with near-constant-time access and storage. The table + associates keys to values, and a value can be very quickly + retrieved by providing the key. Fast lookup tables are typically + implemented as hash tables. + + The entry points are + hash_table_new -- creates the table. + hash_table_destroy -- destroys the table. + hash_table_put -- establishes or updates key->value mapping. + hash_table_get -- retrieves value of key. + hash_table_get_pair -- get key/value pair for key. + hash_table_contains -- test whether the table contains key. + hash_table_remove -- remove key->value mapping for given key. + hash_table_for_each -- call function for each table entry. + hash_table_iterate -- iterate over entries in hash table. + hash_table_iter_next -- return next element during iteration. + hash_table_clear -- clear hash table contents. + hash_table_count -- return the number of entries in the table. + + The hash table grows internally as new entries are added and is not + limited in size, except by available memory. The table doubles + with each resize, which ensures that the amortized time per + operation remains constant. + + If not instructed otherwise, tables created by hash_table_new + consider the keys to be equal if their pointer values are the same. + You can use make_string_hash_table to create tables whose keys are + considered equal if their string contents are the same. In the + general case, the criterion of equality used to compare keys is + specified at table creation time with two callback functions, + "hash" and "test". The hash function transforms the key into an + arbitrary number that must be the same for two equal keys. The + test function accepts two keys and returns non-zero if they are to + be considered equal. + + Note that neither keys nor values are copied when inserted into the + hash table, so they must exist for the lifetime of the table. This + means that e.g. the use of static strings is OK, but objects with a + shorter life-time probably need to be copied (with strdup() or the + like in the case of strings) before being inserted. */ + +/* IMPLEMENTATION: + + The hash table is implemented as an open-addressed table with + linear probing collision resolution. + + The above means that all the cells (each cell containing a key and + a value pointer) are stored in a contiguous array. Array position + of each cell is determined by the hash value of its key and the + size of the table: location := hash(key) % size. If two different + keys end up on the same position (collide), the one that came + second is stored in the first unoccupied cell that follows it. + This collision resolution technique is called "linear probing". + + There are more advanced collision resolution methods (quadratic + probing, double hashing), but we don't use them because they incur + more non-sequential access to the array, which results in worse CPU + cache behavior. Linear probing works well as long as the + count/size ratio (fullness) is kept below 75%. We make sure to + grow and rehash the table whenever this threshold is exceeded. + + Collisions complicate deletion because simply clearing a cell + followed by previously collided entries would cause those neighbors + to not be picked up by find_cell later. One solution is to leave a + "tombstone" marker instead of clearing the cell, and another is to + recalculate the positions of adjacent cells. We take the latter + approach because it results in less bookkeeping garbage and faster + retrieval at the (slight) expense of deletion. */ + +/* Maximum allowed fullness: when hash table's fullness exceeds this + value, the table is resized. */ +#define HASH_MAX_FULLNESS 0.75 + +/* The hash table size is multiplied by this factor (and then rounded + to the next prime) with each resize. This guarantees infrequent + resizes. */ +#define HASH_RESIZE_FACTOR 2 + +struct cell { + void *key; + void *value; +}; + +typedef unsigned long (*hashfun_t) (const void *); +typedef int (*testfun_t) (const void *, const void *); + +struct hash_table { + hashfun_t hash_function; + testfun_t test_function; + + struct cell *cells; /* contiguous array of cells. */ + int size; /* size of the array. */ + + int count; /* number of occupied entries. */ + int resize_threshold; /* after size exceeds this number of + entries, resize the table. */ + int prime_offset; /* the offset of the current prime in + the prime table. */ +}; + +/* We use the all-bits-set constant (INVALID_PTR) marker to mean that + a cell is empty. It is unaligned and therefore illegal as a + pointer. INVALID_PTR_CHAR (0xff) is the single-character constant + used to initialize the entire cells array as empty. + + The all-bits-set value is a better choice than NULL because it + allows the use of NULL/0 keys. Since the keys are either integers + or pointers, the only key that cannot be used is the integer value + -1. This is acceptable because it still allows the use of + nonnegative integer keys. */ + +#define INVALID_PTR ((void *) ~(uintptr_t) 0) +#ifndef UCHAR_MAX +# define UCHAR_MAX 0xff +#endif +#define INVALID_PTR_CHAR UCHAR_MAX + +/* Whether the cell C is occupied (non-empty). */ +#define CELL_OCCUPIED(c) ((c)->key != INVALID_PTR) + +/* Clear the cell C, i.e. mark it as empty (unoccupied). */ +#define CLEAR_CELL(c) ((c)->key = INVALID_PTR) + +/* "Next" cell is the cell following C, but wrapping back to CELLS + when C would reach CELLS+SIZE. */ +#define NEXT_CELL(c, cells, size) (c != cells + (size - 1) ? c + 1 : cells) + +/* Loop over occupied cells starting at C, terminating the loop when + an empty cell is encountered. */ +#define FOREACH_OCCUPIED_ADJACENT(c, cells, size) \ + for (; CELL_OCCUPIED (c); c = NEXT_CELL (c, cells, size)) + +/* Return the position of KEY in hash table SIZE large, hash function + being HASHFUN. */ +#define HASH_POSITION(key, hashfun, size) ((hashfun) (key) % size) + +/* Find a prime near, but greater than or equal to SIZE. The primes + are looked up from a table with a selection of primes convenient + for this purpose. + + PRIME_OFFSET is a minor optimization: it specifies start position + for the search for the large enough prime. The final offset is + stored in the same variable. That way the list of primes does not + have to be scanned from the beginning each time around. */ + +static int +prime_size (int size, int *prime_offset) +{ + static const int primes[] = { + 13, 19, 29, 41, 59, 79, 107, 149, 197, 263, 347, 457, 599, 787, 1031, + 1361, 1777, 2333, 3037, 3967, 5167, 6719, 8737, 11369, 14783, + 19219, 24989, 32491, 42257, 54941, 71429, 92861, 120721, 156941, + 204047, 265271, 344857, 448321, 582821, 757693, 985003, 1280519, + 1664681, 2164111, 2813353, 3657361, 4754591, 6180989, 8035301, + 10445899, 13579681, 17653589, 22949669, 29834603, 38784989, + 50420551, 65546729, 85210757, 110774011, 144006217, 187208107, + 243370577, 316381771, 411296309, 534685237, 695090819, 903618083, + 1174703521, 1527114613, 1837299131, 2147483647 + }; + size_t i; + + for (i = *prime_offset; i < countof (primes); i++) + if (primes[i] >= size) + { + /* Set the offset to the next prime. That is safe because, + next time we are called, it will be with a larger SIZE, + which means we could never return the same prime anyway. + (If that is not the case, the caller can simply reset + *prime_offset.) */ + *prime_offset = i + 1; + return primes[i]; + } + + abort (); +} + +static int cmp_pointer (const void *, const void *); + +/* Create a hash table with hash function HASH_FUNCTION and test + function TEST_FUNCTION. The table is empty (its count is 0), but + pre-allocated to store at least ITEMS items. + + ITEMS is the number of items that the table can accept without + needing to resize. It is useful when creating a table that is to + be immediately filled with a known number of items. In that case, + the regrows are a waste of time, and specifying ITEMS correctly + will avoid them altogether. + + Note that hash tables grow dynamically regardless of ITEMS. The + only use of ITEMS is to preallocate the table and avoid unnecessary + dynamic regrows. Don't bother making ITEMS prime because it's not + used as size unchanged. To start with a small table that grows as + needed, simply specify zero ITEMS. + + If hash and test callbacks are not specified, identity mapping is + assumed, i.e. pointer values are used for key comparison. (Common + Lisp calls such tables EQ hash tables, and Java calls them + IdentityHashMaps.) If your keys require different comparison, + specify hash and test functions. For easy use of C strings as hash + keys, you can use the convenience functions make_string_hash_table + and make_nocase_string_hash_table. */ + +struct hash_table * +hash_table_new (int items, + unsigned long (*hash_function) (const void *), + int (*test_function) (const void *, const void *)) +{ + int size; + struct hash_table *ht = xnew (struct hash_table); + + ht->hash_function = hash_function ? hash_function : hash_pointer; + ht->test_function = test_function ? test_function : cmp_pointer; + + /* If the size of struct hash_table ever becomes a concern, this + field can go. (Wget doesn't create many hashes.) */ + ht->prime_offset = 0; + + /* Calculate the size that ensures that the table will store at + least ITEMS keys without the need to resize. */ + size = (int) (1 + items / HASH_MAX_FULLNESS); + size = prime_size (size, &ht->prime_offset); + ht->size = size; + ht->resize_threshold = (int) (size * HASH_MAX_FULLNESS); + /*assert (ht->resize_threshold >= items);*/ + + ht->cells = xnew_array (struct cell, ht->size); + + /* Mark cells as empty. We use 0xff rather than 0 to mark empty + keys because it allows us to use NULL/0 as keys. */ + memset (ht->cells, INVALID_PTR_CHAR, size * sizeof (struct cell)); + + ht->count = 0; + + return ht; +} + +/* Free the data associated with hash table HT. */ + +void +hash_table_destroy (struct hash_table *ht) +{ + xfree (ht->cells); + xfree (ht); +} + +/* The heart of most functions in this file -- find the cell whose + KEY is equal to key, using linear probing. Returns the cell + that matches KEY, or the first empty cell if none matches. */ + +static inline struct cell * +find_cell (const struct hash_table *ht, const void *key) +{ + struct cell *cells = ht->cells; + int size = ht->size; + struct cell *c = cells + HASH_POSITION (key, ht->hash_function, size); + testfun_t equals = ht->test_function; + + FOREACH_OCCUPIED_ADJACENT (c, cells, size) + if (equals (key, c->key)) + break; + return c; +} + +/* Get the value that corresponds to the key KEY in the hash table HT. + If no value is found, return NULL. Note that NULL is a legal value + for value; if you are storing NULLs in your hash table, you can use + hash_table_contains to be sure that a (possibly NULL) value exists + in the table. Or, you can use hash_table_get_pair instead of this + function. */ + +void * +hash_table_get (const struct hash_table *ht, const void *key) +{ + struct cell *c = find_cell (ht, key); + if (CELL_OCCUPIED (c)) + return c->value; + else + return NULL; +} + +/* Like hash_table_get, but writes out the pointers to both key and + value. Returns non-zero on success. */ + +int +hash_table_get_pair (const struct hash_table *ht, const void *lookup_key, + void *orig_key, void *value) +{ + struct cell *c = find_cell (ht, lookup_key); + if (CELL_OCCUPIED (c)) + { + if (orig_key) + *(void **)orig_key = c->key; + if (value) + *(void **)value = c->value; + return 1; + } + else + return 0; +} + +/* Return 1 if HT contains KEY, 0 otherwise. */ + +int +hash_table_contains (const struct hash_table *ht, const void *key) +{ + struct cell *c = find_cell (ht, key); + return CELL_OCCUPIED (c); +} + +/* Grow hash table HT as necessary, and rehash all the key-value + mappings. */ + +static void +grow_hash_table (struct hash_table *ht) +{ + hashfun_t hasher = ht->hash_function; + struct cell *old_cells = ht->cells; + struct cell *old_end = ht->cells + ht->size; + struct cell *c, *cells; + int newsize; + + newsize = prime_size (ht->size * HASH_RESIZE_FACTOR, &ht->prime_offset); +#if 0 + printf ("growing from %d to %d; fullness %.2f%% to %.2f%%\n", + ht->size, newsize, + 100.0 * ht->count / ht->size, + 100.0 * ht->count / newsize); +#endif + + ht->size = newsize; + ht->resize_threshold = (int) (newsize * HASH_MAX_FULLNESS); + + cells = xnew_array (struct cell, newsize); + memset (cells, INVALID_PTR_CHAR, newsize * sizeof (struct cell)); + ht->cells = cells; + + for (c = old_cells; c < old_end; c++) + if (CELL_OCCUPIED (c)) + { + struct cell *new_c; + /* We don't need to test for uniqueness of keys because they + come from the hash table and are therefore known to be + unique. */ + new_c = cells + HASH_POSITION (c->key, hasher, newsize); + FOREACH_OCCUPIED_ADJACENT (new_c, cells, newsize) + ; + *new_c = *c; + } + + xfree (old_cells); +} + +/* Put VALUE in the hash table HT under the key KEY. This regrows the + table if necessary. */ + +void +hash_table_put (struct hash_table *ht, const void *key, const void *value) +{ + struct cell *c = find_cell (ht, key); + if (CELL_OCCUPIED (c)) + { + /* update existing item */ + c->key = (void *)key; /* const? */ + c->value = (void *)value; + return; + } + + /* If adding the item would make the table exceed max. fullness, + grow the table first. */ + if (ht->count >= ht->resize_threshold) + { + grow_hash_table (ht); + c = find_cell (ht, key); + } + + /* add new item */ + ++ht->count; + c->key = (void *)key; /* const? */ + c->value = (void *)value; +} + +/* Remove KEY->value mapping from HT. Return 0 if there was no such + entry; return 1 if an entry was removed. */ + +int +hash_table_remove (struct hash_table *ht, const void *key) +{ + struct cell *c = find_cell (ht, key); + if (!CELL_OCCUPIED (c)) + return 0; + else + { + int size = ht->size; + struct cell *cells = ht->cells; + hashfun_t hasher = ht->hash_function; + + CLEAR_CELL (c); + --ht->count; + + /* Rehash all the entries following C. The alternative + approach is to mark the entry as deleted, i.e. create a + "tombstone". That speeds up removal, but leaves a lot of + garbage and slows down hash_table_get and hash_table_put. */ + + c = NEXT_CELL (c, cells, size); + FOREACH_OCCUPIED_ADJACENT (c, cells, size) + { + const void *key2 = c->key; + struct cell *c_new; + + /* Find the new location for the key. */ + c_new = cells + HASH_POSITION (key2, hasher, size); + FOREACH_OCCUPIED_ADJACENT (c_new, cells, size) + if (key2 == c_new->key) + /* The cell C (key2) is already where we want it (in + C_NEW's "chain" of keys.) */ + goto next_rehash; + + *c_new = *c; + CLEAR_CELL (c); + + next_rehash: + ; + } + return 1; + } +} + +/* Clear HT of all entries. After calling this function, the count + and the fullness of the hash table will be zero. The size will + remain unchanged. */ + +void +hash_table_clear (struct hash_table *ht) +{ + memset (ht->cells, INVALID_PTR_CHAR, ht->size * sizeof (struct cell)); + ht->count = 0; +} + +/* Call FN for each entry in HT. FN is called with three arguments: + the key, the value, and ARG. When FN returns a non-zero value, the + mapping stops. + + It is undefined what happens if you add or remove entries in the + hash table while hash_table_for_each is running. The exception is + the entry you're currently mapping over; you may call + hash_table_put or hash_table_remove on that entry's key. That is + also the reason why this function cannot be implemented in terms of + hash_table_iterate. */ + +void +hash_table_for_each (struct hash_table *ht, + int (*fn) (void *, void *, void *), void *arg) +{ + struct cell *c = ht->cells; + struct cell *end = ht->cells + ht->size; + + for (; c < end; c++) + if (CELL_OCCUPIED (c)) + { + void *key; + repeat: + key = c->key; + if (fn (key, c->value, arg)) + return; + /* hash_table_remove might have moved the adjacent cells. */ + if (c->key != key && CELL_OCCUPIED (c)) + goto repeat; + } +} + +/* Initiate iteration over HT. Entries are obtained with + hash_table_iter_next, a typical iteration loop looking like this: + + hash_table_iterator iter; + for (hash_table_iterate (ht, &iter); hash_table_iter_next (&iter); ) + ... do something with iter.key and iter.value ... + + The iterator does not need to be deallocated after use. The hash + table must not be modified while being iterated over. */ + +void +hash_table_iterate (struct hash_table *ht, hash_table_iterator *iter) +{ + iter->pos = ht->cells; + iter->end = ht->cells + ht->size; +} + +/* Get the next hash table entry. ITER is an iterator object + initialized using hash_table_iterate. While there are more + entries, the key and value pointers are stored to ITER->key and + ITER->value respectively and 1 is returned. When there are no more + entries, 0 is returned. + + If the hash table is modified between calls to this function, the + result is undefined. */ + +int +hash_table_iter_next (hash_table_iterator *iter) +{ + struct cell *c = iter->pos; + struct cell *end = iter->end; + for (; c < end; c++) + if (CELL_OCCUPIED (c)) + { + iter->key = c->key; + iter->value = c->value; + iter->pos = c + 1; + return 1; + } + return 0; +} + +/* Return the number of elements in the hash table. This is not the + same as the physical size of the hash table, which is always + greater than the number of elements. */ + +int +hash_table_count (const struct hash_table *ht) +{ + return ht->count; +} + +/* Functions from this point onward are meant for convenience and + don't strictly belong to this file. However, this is as good a + place for them as any. */ + +/* Guidelines for creating custom hash and test functions: + + - The test function returns non-zero for keys that are considered + "equal", zero otherwise. + + - The hash function returns a number that represents the + "distinctness" of the object. In more precise terms, it means + that for any two objects that test "equal" under the test + function, the hash function MUST produce the same result. + + This does not mean that all different objects must produce + different values (that would be "perfect" hashing), only that + non-distinct objects must produce the same values! For instance, + a hash function that returns 0 for any given object is a + perfectly valid (albeit extremely bad) hash function. A hash + function that hashes a string by adding up all its characters is + another example of a valid (but still quite bad) hash function. + + It is not hard to make hash and test functions agree about + equality. For example, if the test function compares strings + case-insensitively, the hash function can lower-case the + characters when calculating the hash value. That ensures that + two strings differing only in case will hash the same. + + - To prevent performance degradation, choose a hash function with + as good "spreading" as possible. A good hash function will use + all the bits of the input when calculating the hash, and will + react to even small changes in input with a completely different + output. But don't make the hash function itself overly slow, + because you'll be incurring a non-negligible overhead to all hash + table operations. */ + +/* + * Support for hash tables whose keys are strings. + * + */ + +/* Base 31 hash function. Taken from Gnome's glib, modified to use + standard C types. + + We used to use the popular hash function from the Dragon Book, but + this one seems to perform much better, both by being faster and by + generating less collisions. */ + +#ifdef __clang__ +__attribute__((no_sanitize("integer"))) +#endif +static unsigned long +hash_string (const void *key) +{ + const char *p = key; + unsigned int h = *p; + + if (h) + for (p += 1; *p != '\0'; p++) + h = (h << 5) - h + *p; + + return h; +} + +/* Frontend for strcmp usable for hash tables. */ + +static int +cmp_string (const void *s1, const void *s2) +{ + return !strcmp ((const char *)s1, (const char *)s2); +} + +/* Return a hash table of preallocated to store at least ITEMS items + suitable to use strings as keys. */ + +struct hash_table * +make_string_hash_table (int items) +{ + return hash_table_new (items, hash_string, cmp_string); +} + +/* + * Support for hash tables whose keys are strings, but which are + * compared case-insensitively. + * + */ + +/* Like hash_string, but produce the same hash regardless of the case. */ + +#ifdef __clang__ +__attribute__((no_sanitize("integer"))) +#endif +static unsigned long +hash_string_nocase (const void *key) +{ + const char *p = key; + unsigned int h = c_tolower (*p); + + if (h) + for (p += 1; *p != '\0'; p++) + h = (h << 5) - h + c_tolower (*p); + + return h; +} + +/* Like string_cmp, but doing case-insensitive comparison. */ + +static int +string_cmp_nocase (const void *s1, const void *s2) +{ + return !strcasecmp ((const char *)s1, (const char *)s2); +} + +/* Like make_string_hash_table, but uses string_hash_nocase and + string_cmp_nocase. */ + +struct hash_table * +make_nocase_string_hash_table (int items) +{ + return hash_table_new (items, hash_string_nocase, string_cmp_nocase); +} + +/* Hashing of numeric values, such as pointers and integers. + + This implementation is the Robert Jenkins' 32 bit Mix Function, + with a simple adaptation for 64-bit values. According to Jenkins + it should offer excellent spreading of values. Unlike the popular + Knuth's multiplication hash, this function doesn't need to know the + hash table size to work. */ + +#ifdef __clang__ +__attribute__((no_sanitize("integer"))) +#endif +unsigned long +hash_pointer (const void *ptr) +{ + uintptr_t key = (uintptr_t) ptr; + key += (key << 12); + key ^= (key >> 22); + key += (key << 4); + key ^= (key >> 9); + key += (key << 10); + key ^= (key >> 2); + key += (key << 7); + key ^= (key >> 12); +#if SIZEOF_VOID_P > 4 + key += (key << 44); + key ^= (key >> 54); + key += (key << 36); + key ^= (key >> 41); + key += (key << 42); + key ^= (key >> 34); + key += (key << 39); + key ^= (key >> 44); +#endif + return (unsigned long) key; +} + +static int +cmp_pointer (const void *ptr1, const void *ptr2) +{ + return ptr1 == ptr2; +} + +#ifdef TEST + +#include <stdio.h> +#include <string.h> + +void +print_hash (struct hash_table *sht) +{ + hash_table_iterator iter; + int count = 0; + + for (hash_table_iterate (sht, &iter); hash_table_iter_next (&iter); + ++count) + printf ("%s: %s\n", iter.key, iter.value); + assert (count == sht->count); +} + +int +main (void) +{ + struct hash_table *ht = make_string_hash_table (0); + char line[80]; + +#ifdef ENABLE_NLS + /* Set the current locale. */ + setlocale (LC_ALL, ""); + /* Set the text message domain. */ + bindtextdomain ("wget", LOCALEDIR); + textdomain ("wget"); +#endif /* ENABLE_NLS */ + + while ((fgets (line, sizeof (line), stdin))) + { + int len = strlen (line); + if (len <= 1) + continue; + line[--len] = '\0'; + if (!hash_table_contains (ht, line)) + hash_table_put (ht, strdup (line), "here I am!"); +#if 1 + if (len % 5 == 0) + { + char *line_copy; + if (hash_table_get_pair (ht, line, &line_copy, NULL)) + { + hash_table_remove (ht, line); + xfree (line_copy); + } + } +#endif + } +#if 0 + print_hash (ht); +#endif +#if 1 + printf ("%d %d\n", ht->count, ht->size); +#endif + return 0; +} +#endif /* TEST */ diff --git a/src/hash.h b/src/hash.h new file mode 100644 index 0000000..b8f7965 --- /dev/null +++ b/src/hash.h @@ -0,0 +1,66 @@ +/* Hash table declarations. + Copyright (C) 2000, 2007-2011, 2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef HASH_H +#define HASH_H + +struct hash_table; + +struct hash_table *hash_table_new (int, unsigned long (*) (const void *), + int (*) (const void *, const void *)); +void hash_table_destroy (struct hash_table *); + +void *hash_table_get (const struct hash_table *, const void *); +int hash_table_get_pair (const struct hash_table *, const void *, + void *, void *); +int hash_table_contains (const struct hash_table *, const void *); + +void hash_table_put (struct hash_table *, const void *, const void *); +int hash_table_remove (struct hash_table *, const void *); +void hash_table_clear (struct hash_table *); + +void hash_table_for_each (struct hash_table *, + int (*) (void *, void *, void *), void *); + +typedef struct { + void *key, *value; /* public members */ + void *pos, *end; /* private members */ +} hash_table_iterator; +void hash_table_iterate (struct hash_table *, hash_table_iterator *); +int hash_table_iter_next (hash_table_iterator *); + +int hash_table_count (const struct hash_table *); + +struct hash_table *make_string_hash_table (int); +struct hash_table *make_nocase_string_hash_table (int); + +unsigned long hash_pointer (const void *); + +#endif /* HASH_H */ diff --git a/src/host.c b/src/host.c new file mode 100644 index 0000000..353c1cc --- /dev/null +++ b/src/host.c @@ -0,0 +1,1082 @@ +/* Host name resolution and matching. + Copyright (C) 1996-2012, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#ifndef WINDOWS +# include <sys/types.h> +# include <sys/socket.h> +# include <netinet/in.h> +# ifndef __BEOS__ +# include <arpa/inet.h> +# endif +# ifdef __VMS +# include "vms_ip.h" +# else /* def __VMS */ +# include <netdb.h> +# endif /* def __VMS [else] */ +# define SET_H_ERRNO(err) ((void)(h_errno = (err))) +#else /* WINDOWS */ +# include <winsock2.h> +# include <ws2tcpip.h> +# define SET_H_ERRNO(err) WSASetLastError (err) +#endif /* WINDOWS */ + +#include <errno.h> + +#include "utils.h" +#include "host.h" +#include "url.h" +#include "hash.h" +#include "ptimer.h" + +#ifndef NO_ADDRESS +# define NO_ADDRESS NO_DATA +#endif + +#if !HAVE_DECL_H_ERRNO && !defined(WINDOWS) +extern int h_errno; +#endif + + +/* Lists of IP addresses that result from running DNS queries. See + lookup_host for details. */ + +struct address_list { + int count; /* number of addresses */ + ip_address *addresses; /* pointer to the string of addresses */ + + int faulty; /* number of addresses known not to work. */ + bool connected; /* whether we were able to connect to + one of the addresses in the list, + at least once. */ + + int refcount; /* reference count; when it drops to + 0, the entry is freed. */ +}; + +/* Get the bounds of the address list. */ + +void +address_list_get_bounds (const struct address_list *al, int *start, int *end) +{ + *start = al->faulty; + *end = al->count; +} + +/* Return a pointer to the address at position POS. */ + +const ip_address * +address_list_address_at (const struct address_list *al, int pos) +{ + assert (pos >= al->faulty && pos < al->count); + return al->addresses + pos; +} + +/* Return true if AL contains IP, false otherwise. */ + +bool +address_list_contains (const struct address_list *al, const ip_address *ip) +{ + int i; + switch (ip->family) + { + case AF_INET: + for (i = 0; i < al->count; i++) + { + ip_address *cur = al->addresses + i; + if (cur->family == AF_INET + && (cur->data.d4.s_addr == ip->data.d4.s_addr)) + return true; + } + return false; +#ifdef ENABLE_IPV6 + case AF_INET6: + for (i = 0; i < al->count; i++) + { + ip_address *cur = al->addresses + i; + if (cur->family == AF_INET6 +#ifdef HAVE_SOCKADDR_IN6_SCOPE_ID + && cur->ipv6_scope == ip->ipv6_scope +#endif + && IN6_ARE_ADDR_EQUAL (&cur->data.d6, &ip->data.d6)) + return true; + } + return false; +#endif /* ENABLE_IPV6 */ + default: + abort (); + } +} + +/* Mark the INDEXth element of AL as faulty, so that the next time + this address list is used, the faulty element will be skipped. */ + +void +address_list_set_faulty (struct address_list *al, int index) +{ + /* We assume that the address list is traversed in order, so that a + "faulty" attempt is always preceded with all-faulty addresses, + and this is how Wget uses it. */ + assert (index == al->faulty); + if (index != al->faulty) + { + logprintf (LOG_ALWAYS, "index: %d\nal->faulty: %d\n", index, al->faulty); + logprintf (LOG_ALWAYS, _("Error in handling the address list.\n")); + logprintf (LOG_ALWAYS, _("Please report this issue to bug-wget@gnu.org\n")); + abort(); + } + + ++al->faulty; + if (al->faulty >= al->count) + /* All addresses have been proven faulty. Since there's not much + sense in returning the user an empty address list the next + time, we'll rather make them all clean, so that they can be + retried anew. */ + al->faulty = 0; +} + +/* Set the "connected" flag to true. This flag used by connect.c to + see if the host perhaps needs to be resolved again. */ + +void +address_list_set_connected (struct address_list *al) +{ + al->connected = true; +} + +/* Return the value of the "connected" flag. */ + +bool +address_list_connected_p (const struct address_list *al) +{ + return al->connected; +} + +#ifdef ENABLE_IPV6 + +/* Create an address_list from the addresses in the given struct + addrinfo. */ + +static struct address_list * +address_list_from_addrinfo (const struct addrinfo *ai) +{ + struct address_list *al; + const struct addrinfo *ptr; + int cnt; + ip_address *ip; + + cnt = 0; + for (ptr = ai; ptr != NULL ; ptr = ptr->ai_next) + if (ptr->ai_family == AF_INET || ptr->ai_family == AF_INET6) + ++cnt; + if (cnt == 0) + return NULL; + + al = xnew0 (struct address_list); + al->addresses = xnew_array (ip_address, cnt); + al->count = cnt; + al->refcount = 1; + + ip = al->addresses; + for (ptr = ai; ptr != NULL; ptr = ptr->ai_next) + if (ptr->ai_family == AF_INET6) + { + const struct sockaddr_in6 *sin6 = + (const struct sockaddr_in6 *)ptr->ai_addr; + ip->family = AF_INET6; + ip->data.d6 = sin6->sin6_addr; +#ifdef HAVE_SOCKADDR_IN6_SCOPE_ID + ip->ipv6_scope = sin6->sin6_scope_id; +#endif + ++ip; + } + else if (ptr->ai_family == AF_INET) + { + const struct sockaddr_in *sin = + (const struct sockaddr_in *)ptr->ai_addr; + ip->family = AF_INET; + ip->data.d4 = sin->sin_addr; + ++ip; + } + assert (ip - al->addresses == cnt); + return al; +} + +#define IS_IPV4(addr) (((const ip_address *) addr)->family == AF_INET) + +/* Compare two IP addresses by family, giving preference to the IPv4 + address (sorting it first). In other words, return -1 if ADDR1 is + IPv4 and ADDR2 is IPv6, +1 if ADDR1 is IPv6 and ADDR2 is IPv4, and + 0 otherwise. + + This is intended to be used as the comparator arg to a qsort-like + sorting function, which is why it accepts generic pointers. */ + +static int +cmp_prefer_ipv4 (const void *addr1, const void *addr2) +{ + return !IS_IPV4 (addr1) - !IS_IPV4 (addr2); +} + +#define IS_IPV6(addr) (((const ip_address *) addr)->family == AF_INET6) + +/* Like the above, but give preference to the IPv6 address. */ + +static int +cmp_prefer_ipv6 (const void *addr1, const void *addr2) +{ + return !IS_IPV6 (addr1) - !IS_IPV6 (addr2); +} + +#else /* not ENABLE_IPV6 */ + +/* Create an address_list from a NULL-terminated vector of IPv4 + addresses. This kind of vector is returned by gethostbyname. */ + +static struct address_list * +address_list_from_ipv4_addresses (char **vec) +{ + int count, i; + struct address_list *al = xnew0 (struct address_list); + + count = 0; + while (vec[count]) + ++count; + assert (count > 0); + + al->addresses = xnew_array (ip_address, count); + al->count = count; + al->refcount = 1; + + for (i = 0; i < count; i++) + { + ip_address *ip = &al->addresses[i]; + ip->family = AF_INET; + memcpy (IP_INADDR_DATA (ip), vec[i], 4); + } + + return al; +} + +#endif /* not ENABLE_IPV6 */ + +static void +address_list_delete (struct address_list *al) +{ + xfree (al->addresses); + xfree (al); +} + +/* Mark the address list as being no longer in use. This will reduce + its reference count which will cause the list to be freed when the + count reaches 0. */ + +void +address_list_release (struct address_list *al) +{ + --al->refcount; + DEBUGP (("Releasing 0x%0*lx (new refcount %d).\n", PTR_FORMAT (al), + al->refcount)); + if (al->refcount <= 0) + { + DEBUGP (("Deleting unused 0x%0*lx.\n", PTR_FORMAT (al))); + address_list_delete (al); + } +} + +/* Versions of gethostbyname and getaddrinfo that support timeout. */ + +#ifndef ENABLE_IPV6 + +struct ghbnwt_context { + const char *host_name; + struct hostent *hptr; +}; + +static void +gethostbyname_with_timeout_callback (void *arg) +{ + struct ghbnwt_context *ctx = (struct ghbnwt_context *)arg; + ctx->hptr = gethostbyname (ctx->host_name); +} + +/* Just like gethostbyname, except it times out after TIMEOUT seconds. + In case of timeout, NULL is returned and errno is set to ETIMEDOUT. + The function makes sure that when NULL is returned for reasons + other than timeout, errno is reset. */ + +static struct hostent * +gethostbyname_with_timeout (const char *host_name, double timeout) +{ + struct ghbnwt_context ctx; + ctx.host_name = host_name; + if (run_with_timeout (timeout, gethostbyname_with_timeout_callback, &ctx)) + { + SET_H_ERRNO (HOST_NOT_FOUND); + errno = ETIMEDOUT; + return NULL; + } + if (!ctx.hptr) + errno = 0; + return ctx.hptr; +} + +/* Print error messages for host errors. */ +static const char * +host_errstr (int error) +{ + /* Can't use switch since some of these constants can be equal, + which makes the compiler complain about duplicate case + values. */ + if (error == HOST_NOT_FOUND + || error == NO_RECOVERY + || error == NO_DATA + || error == NO_ADDRESS) + return _("Unknown host"); + else if (error == TRY_AGAIN) + /* Message modeled after what gai_strerror returns in similar + circumstances. */ + return _("Temporary failure in name resolution"); + else + return _("Unknown error"); +} + +#else /* ENABLE_IPV6 */ + +struct gaiwt_context { + const char *node; + const char *service; + const struct addrinfo *hints; + struct addrinfo **res; + int exit_code; +}; + +static void +getaddrinfo_with_timeout_callback (void *arg) +{ + struct gaiwt_context *ctx = (struct gaiwt_context *)arg; + ctx->exit_code = getaddrinfo (ctx->node, ctx->service, ctx->hints, ctx->res); +} + +/* Just like getaddrinfo, except it times out after TIMEOUT seconds. + In case of timeout, the EAI_SYSTEM error code is returned and errno + is set to ETIMEDOUT. */ + +static int +getaddrinfo_with_timeout (const char *node, const char *service, + const struct addrinfo *hints, struct addrinfo **res, + double timeout) +{ + struct gaiwt_context ctx; + ctx.node = node; + ctx.service = service; + ctx.hints = hints; + ctx.res = res; + + if (run_with_timeout (timeout, getaddrinfo_with_timeout_callback, &ctx)) + { + errno = ETIMEDOUT; + return EAI_SYSTEM; + } + return ctx.exit_code; +} + +#endif /* ENABLE_IPV6 */ + +/* Return a textual representation of ADDR, i.e. the dotted quad for + IPv4 addresses, and the colon-separated list of hex words (with all + zeros omitted, etc.) for IPv6 addresses. */ + +const char * +print_address (const ip_address *addr) +{ + static char buf[64]; + + if (!inet_ntop (addr->family, IP_INADDR_DATA (addr), buf, sizeof buf)) + snprintf (buf, sizeof buf, "<error: %s>", strerror (errno)); + + return buf; +} + +/* The following two functions were adapted from glibc's + implementation of inet_pton, written by Paul Vixie. */ + +static bool +is_valid_ipv4_address (const char *str, const char *end) +{ + bool saw_digit = false; + int octets = 0; + int val = 0; + + while (str < end) + { + int ch = *str++; + + if (ch >= '0' && ch <= '9') + { + val = val * 10 + (ch - '0'); + + if (val > 255) + return false; + if (!saw_digit) + { + if (++octets > 4) + return false; + saw_digit = true; + } + } + else if (ch == '.' && saw_digit) + { + if (octets == 4) + return false; + val = 0; + saw_digit = false; + } + else + return false; + } + if (octets < 4) + return false; + + return true; +} + +bool +is_valid_ipv6_address (const char *str, const char *end) +{ + /* Use lower-case for these to avoid clash with system headers. */ + enum { + ns_inaddrsz = 4, + ns_in6addrsz = 16, + ns_int16sz = 2 + }; + + const char *curtok; + int tp; + const char *colonp; + bool saw_xdigit; + unsigned int val; + + tp = 0; + colonp = NULL; + + if (str == end) + return false; + + /* Leading :: requires some special handling. */ + if (*str == ':') + { + ++str; + if (str == end || *str != ':') + return false; + } + + curtok = str; + saw_xdigit = false; + val = 0; + + while (str < end) + { + int ch = *str++; + + /* if ch is a number, add it to val. */ + if (c_isxdigit (ch)) + { + val <<= 4; + val |= _unhex (ch); + if (val > 0xffff) + return false; + saw_xdigit = true; + continue; + } + + /* if ch is a colon ... */ + if (ch == ':') + { + curtok = str; + if (!saw_xdigit) + { + if (colonp != NULL) + return false; + colonp = str + tp; + continue; + } + else if (str == end) + return false; + if (tp > ns_in6addrsz - ns_int16sz) + return false; + tp += ns_int16sz; + saw_xdigit = false; + val = 0; + continue; + } + + /* if ch is a dot ... */ + if (ch == '.' && (tp <= ns_in6addrsz - ns_inaddrsz) + && is_valid_ipv4_address (curtok, end) == 1) + { + tp += ns_inaddrsz; + saw_xdigit = false; + break; + } + + return false; + } + + if (saw_xdigit) + { + if (tp > ns_in6addrsz - ns_int16sz) + return false; + tp += ns_int16sz; + } + + if (colonp != NULL) + { + if (tp == ns_in6addrsz) + return false; + tp = ns_in6addrsz; + } + + if (tp != ns_in6addrsz) + return false; + + return true; +} + +/* Simple host cache, used by lookup_host to speed up resolving. The + cache doesn't handle TTL because Wget is a fairly short-lived + application. Refreshing is attempted when connect fails, though -- + see connect_to_host. */ + +/* Mapping between known hosts and to lists of their addresses. */ +static struct hash_table *host_name_addresses_map; + + +/* Return the host's resolved addresses from the cache, if + available. */ + +static struct address_list * +cache_query (const char *host) +{ + struct address_list *al; + if (!host_name_addresses_map) + return NULL; + al = hash_table_get (host_name_addresses_map, host); + if (al) + { + DEBUGP (("Found %s in host_name_addresses_map (%p)\n", host, (void *) al)); + ++al->refcount; + return al; + } + return NULL; +} + +/* Cache the DNS lookup of HOST. Subsequent invocations of + lookup_host will return the cached value. */ + +static void +cache_store (const char *host, struct address_list *al) +{ + if (!host_name_addresses_map) + host_name_addresses_map = make_nocase_string_hash_table (0); + + ++al->refcount; + hash_table_put (host_name_addresses_map, xstrdup_lower (host), al); + + IF_DEBUG + { + int i; + debug_logprintf ("Caching %s =>", host); + for (i = 0; i < al->count; i++) + debug_logprintf (" %s", print_address (al->addresses + i)); + debug_logprintf ("\n"); + } +} + +/* Remove HOST from the DNS cache. Does nothing is HOST is not in + the cache. */ + +static void +cache_remove (const char *host) +{ + struct address_list *al; + if (!host_name_addresses_map) + return; + al = hash_table_get (host_name_addresses_map, host); + if (al) + { + address_list_release (al); + hash_table_remove (host_name_addresses_map, host); + } +} + +#ifdef HAVE_LIBCARES +#include <sys/select.h> +#include <ares.h> +extern ares_channel ares; + +static struct address_list * +merge_address_lists (struct address_list *al1, struct address_list *al2) +{ + int count = al1->count + al2->count; + + /* merge al2 into al1 */ + al1->addresses = xrealloc (al1->addresses, sizeof (ip_address) * count); + memcpy (al1->addresses + al1->count, al2->addresses, sizeof (ip_address) * al2->count); + al1->count = count; + + address_list_delete (al2); + + return al1; +} + +static struct address_list * +address_list_from_hostent (struct hostent *host) +{ + int count, i; + struct address_list *al = xnew0 (struct address_list); + + for (count = 0; host->h_addr_list[count]; count++) + ; + + assert (count > 0); + + al->addresses = xnew_array (ip_address, count); + al->count = count; + al->refcount = 1; + + for (i = 0; i < count; i++) + { + ip_address *ip = &al->addresses[i]; + ip->family = host->h_addrtype; + memcpy (IP_INADDR_DATA (ip), host->h_addr_list[i], ip->family == AF_INET ? 4 : 16); + } + + return al; +} + +/* Since GnuLib's select() (i.e. rpl_select()) cannot handle socket-numbers + * returned from C-ares, we must use the original select() from Winsock. + */ +#ifdef WINDOWS +#undef select +#endif + +static void +wait_ares (ares_channel channel) +{ + struct ptimer *timer = NULL; + + if (opt.dns_timeout) + timer = ptimer_new (); + + for (;;) + { + struct timeval *tvp, tv; + fd_set read_fds, write_fds; + int nfds, rc; + + FD_ZERO (&read_fds); + FD_ZERO (&write_fds); + nfds = ares_fds (channel, &read_fds, &write_fds); + if (nfds == 0) + break; + + if (timer) + { + double max = opt.dns_timeout - ptimer_measure (timer); + + tv.tv_sec = (long) max; + tv.tv_usec = 1000000 * (max - (long) max); + tvp = ares_timeout (channel, &tv, &tv); + } + else + tvp = ares_timeout (channel, NULL, &tv); + + rc = select (nfds, &read_fds, &write_fds, NULL, tvp); + if (rc == 0 && timer && ptimer_measure (timer) >= opt.dns_timeout) + ares_cancel (channel); + else + ares_process (channel, &read_fds, &write_fds); + } + if (timer) + ptimer_destroy (timer); +} + +static void +callback (void *arg, int status, int timeouts _GL_UNUSED, struct hostent *host) +{ + struct address_list **al = (struct address_list **) arg; + + if (!host || status != ARES_SUCCESS) + { + *al = NULL; + return; + } + + *al = address_list_from_hostent (host); +} +#endif + +/* Look up HOST in DNS and return a list of IP addresses. + + This function caches its result so that, if the same host is passed + the second time, the addresses are returned without DNS lookup. + (Use LH_REFRESH to force lookup, or set opt.dns_cache to 0 to + globally disable caching.) + + The order of the returned addresses is affected by the setting of + opt.prefer_family: if it is set to prefer_ipv4, IPv4 addresses are + placed at the beginning; if it is prefer_ipv6, IPv6 ones are placed + at the beginning; otherwise, the order is left intact. The + relative order of addresses with the same family is left + undisturbed in either case. + + FLAGS can be a combination of: + LH_SILENT - don't print the "resolving ... done" messages. + LH_BIND - resolve addresses for use with bind, which under + IPv6 means to use AI_PASSIVE flag to getaddrinfo. + Passive lookups are not cached under IPv6. + LH_REFRESH - if HOST is cached, remove the entry from the cache + and resolve it anew. */ + +struct address_list * +lookup_host (const char *host, int flags) +{ + struct address_list *al; + bool silent = !!(flags & LH_SILENT); + bool use_cache; + bool numeric_address = false; + double timeout = opt.dns_timeout; + +#ifndef ENABLE_IPV6 + /* If we're not using getaddrinfo, first check if HOST specifies a + numeric IPv4 address. Some implementations of gethostbyname + (e.g. the Ultrix one and possibly Winsock) don't accept + dotted-decimal IPv4 addresses. */ + { + uint32_t addr_ipv4 = (uint32_t)inet_addr (host); + if (addr_ipv4 != (uint32_t) -1) + { + /* No need to cache host->addr relation, just return the + address. */ + char *vec[2]; + vec[0] = (char *)&addr_ipv4; + vec[1] = NULL; + return address_list_from_ipv4_addresses (vec); + } + } +#else /* ENABLE_IPV6 */ + /* If we're using getaddrinfo, at least check whether the address is + already numeric, in which case there is no need to print the + "Resolving..." output. (This comes at no additional cost since + the is_valid_ipv*_address are already required for + url_parse.) */ + { + const char *end = host + strlen (host); + if (is_valid_ipv4_address (host, end) || is_valid_ipv6_address (host, end)) + numeric_address = true; + } +#endif + + /* Cache is normally on, but can be turned off with --no-dns-cache. + Don't cache passive lookups under IPv6. */ + use_cache = opt.dns_cache; +#ifdef ENABLE_IPV6 + if ((flags & LH_BIND) || numeric_address) + use_cache = false; +#endif + + /* Try to find the host in the cache so we don't need to talk to the + resolver. If LH_REFRESH is requested, remove HOST from the cache + instead. */ + if (use_cache) + { + if (!(flags & LH_REFRESH)) + { + al = cache_query (host); + if (al) + return al; + } + else + cache_remove (host); + } + + /* No luck with the cache; resolve HOST. */ + + if (!silent && !numeric_address) + { + char *str = NULL, *name; + + if (opt.enable_iri && (name = idn_decode ((char *) host)) != NULL) + { + str = aprintf ("%s (%s)", name, host); + xfree (name); + } + + logprintf (LOG_VERBOSE, _("Resolving %s... "), + quotearg_style (escape_quoting_style, str ? str : host)); + + xfree (str); + } + +#ifdef ENABLE_IPV6 +#ifdef HAVE_LIBCARES + if (ares) + { + struct address_list *al4 = NULL; + struct address_list *al6 = NULL; + + if (opt.ipv4_only || !opt.ipv6_only) + ares_gethostbyname (ares, host, AF_INET, callback, &al4); + if (opt.ipv6_only || !opt.ipv4_only) + ares_gethostbyname (ares, host, AF_INET6, callback, &al6); + + wait_ares (ares); + + if (al4 && al6) + al = merge_address_lists (al4, al6); + else if (al4) + al = al4; + else + al = al6; + } + else +#endif + { + int err; + struct addrinfo hints, *res; + + xzero (hints); + hints.ai_socktype = SOCK_STREAM; + if (opt.ipv4_only) + hints.ai_family = AF_INET; + else if (opt.ipv6_only) + hints.ai_family = AF_INET6; + else + /* We tried using AI_ADDRCONFIG, but removed it because: it + misinterprets IPv6 loopbacks, it is broken on AIX 5.1, and + it's unneeded since we sort the addresses anyway. */ + hints.ai_family = AF_UNSPEC; + + if (flags & LH_BIND) + hints.ai_flags |= AI_PASSIVE; + +#ifdef AI_NUMERICHOST + if (numeric_address) + { + /* Where available, the AI_NUMERICHOST hint can prevent costly + access to DNS servers. */ + hints.ai_flags |= AI_NUMERICHOST; + timeout = 0; /* no timeout needed when "resolving" + numeric hosts -- avoid setting up + signal handlers and such. */ + } +#endif + + err = getaddrinfo_with_timeout (host, NULL, &hints, &res, timeout); + + if (err != 0 || res == NULL) + { + if (!silent) + logprintf (LOG_VERBOSE, _ ("failed: %s.\n"), + err != EAI_SYSTEM ? gai_strerror (err) : strerror (errno)); + return NULL; + } + al = address_list_from_addrinfo (res); + freeaddrinfo (res); + } + + if (!al) + { + logprintf (LOG_VERBOSE, + _ ("failed: No IPv4/IPv6 addresses for host.\n")); + return NULL; + } + + /* Reorder addresses so that IPv4 ones (or IPv6 ones, as per + --prefer-family) come first. Sorting is stable so the order of + the addresses with the same family is undisturbed. */ + if (al->count > 1 && opt.prefer_family != prefer_none) + stable_sort (al->addresses, al->count, sizeof (ip_address), + opt.prefer_family == prefer_ipv4 + ? cmp_prefer_ipv4 : cmp_prefer_ipv6); +#else /* not ENABLE_IPV6 */ +#ifdef HAVE_LIBCARES + if (ares) + { + ares_gethostbyname (ares, host, AF_INET, callback, &al); + wait_ares (ares); + } + else +#endif + { + struct hostent *hptr = gethostbyname_with_timeout (host, timeout); + if (!hptr) + { + if (!silent) + { + if (errno != ETIMEDOUT) + logprintf (LOG_VERBOSE, _ ("failed: %s.\n"), + host_errstr (h_errno)); + else + logputs (LOG_VERBOSE, _ ("failed: timed out.\n")); + } + return NULL; + } + /* Do older systems have h_addr_list? */ + al = address_list_from_ipv4_addresses (hptr->h_addr_list); + } +#endif /* not ENABLE_IPV6 */ + + /* Print the addresses determined by DNS lookup, but no more than + three if show_all_dns_entries is not specified. */ + if (!silent && !numeric_address) + { + int i; + int printmax = al->count; + + if (!opt.show_all_dns_entries && printmax > 3) + printmax = 3; + + for (i = 0; i < printmax; i++) + { + logputs (LOG_VERBOSE, print_address (al->addresses + i)); + if (i < printmax - 1) + logputs (LOG_VERBOSE, ", "); + } + if (printmax != al->count) + logputs (LOG_VERBOSE, ", ..."); + logputs (LOG_VERBOSE, "\n"); + } + + /* Cache the lookup information. */ + if (use_cache) + cache_store (host, al); + + return al; +} + +/* Determine whether a URL is acceptable to be followed, according to + a list of domains to accept. */ +bool +accept_domain (struct url *u) +{ + assert (u->host != NULL); + if (opt.domains) + { + if (!sufmatch ((const char **)opt.domains, u->host)) + return false; + } + if (opt.exclude_domains) + { + if (sufmatch ((const char **)opt.exclude_domains, u->host)) + return false; + } + return true; +} + +/* Check whether WHAT is matched in LIST, each element of LIST being a + pattern to match WHAT against, using backward matching (see + match_backwards() in utils.c). + + If an element of LIST matched, 1 is returned, 0 otherwise. */ +bool +sufmatch (const char **list, const char *what) +{ + int i, j, k, lw; + + lw = strlen (what); + + for (i = 0; list[i]; i++) + { + j = strlen (list[i]); + if (lw < j) + continue; /* what is no (sub)domain of list[i] */ + + for (k = lw; j >= 0 && k >= 0; j--, k--) + if (c_tolower (list[i][j]) != c_tolower (what[k])) + break; + + /* Domain or subdomain match + * k == -1: exact match + * k >= 0 && what[k] == '.': subdomain match + * k >= 0 && list[i][0] == '.': dot-prefixed subdomain match + */ + if (j == -1 && (k == -1 || what[k] == '.' || list[i][0] == '.')) + return true; + } + + return false; +} + +#if defined DEBUG_MALLOC || defined TESTING +void +host_cleanup (void) +{ + if (host_name_addresses_map) + { + hash_table_iterator iter; + for (hash_table_iterate (host_name_addresses_map, &iter); + hash_table_iter_next (&iter); + ) + { + char *host = iter.key; + struct address_list *al = iter.value; + xfree (host); + assert (al->refcount == 1); + address_list_delete (al); + } + hash_table_destroy (host_name_addresses_map); + host_name_addresses_map = NULL; + } +} +#endif + +bool +is_valid_ip_address (const char *name) +{ + const char *endp; + + endp = name + strlen(name); + if (is_valid_ipv4_address (name, endp)) + return true; +#ifdef ENABLE_IPV6 + if (is_valid_ipv6_address (name, endp)) + return true; +#endif + return false; +} diff --git a/src/host.h b/src/host.h new file mode 100644 index 0000000..a199d26 --- /dev/null +++ b/src/host.h @@ -0,0 +1,107 @@ +/* Declarations for host.c + Copyright (C) 1996-2012, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef HOST_H +#define HOST_H + +#ifdef WINDOWS +# include <winsock2.h> +#else +# ifdef __VMS +# include "vms_ip.h" +# else /* def __VMS */ +# include <netdb.h> +# endif /* def __VMS [else] */ +# include <sys/socket.h> +# include <netinet/in.h> +#ifndef __BEOS__ +# include <arpa/inet.h> +#endif +#endif + +struct url; +struct address_list; + +/* This struct defines an IP address, tagged with family type. */ + +typedef struct { + /* Address family, one of AF_INET or AF_INET6. */ + int family; + + /* The actual data, in the form of struct in_addr or in6_addr: */ + union { + struct in_addr d4; /* IPv4 address */ +#ifdef ENABLE_IPV6 + struct in6_addr d6; /* IPv6 address */ +#endif + } data; + + /* Under IPv6 getaddrinfo also returns scope_id. Since it's + IPv6-specific it strictly belongs in the above union, but we put + it here for simplicity. */ +#if defined ENABLE_IPV6 && defined HAVE_SOCKADDR_IN6_SCOPE_ID + int ipv6_scope; +#endif +} ip_address; + +/* IP_INADDR_DATA macro returns a void pointer that can be interpreted + as a pointer to struct in_addr in IPv4 context or a pointer to + struct in6_addr in IPv4 context. This pointer can be passed to + functions that work on either, such as inet_ntop. */ +#define IP_INADDR_DATA(x) ((void *) &(x)->data) + +enum { + LH_SILENT = 1, + LH_BIND = 2, + LH_REFRESH = 4 +}; +struct address_list *lookup_host (const char *, int); + +void address_list_get_bounds (const struct address_list *, int *, int *); +const ip_address *address_list_address_at (const struct address_list *, int); +bool address_list_contains (const struct address_list *, const ip_address *); +void address_list_set_faulty (struct address_list *, int); +void address_list_set_connected (struct address_list *); +bool address_list_connected_p (const struct address_list *); +void address_list_release (struct address_list *); + +const char *print_address (const ip_address *); +#ifdef ENABLE_IPV6 +bool is_valid_ipv6_address (const char *, const char *); +#endif + +bool is_valid_ip_address (const char *name); + +bool accept_domain (struct url *); +bool sufmatch (const char **, const char *); + +void host_cleanup (void); + +#endif /* HOST_H */ diff --git a/src/hsts.c b/src/hsts.c new file mode 100644 index 0000000..914d270 --- /dev/null +++ b/src/hsts.c @@ -0,0 +1,829 @@ +/* HTTP Strict Transport Security (HSTS) support. + Copyright (C) 1996-2012, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ +#include "wget.h" + +#ifdef HAVE_HSTS +#include "hsts.h" +#include "utils.h" +#include "host.h" /* for is_valid_ip_address() */ +#include "hash.h" +#include "c-ctype.h" +#ifdef TESTING +#include "init.h" /* for ajoin_dir_file() */ +#include "../tests/unit-tests.h" +#endif + +#include <unistd.h> +#include <sys/types.h> +#include <stdlib.h> +#include <time.h> +#include <sys/stat.h> +#include <string.h> +#include <stdio.h> +#include <sys/file.h> + +struct hsts_store { + struct hash_table *table; + time_t last_mtime; + bool changed; +}; + +struct hsts_kh { + char *host; + int explicit_port; +}; + +struct hsts_kh_info { + time_t created; + time_t max_age; + bool include_subdomains; +}; + +enum hsts_kh_match { + NO_MATCH, + SUPERDOMAIN_MATCH, + CONGRUENT_MATCH +}; + +#define hsts_is_host_name_valid(host) (!is_valid_ip_address (host)) +#define hsts_is_scheme_valid(scheme) (scheme == SCHEME_HTTPS) +#define hsts_is_host_eligible(scheme, host) \ + (hsts_is_scheme_valid (scheme) && hsts_is_host_name_valid (host)) + +#define DEFAULT_HTTP_PORT 80 +#define DEFAULT_SSL_PORT 443 +#define MAKE_EXPLICIT_PORT(s, p) (s == SCHEME_HTTPS ? (p == DEFAULT_SSL_PORT ? 0 : p) \ + : (p == DEFAULT_HTTP_PORT ? 0 : p)) + +/* Hashing and comparison functions for the hash table */ + +#ifdef __clang__ +__attribute__((no_sanitize("integer"))) +#endif +static unsigned long +hsts_hash_func (const void *key) +{ + struct hsts_kh *k = (struct hsts_kh *) key; + const char *h = NULL; + unsigned int hash = k->explicit_port; + + for (h = k->host; *h; h++) + hash = hash * 31 + *h; + + return hash; +} + +static int +hsts_cmp_func (const void *h1, const void *h2) +{ + struct hsts_kh *kh1 = (struct hsts_kh *) h1, + *kh2 = (struct hsts_kh *) h2; + + return (!strcmp (kh1->host, kh2->host)) && (kh1->explicit_port == kh2->explicit_port); +} + +/* Private functions. Feel free to make some of these public when needed. */ + +static struct hsts_kh_info * +hsts_find_entry (hsts_store_t store, + const char *host, int explicit_port, + enum hsts_kh_match *match_type, + struct hsts_kh *kh) +{ + struct hsts_kh *k = NULL; + struct hsts_kh_info *khi = NULL; + enum hsts_kh_match match = NO_MATCH; + char *pos = NULL; + char *org_ptr = NULL; + + k = (struct hsts_kh *) xnew (struct hsts_kh); + k->host = xstrdup_lower (host); + k->explicit_port = explicit_port; + + /* save pointer so that we don't get into trouble later when freeing */ + org_ptr = k->host; + + khi = (struct hsts_kh_info *) hash_table_get (store->table, k); + if (khi) + { + match = CONGRUENT_MATCH; + goto end; + } + + while (match == NO_MATCH && + (pos = strchr (k->host, '.')) && pos - k->host > 0 && + strchr (pos + 1, '.')) + { + k->host += (pos - k->host + 1); + khi = (struct hsts_kh_info *) hash_table_get (store->table, k); + if (khi) + match = SUPERDOMAIN_MATCH; + } + +end: + /* restore pointer or we'll get a SEGV */ + k->host = org_ptr; + + /* copy parameters to previous frame */ + if (match_type) + *match_type = match; + if (kh) + memcpy (kh, k, sizeof (struct hsts_kh)); + else + xfree (k->host); + + xfree (k); + return khi; +} + +static bool +hsts_new_entry_internal (hsts_store_t store, + const char *host, int port, + time_t created, time_t max_age, + bool include_subdomains, + bool check_validity, + bool check_expired, + bool check_duplicates) +{ + struct hsts_kh *kh = xnew (struct hsts_kh); + struct hsts_kh_info *khi = xnew0 (struct hsts_kh_info); + bool success = false; + + kh->host = xstrdup_lower (host); + kh->explicit_port = MAKE_EXPLICIT_PORT (SCHEME_HTTPS, port); + + khi->created = created; + khi->max_age = max_age; + khi->include_subdomains = include_subdomains; + + /* Check validity */ + if (check_validity && !hsts_is_host_name_valid (host)) + goto bail; + + if (check_expired && ((khi->created + khi->max_age) < khi->created)) + goto bail; + + if (check_duplicates && hash_table_contains (store->table, kh)) + goto bail; + + /* Now store the new entry */ + hash_table_put (store->table, kh, khi); + success = true; + +bail: + if (!success) + { + /* abort! */ + xfree (kh->host); + xfree (kh); + xfree (khi); + } + + return success; +} + +/* + Creates a new entry, but does not check whether that entry already exists. + This function assumes that check has already been done by the caller. + */ +static bool +hsts_add_entry (hsts_store_t store, + const char *host, int port, + time_t max_age, bool include_subdomains) +{ + time_t t = time (NULL); + + /* It might happen time() returned -1 */ + return (t == (time_t)(-1) ? + false : + hsts_new_entry_internal (store, host, port, t, max_age, include_subdomains, false, true, false)); +} + +/* Creates a new entry, unless an identical one already exists. */ +static bool +hsts_new_entry (hsts_store_t store, + const char *host, int port, + time_t created, time_t max_age, + bool include_subdomains) +{ + return hsts_new_entry_internal (store, host, port, created, max_age, include_subdomains, true, true, true); +} + +static void +hsts_remove_entry (hsts_store_t store, struct hsts_kh *kh) +{ + hash_table_remove (store->table, kh); +} + +static bool +hsts_store_merge (hsts_store_t store, + const char *host, int port, + time_t created, time_t max_age, + bool include_subdomains) +{ + enum hsts_kh_match match_type = NO_MATCH; + struct hsts_kh_info *khi = NULL; + bool success = false; + + port = MAKE_EXPLICIT_PORT (SCHEME_HTTPS, port); + khi = hsts_find_entry (store, host, port, &match_type, NULL); + if (khi && match_type == CONGRUENT_MATCH && created > khi->created) + { + /* update the entry with the new info */ + khi->created = created; + khi->max_age = max_age; + khi->include_subdomains = include_subdomains; + + success = true; + } + else if (!khi) + success = hsts_new_entry (store, host, port, created, max_age, include_subdomains); + + return success; +} + +static bool +hsts_read_database (hsts_store_t store, FILE *fp, bool merge_with_existing_entries) +{ + char *line = NULL, *p; + size_t len = 0; + int items_read; + bool result = false; + bool (*func)(hsts_store_t, const char *, int, time_t, time_t, bool); + + char host[256]; + int port; + time_t created, max_age; + int include_subdomains; + + func = (merge_with_existing_entries ? hsts_store_merge : hsts_new_entry); + + while (getline (&line, &len, fp) > 0) + { + for (p = line; c_isspace (*p); p++) + ; + + if (*p == '#') + continue; + + items_read = sscanf (p, "%255s %d %d %lu %lu", + host, + &port, + &include_subdomains, + (unsigned long *) &created, + (unsigned long *) &max_age); + + if (items_read == 5) + func (store, host, port, created, max_age, !!include_subdomains); + } + + xfree (line); + result = true; + + return result; +} + +static void +hsts_store_dump (hsts_store_t store, FILE *fp) +{ + hash_table_iterator it; + + /* Print preliminary comments. We don't care if any of these fail. */ + fputs ("# HSTS 1.0 Known Hosts database for GNU Wget.\n", fp); + fputs ("# Edit at your own risk.\n", fp); + fputs ("# <hostname>\t<port>\t<incl. subdomains>\t<created>\t<max-age>\n", fp); + + /* Now cycle through the HSTS store in memory and dump the entries */ + for (hash_table_iterate (store->table, &it); hash_table_iter_next (&it);) + { + struct hsts_kh *kh = (struct hsts_kh *) it.key; + struct hsts_kh_info *khi = (struct hsts_kh_info *) it.value; + + if (fprintf (fp, "%s\t%d\t%d\t%lu\t%lu\n", + kh->host, kh->explicit_port, khi->include_subdomains, + (unsigned long) khi->created, + (unsigned long) khi->max_age) < 0) + { + logprintf (LOG_ALWAYS, "Could not write the HSTS database correctly.\n"); + break; + } + } +} + +/* + * Test: + * - The file is a regular file (ie. not a symlink), and + * - The file is not world-writable. + */ +static bool +hsts_file_access_valid (const char *filename) +{ + struct stat st; + + if (stat (filename, &st) == -1) + return false; + + return +#ifndef WINDOWS + /* + * The world-writable concept is a Unix-centric notion. + * We bypass this test on Windows. + */ + !(st.st_mode & S_IWOTH) && +#endif + S_ISREG (st.st_mode); +} + +/* HSTS API */ + +/* + Changes the given URLs according to the HSTS policy. + + If there's no host in the store that either congruently + or not, matches the given URL, no changes are made. + Returns true if the URL was changed, or false + if it was left intact. + */ +bool +hsts_match (hsts_store_t store, struct url *u) +{ + bool url_changed = false; + struct hsts_kh_info *entry = NULL; + struct hsts_kh *kh = xnew(struct hsts_kh); + enum hsts_kh_match match = NO_MATCH; + int port = MAKE_EXPLICIT_PORT (u->scheme, u->port); + + /* avoid doing any computation if we're already in HTTPS */ + if (!hsts_is_scheme_valid (u->scheme)) + { + entry = hsts_find_entry (store, u->host, port, &match, kh); + if (entry) + { + if ((entry->created + entry->max_age) >= time(NULL)) + { + if ((match == CONGRUENT_MATCH) || + (match == SUPERDOMAIN_MATCH && entry->include_subdomains)) + { + /* we found a matching Known HSTS Host + rewrite the URL */ + u->scheme = SCHEME_HTTPS; + if (u->port == 80) + u->port = 443; + url_changed = true; + store->changed = true; + } + } + else + { + hsts_remove_entry (store, kh); + store->changed = true; + } + } + xfree (kh->host); + } + + xfree (kh); + + return url_changed; +} + +/* + Add a new HSTS Known Host to the HSTS store. + + If the host already exists, its information is updated, + or it'll be removed from the store if max_age is zero. + + Bear in mind that the store is kept in memory, and will not + be written to disk until hsts_store_save is called. + This function regrows the in-memory HSTS store if necessary. + + Currently, for a host to be taken into consideration, + two conditions have to be met: + - Connection must be through a secure channel (HTTPS). + - The host must not be an IPv4 or IPv6 address. + + The RFC 6797 states that hosts that match IPv4 or IPv6 format + should be discarded at URI rewrite time. But we short-circuit + that check here, since there's no point in storing a host that + will never be matched. + + Returns true if a new entry was actually created, or false + if an existing entry was updated/deleted. */ +bool +hsts_store_entry (hsts_store_t store, + enum url_scheme scheme, const char *host, int port, + time_t max_age, bool include_subdomains) +{ + bool result = false; + enum hsts_kh_match match = NO_MATCH; + struct hsts_kh *kh = xnew(struct hsts_kh); + struct hsts_kh_info *entry = NULL; + + if (hsts_is_host_eligible (scheme, host)) + { + port = MAKE_EXPLICIT_PORT (scheme, port); + entry = hsts_find_entry (store, host, port, &match, kh); + if (entry && match == CONGRUENT_MATCH) + { + if (max_age == 0) + { + hsts_remove_entry (store, kh); + store->changed = true; + } + else if (max_age > 0) + { + /* RFC 6797 states that 'max_age' is a TTL relative to the + * reception of the STS header so we have to update the + * 'created' field too. The RFC also states that we have to + * update the entry each time we see HSTS header. + * See also Section 11.2. */ + time_t t = time (NULL); + + if (t != (time_t)(-1) && t != entry->created) + { + entry->created = t; + entry->max_age = max_age; + entry->include_subdomains = include_subdomains; + store->changed = true; + } + } + /* we ignore negative max_ages */ + } + else if (entry == NULL || match == SUPERDOMAIN_MATCH) + { + /* Either we didn't find a matching host, + or we got a superdomain match. + In either case, we create a new entry. + + We have to perform an explicit check because it might + happen we got a non-existent entry with max_age == 0. + */ + result = hsts_add_entry (store, host, port, max_age, include_subdomains); + if (result) + store->changed = true; + } + /* we ignore new entries with max_age == 0 */ + xfree (kh->host); + } + + xfree (kh); + + return result; +} + +hsts_store_t +hsts_store_open (const char *filename) +{ + hsts_store_t store = NULL; + file_stats_t fstats; + + store = xnew0 (struct hsts_store); + store->table = hash_table_new (0, hsts_hash_func, hsts_cmp_func); + store->last_mtime = 0; + store->changed = false; + + if (file_exists_p (filename, &fstats)) + { + if (hsts_file_access_valid (filename)) + { + struct stat st; + FILE *fp = fopen_stat (filename, "r", &fstats); + + if (!fp || !hsts_read_database (store, fp, false)) + { + /* abort! */ + hsts_store_close (store); + xfree (store); + if (fp) + fclose (fp); + goto out; + } + + if (fstat (fileno (fp), &st) == 0) + store->last_mtime = st.st_mtime; + + fclose (fp); + } + else + { + /* + * If we're not reading the HSTS database, + * then by all means act as if HSTS was disabled. + */ + hsts_store_close (store); + xfree (store); + + logprintf (LOG_NOTQUIET, "Will not apply HSTS. " + "The HSTS database must be a regular and non-world-writable file.\n"); + } + } + +out: + return store; +} + +void +hsts_store_save (hsts_store_t store, const char *filename) +{ + struct stat st; + FILE *fp = NULL; + int fd = 0; + + if (filename && hash_table_count (store->table) > 0) + { + fp = fopen (filename, "a+"); + if (fp) + { + /* Lock the file to avoid potential race conditions */ + fd = fileno (fp); + flock (fd, LOCK_EX); + + /* If the file has changed, merge the changes with our in-memory data + before dumping them to the file. + Otherwise we could potentially overwrite the data stored by other Wget processes. + */ + if (store->last_mtime && stat (filename, &st) == 0 && st.st_mtime > store->last_mtime) + hsts_read_database (store, fp, true); + + /* We've merged the latest changes so we can now truncate the file + and dump everything. */ + fseek (fp, 0, SEEK_SET); + ftruncate (fd, 0); + + /* now dump to the file */ + hsts_store_dump (store, fp); + + /* fclose is expected to unlock the file for us */ + fclose (fp); + } + } +} + +bool +hsts_store_has_changed (hsts_store_t store) +{ + return (store ? store->changed : false); +} + +void +hsts_store_close (hsts_store_t store) +{ + hash_table_iterator it; + + /* free all the host fields */ + for (hash_table_iterate (store->table, &it); hash_table_iter_next (&it);) + { + xfree (((struct hsts_kh *) it.key)->host); + xfree (it.key); + xfree (it.value); + } + + hash_table_destroy (store->table); +} + +#ifdef TESTING +/* I know I'm really evil because I'm writing macros + that change control flow. But we're testing, who will tell? :D + */ +#define TEST_URL_RW(s, u, p) do { \ + if (test_url_rewrite (s, u, p, true)) \ + return test_url_rewrite (s, u, p, true); \ + } while (0) + +#define TEST_URL_NORW(s, u, p) do { \ + if (test_url_rewrite (s, u, p, false)) \ + return test_url_rewrite (s, u, p, false); \ + } while (0) + +static char * +get_hsts_store_filename (void) +{ + char *filename = NULL; + FILE *fp = NULL; + + if (opt.homedir) + { + filename = ajoin_dir_file (opt.homedir, ".wget-hsts-test"); + fp = fopen (filename, "w"); + if (fp) + fclose (fp); + } + + return filename; +} + +static hsts_store_t +open_hsts_test_store (void) +{ + char *filename = NULL; + hsts_store_t table = NULL; + + filename = get_hsts_store_filename (); + table = hsts_store_open (filename); + xfree (filename); + + return table; +} + +static void +close_hsts_test_store (hsts_store_t store) +{ + char *filename; + + if ((filename = get_hsts_store_filename ())) + { + unlink (filename); + xfree (filename); + } + xfree (store); +} + +static const char* +test_url_rewrite (hsts_store_t s, const char *url, int port, bool rewrite) +{ + bool result; + struct url u; + + u.host = xstrdup (url); + u.port = port; + u.scheme = SCHEME_HTTP; + + result = hsts_match (s, &u); + + if (rewrite) + { + if (port == 80) + mu_assert("URL: port should've been rewritten to 443", u.port == 443); + else + mu_assert("URL: port should've been left intact", u.port == port); + mu_assert("URL: scheme should've been rewritten to HTTPS", u.scheme == SCHEME_HTTPS); + mu_assert("result should've been true", result == true); + } + else + { + mu_assert("URL: port should've been left intact", u.port == port); + mu_assert("URL: scheme should've been left intact", u.scheme == SCHEME_HTTP); + mu_assert("result should've been false", result == false); + } + + xfree (u.host); + return NULL; +} + +const char * +test_hsts_new_entry (void) +{ + enum hsts_kh_match match = NO_MATCH; + struct hsts_kh_info *khi; + hsts_store_t s; + bool created; + + s = open_hsts_test_store (); + mu_assert("Could not open the HSTS store. This could be due to lack of memory.", s != NULL); + + created = hsts_store_entry (s, SCHEME_HTTP, "www.foo.com", 80, 1234, true); + mu_assert("No entry should have been created.", created == false); + + created = hsts_store_entry (s, SCHEME_HTTPS, "www.foo.com", 443, 1234, true); + mu_assert("A new entry should have been created", created == true); + + khi = hsts_find_entry (s, "www.foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL); + mu_assert("Should've been a congruent match", match == CONGRUENT_MATCH); + mu_assert("No valid HSTS info was returned", khi != NULL); + mu_assert("Variable 'max_age' should be 1234", khi->max_age == 1234); + mu_assert("Variable 'include_subdomains' should be asserted", khi->include_subdomains == true); + + khi = hsts_find_entry (s, "b.www.foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL); + mu_assert("Should've been a superdomain match", match == SUPERDOMAIN_MATCH); + mu_assert("No valid HSTS info was returned", khi != NULL); + mu_assert("Variable 'max_age' should be 1234", khi->max_age == 1234); + mu_assert("Variable 'include_subdomains' should be asserted", khi->include_subdomains == true); + + khi = hsts_find_entry (s, "ww.foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL); + mu_assert("Should've been no match", match == NO_MATCH); + + khi = hsts_find_entry (s, "foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL); + mu_assert("Should've been no match", match == NO_MATCH); + + khi = hsts_find_entry (s, ".foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL); + mu_assert("Should've been no match", match == NO_MATCH); + + khi = hsts_find_entry (s, ".www.foo.com", MAKE_EXPLICIT_PORT (SCHEME_HTTPS, 443), &match, NULL); + mu_assert("Should've been no match", match == NO_MATCH); + + hsts_store_close (s); + close_hsts_test_store (s); + + return NULL; +} + +const char* +test_hsts_url_rewrite_superdomain (void) +{ + hsts_store_t s; + bool created; + + s = open_hsts_test_store (); + mu_assert("Could not open the HSTS store", s != NULL); + + created = hsts_store_entry (s, SCHEME_HTTPS, "www.foo.com", 443, 1234, true); + mu_assert("A new entry should've been created", created == true); + + TEST_URL_RW (s, "www.foo.com", 80); + TEST_URL_RW (s, "bar.www.foo.com", 80); + + hsts_store_close (s); + close_hsts_test_store (s); + + return NULL; +} + +const char* +test_hsts_url_rewrite_congruent (void) +{ + hsts_store_t s; + bool created; + + s = open_hsts_test_store (); + mu_assert("Could not open the HSTS store", s != NULL); + + created = hsts_store_entry (s, SCHEME_HTTPS, "foo.com", 443, 1234, false); + mu_assert("A new entry should've been created", created == true); + + TEST_URL_RW (s, "foo.com", 80); + TEST_URL_NORW (s, "www.foo.com", 80); + + hsts_store_close (s); + close_hsts_test_store (s); + + return NULL; +} + +const char* +test_hsts_read_database (void) +{ + hsts_store_t table; + char *file = NULL; + FILE *fp = NULL; + time_t created = time(NULL) - 10; + + if (opt.homedir) + { + file = ajoin_dir_file (opt.homedir, ".wget-hsts-testing"); + fp = fopen (file, "w"); + if (fp) + { + fputs ("# dummy comment\n", fp); + fprintf (fp, "foo.example.com\t0\t1\t%lu\t123\n",(unsigned long) created); + fprintf (fp, "bar.example.com\t0\t0\t%lu\t456\n", (unsigned long) created); + fprintf (fp, "test.example.com\t8080\t0\t%lu\t789\n", (unsigned long) created); + fclose (fp); + + table = hsts_store_open (file); + + TEST_URL_RW (table, "foo.example.com", 80); + TEST_URL_RW (table, "www.foo.example.com", 80); + TEST_URL_RW (table, "bar.example.com", 80); + + TEST_URL_NORW(table, "www.bar.example.com", 80); + + TEST_URL_RW (table, "test.example.com", 8080); + + hsts_store_close (table); + close_hsts_test_store (table); + unlink (file); + } + xfree (file); + } + + return NULL; +} +#endif /* TESTING */ +#endif /* HAVE_HSTS */ diff --git a/src/hsts.h b/src/hsts.h new file mode 100644 index 0000000..7cc41e3 --- /dev/null +++ b/src/hsts.h @@ -0,0 +1,53 @@ +/* Declarations for hsts.c + Copyright (C) 1996-2012, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef WGET_HSTS_H +#define WGET_HSTS_H + +#ifdef HAVE_HSTS + +#include "wget.h" +#include "url.h" + +typedef struct hsts_store *hsts_store_t; + +hsts_store_t hsts_store_open (const char *); + +void hsts_store_save (hsts_store_t, const char *); +void hsts_store_close (hsts_store_t); +bool hsts_store_has_changed (hsts_store_t); + +bool hsts_store_entry (hsts_store_t, + enum url_scheme, const char *, int, + time_t, bool); +bool hsts_match (hsts_store_t, struct url *); + +#endif /* HAVE_HSTS */ +#endif /* WGET_HSTS_H */ diff --git a/src/html-parse.c b/src/html-parse.c new file mode 100644 index 0000000..6367732 --- /dev/null +++ b/src/html-parse.c @@ -0,0 +1,1221 @@ +/* HTML parser for Wget. + Copyright (C) 1998-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +/* The only entry point to this module is map_html_tags(), which see. */ + +/* TODO: + + - Allow hooks for callers to process contents outside tags. This + is needed to implement handling <style> and <script>. The + taginfo structure already carries the information about where the + tags are, but this is not enough, because one would also want to + skip the comments. (The funny thing is that for <style> and + <script> you *don't* want to skip comments!) + + - Create a test suite for regression testing. */ + +/* HISTORY: + + This is the third HTML parser written for Wget. The first one was + written some time during the Geturl 1.0 beta cycle, and was very + inefficient and buggy. It also contained some very complex code to + remember a list of parser states, because it was supposed to be + reentrant. + + The second HTML parser was written for Wget 1.4 (the first version + by the name `Wget'), and was a complete rewrite. Although the new + parser behaved much better and made no claims of reentrancy, it + still shared many of the fundamental flaws of the old version -- it + only regarded HTML in terms tag-attribute pairs, where the + attribute's value was a URL to be returned. Any other property of + HTML, such as <base href=...>, or strange way to specify a URL, + such as <meta http-equiv=Refresh content="0; URL=..."> had to be + crudely hacked in -- and the caller had to be aware of these hacks. + Like its predecessor, this parser did not support HTML comments. + + After Wget 1.5.1 was released, I set out to write a third HTML + parser. The objectives of the new parser were to: (1) provide a + clean way to analyze HTML lexically, (2) separate interpretation of + the markup from the parsing process, (3) be as correct as possible, + e.g. correctly skipping comments and other SGML declarations, (4) + understand the most common errors in markup and skip them or be + relaxed towrds them, and (5) be reasonably efficient (no regexps, + minimum copying and minimum or no heap allocation). + + I believe this parser meets all of the above goals. It is + reasonably well structured, and could be relatively easily + separated from Wget and used elsewhere. While some of its + intrinsic properties limit its value as a general-purpose HTML + parser, I believe that, with minimum modifications, it could serve + as a backend for one. + + Due to time and other constraints, this parser was not integrated + into Wget until the version 1.7. */ + +/* DESCRIPTION: + + The single entry point of this parser is map_html_tags(), which + works by calling a function you specify for each tag. The function + gets called with the pointer to a structure describing the tag and + its attributes. */ + +/* To test as standalone, compile with `-DSTANDALONE -I.'. You'll + still need Wget headers to compile. */ + +#include "wget.h" + +#ifdef STANDALONE +# define I_REALLY_WANT_CTYPE_MACROS +#endif + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#include "utils.h" +#include "html-parse.h" + +#ifdef STANDALONE +# undef xmalloc +# undef xrealloc +# undef xfree +# define xmalloc malloc +# define xrealloc realloc +# define xfree free + +# undef c_isspace +# undef c_isdigit +# undef c_isxdigit +# undef c_isalpha +# undef c_isalnum +# undef c_tolower +# undef c_toupper + +# define c_isspace(x) isspace (x) +# define c_isdigit(x) isdigit (x) +# define c_isxdigit(x) isxdigit (x) +# define c_isalpha(x) isalpha (x) +# define c_isalnum(x) isalnum (x) +# define c_tolower(x) tolower (x) +# define c_toupper(x) toupper (x) + +struct hash_table { + int dummy; +}; +static void * +hash_table_get (const struct hash_table *ht, void *ptr) +{ + return ptr; +} +#else /* not STANDALONE */ +# include "hash.h" +#endif + +/* Pool support. A pool is a resizable chunk of memory. It is first + allocated on the stack, and moved to the heap if it needs to be + larger than originally expected. map_html_tags() uses it to store + the zero-terminated names and values of tags and attributes. + + Thus taginfo->name, and attr->name and attr->value for each + attribute, do not point into separately allocated areas, but into + different parts of the pool, separated only by terminating zeros. + This ensures minimum amount of allocation and, for most tags, no + allocation because the entire pool is kept on the stack. */ + +struct pool { + char *contents; /* pointer to the contents. */ + int size; /* size of the pool. */ + int tail; /* next available position index. */ + bool resized; /* whether the pool has been resized + using malloc. */ + + char *orig_contents; /* original pool contents, usually + stack-allocated. used by POOL_FREE + to restore the pool to the initial + state. */ + int orig_size; +}; + +/* Initialize the pool to hold INITIAL_SIZE bytes of storage. */ + +#define POOL_INIT(p, initial_storage, initial_size) do { \ + struct pool *P = (p); \ + P->contents = (initial_storage); \ + P->size = (initial_size); \ + P->tail = 0; \ + P->resized = false; \ + P->orig_contents = P->contents; \ + P->orig_size = P->size; \ +} while (0) + +/* Grow the pool to accommodate at least SIZE new bytes. If the pool + already has room to accommodate SIZE bytes of data, this is a no-op. */ + +#define POOL_GROW(p, increase) \ + GROW_ARRAY ((p)->contents, (p)->size, (p)->tail + (increase), \ + (p)->resized, char) + +/* Append text in the range [beg, end) to POOL. No zero-termination + is done. */ + +#define POOL_APPEND(p, beg, end) do { \ + const char *PA_beg = (beg); \ + int PA_size = (end) - PA_beg; \ + POOL_GROW (p, PA_size); \ + memcpy ((p)->contents + (p)->tail, PA_beg, PA_size); \ + (p)->tail += PA_size; \ +} while (0) + +/* Append one character to the pool. Can be used to zero-terminate + pool strings. */ + +#define POOL_APPEND_CHR(p, ch) do { \ + char PAC_char = (ch); \ + POOL_GROW (p, 1); \ + (p)->contents[(p)->tail++] = PAC_char; \ +} while (0) + +/* Forget old pool contents. The allocated memory is not freed. */ +#define POOL_REWIND(p) (p)->tail = 0 + +/* Free heap-allocated memory for contents of POOL. This calls + xfree() if the memory was allocated through malloc. It also + restores `contents' and `size' to their original, pre-malloc + values. That way after POOL_FREE, the pool is fully usable, just + as if it were freshly initialized with POOL_INIT. */ + +#define POOL_FREE(p) do { \ + struct pool *P = p; \ + if (P->resized) \ + xfree (P->contents); \ + P->contents = P->orig_contents; \ + P->size = P->orig_size; \ + P->tail = 0; \ + P->resized = false; \ +} while (0) + +/* Used for small stack-allocated memory chunks that might grow. Like + DO_REALLOC, this macro grows BASEVAR as necessary to take + NEEDED_SIZE items of TYPE. + + The difference is that on the first resize, it will use + malloc+memcpy rather than realloc. That way you can stack-allocate + the initial chunk, and only resort to heap allocation if you + stumble upon large data. + + After the first resize, subsequent ones are performed with realloc, + just like DO_REALLOC. */ + +#define GROW_ARRAY(basevar, sizevar, needed_size, resized, type) do { \ + long ga_needed_size = (needed_size); \ + long ga_newsize = (sizevar); \ + while (ga_newsize < ga_needed_size) \ + ga_newsize <<= 1; \ + if (ga_newsize != (sizevar)) \ + { \ + if (resized) \ + basevar = xrealloc (basevar, ga_newsize * sizeof (type)); \ + else \ + { \ + void *ga_new = xmalloc (ga_newsize * sizeof (type)); \ + memcpy (ga_new, basevar, (sizevar) * sizeof (type)); \ + (basevar) = ga_new; \ + resized = true; \ + } \ + (sizevar) = ga_newsize; \ + } \ +} while (0) + +/* Test whether n+1-sized entity name fits in P. We don't support + IE-style non-terminated entities, e.g. "<foo" -> "<foo". + However, "<foo" will work, as will "<!foo", "<", etc. In + other words an entity needs to be terminated by either a + non-alphanumeric or the end of string. */ +#define FITS(p, n) (p + n == end || (p + n < end && !c_isalnum (p[n]))) + +/* Macros that test entity names by returning true if P is followed by + the specified characters. */ +#define ENT1(p, c0) (FITS (p, 1) && p[0] == c0) +#define ENT2(p, c0, c1) (FITS (p, 2) && p[0] == c0 && p[1] == c1) +#define ENT3(p, c0, c1, c2) (FITS (p, 3) && p[0]==c0 && p[1]==c1 && p[2]==c2) + +/* Increment P by INC chars. If P lands at a semicolon, increment it + past the semicolon. This ensures that e.g. "<foo" is converted + to "<foo", but "<,foo" to "<,foo". */ +#define SKIP_SEMI(p, inc) (p += inc, p < end && *p == ';' ? ++p : p) + +struct tagstack_item { + const char *tagname_begin; + const char *tagname_end; + const char *contents_begin; + struct tagstack_item *prev; + struct tagstack_item *next; +}; + +static struct tagstack_item * +tagstack_push (struct tagstack_item **head, struct tagstack_item **tail) +{ + struct tagstack_item *ts = xmalloc(sizeof(struct tagstack_item)); + if (*head == NULL) + { + *head = *tail = ts; + ts->prev = ts->next = NULL; + } + else + { + (*tail)->next = ts; + ts->prev = *tail; + *tail = ts; + ts->next = NULL; + } + + return ts; +} + +/* remove ts and everything after it from the stack */ +static void +tagstack_pop (struct tagstack_item **head, struct tagstack_item **tail, + struct tagstack_item *ts) +{ + if (*head == NULL) + return; + + if (ts == *tail) + { + if (ts == *head) + { + xfree (ts); + *head = *tail = NULL; + } + else + { + ts->prev->next = NULL; + *tail = ts->prev; + xfree (ts); + } + } + else + { + if (ts == *head) + { + *head = NULL; + } + *tail = ts->prev; + + if (ts->prev) + { + ts->prev->next = NULL; + } + while (ts) + { + struct tagstack_item *p = ts->next; + xfree (ts); + ts = p; + } + } +} + +static struct tagstack_item * +tagstack_find (struct tagstack_item *tail, const char *tagname_begin, + const char *tagname_end) +{ + int len = tagname_end - tagname_begin; + while (tail) + { + if (len == (tail->tagname_end - tail->tagname_begin)) + { + if (0 == strncasecmp (tail->tagname_begin, tagname_begin, len)) + return tail; + } + tail = tail->prev; + } + return NULL; +} + +/* Decode the HTML character entity at *PTR, considering END to be end + of buffer. It is assumed that the "&" character that marks the + beginning of the entity has been seen at *PTR-1. If a recognized + ASCII entity is seen, it is returned, and *PTR is moved to the end + of the entity. Otherwise, -1 is returned and *PTR left unmodified. + + The recognized entities are: <, >, &, &apos, and ". */ + +static int +decode_entity (const char **ptr, const char *end) +{ + const char *p = *ptr; + int value = -1; + + if (++p == end) + return -1; + + switch (*p++) + { + case '#': + /* Process numeric entities "&#DDD;" and "&#xHH;". */ + { + int digits = 0; + value = 0; + if (*p == 'x') + for (++p; value < 256 && p < end && c_isxdigit (*p); p++, digits++) + value = (value << 4) + _unhex (*p); + else + for (; value < 256 && p < end && c_isdigit (*p); p++, digits++) + value = (value * 10) + (*p - '0'); + if (!digits) + return -1; + /* Don't interpret 128+ codes and NUL because we cannot + portably reinserted them into HTML. */ + if (!value || (value & ~0x7f)) + return -1; + *ptr = SKIP_SEMI (p, 0); + return value; + } + /* Process named ASCII entities. */ + case 'g': + if (ENT1 (p, 't')) + value = '>', *ptr = SKIP_SEMI (p, 1); + break; + case 'l': + if (ENT1 (p, 't')) + value = '<', *ptr = SKIP_SEMI (p, 1); + break; + case 'a': + if (ENT2 (p, 'm', 'p')) + value = '&', *ptr = SKIP_SEMI (p, 2); + else if (ENT3 (p, 'p', 'o', 's')) + /* handle &apos for the sake of the XML/XHTML crowd. */ + value = '\'', *ptr = SKIP_SEMI (p, 3); + break; + case 'q': + if (ENT3 (p, 'u', 'o', 't')) + value = '\"', *ptr = SKIP_SEMI (p, 3); + break; + } + return value; +} +#undef ENT1 +#undef ENT2 +#undef ENT3 +#undef FITS +#undef SKIP_SEMI + +enum { + AP_DOWNCASE = 1, + AP_DECODE_ENTITIES = 2, + AP_TRIM_BLANKS = 4 +}; + +/* Copy the text in the range [BEG, END) to POOL, optionally + performing operations specified by FLAGS. FLAGS may be any + combination of AP_DOWNCASE, AP_DECODE_ENTITIES and AP_TRIM_BLANKS + with the following meaning: + + * AP_DOWNCASE -- downcase all the letters; + + * AP_DECODE_ENTITIES -- decode the named and numeric entities in + the ASCII range when copying the string. + + * AP_TRIM_BLANKS -- ignore blanks at the beginning and at the end + of text, as well as embedded newlines. */ + +static void +convert_and_copy (struct pool *pool, const char *beg, const char *end, int flags) +{ + int old_tail = pool->tail; + + /* Skip blanks if required. We must do this before entities are + processed, so that blanks can still be inserted as, for instance, + ` '. */ + if (flags & AP_TRIM_BLANKS) + { + while (beg < end && c_isspace (*beg)) + ++beg; + while (end > beg && c_isspace (end[-1])) + --end; + } + + if (flags & AP_DECODE_ENTITIES) + { + /* Grow the pool, then copy the text to the pool character by + character, processing the encountered entities as we go + along. + + It's safe (and necessary) to grow the pool in advance because + processing the entities can only *shorten* the string, it can + never lengthen it. */ + const char *from = beg; + char *to; + bool squash_newlines = !!(flags & AP_TRIM_BLANKS); + + POOL_GROW (pool, end - beg); + to = pool->contents + pool->tail; + + while (from < end) + { + if (*from == '&') + { + int entity = decode_entity (&from, end); + if (entity != -1) + *to++ = entity; + else + *to++ = *from++; + } + else if ((*from == '\n' || *from == '\r') && squash_newlines) + ++from; + else + *to++ = *from++; + } + /* Verify that we haven't exceeded the original size. (It + shouldn't happen, hence the assert.) */ + assert (to - (pool->contents + pool->tail) <= end - beg); + + /* Make POOL's tail point to the position following the string + we've written. */ + pool->tail = to - pool->contents; + POOL_APPEND_CHR (pool, '\0'); + } + else + { + /* Just copy the text to the pool. */ + POOL_APPEND (pool, beg, end); + POOL_APPEND_CHR (pool, '\0'); + } + + if (flags & AP_DOWNCASE) + { + char *p = pool->contents + old_tail; + for (; *p; p++) + *p = c_tolower (*p); + } +} + +/* Originally we used to adhere to rfc 1866 here, and allowed only + letters, digits, periods, and hyphens as names (of tags or + attributes). However, this broke too many pages which used + proprietary or strange attributes, e.g. <img src="a.gif" + v:shapes="whatever">. + + So now we allow any character except: + * whitespace + * 8-bit and control chars + * characters that clearly cannot be part of name: + '=', '<', '>', '/'. + + This only affects attribute and tag names; attribute values allow + an even greater variety of characters. */ + +#define NAME_CHAR_P(x) ((x) > 32 && (x) < 127 \ + && (x) != '=' && (x) != '<' && (x) != '>' \ + && (x) != '/') + +#ifdef STANDALONE +static int comment_backout_count; +#endif + +/* Advance over an SGML declaration, such as <!DOCTYPE ...>. In + strict comments mode, this is used for skipping over comments as + well. + + To recap: any SGML declaration may have comments associated with + it, e.g. + <!MY-DECL -- isn't this fun? -- foo bar> + + An HTML comment is merely an empty declaration (<!>) with a comment + attached, like this: + <!-- some stuff here --> + + Several comments may be embedded in one comment declaration: + <!-- have -- -- fun --> + + Whitespace is allowed between and after the comments, but not + before the first comment. Additionally, this function attempts to + handle double quotes in SGML declarations correctly. */ + +static const char * +advance_declaration (const char *beg, const char *end) +{ + const char *p = beg; + char quote_char = '\0'; /* shut up, gcc! */ + char ch; + + enum { + AC_S_DONE, + AC_S_BACKOUT, + AC_S_BANG, + AC_S_DEFAULT, + AC_S_DCLNAME, + AC_S_DASH1, + AC_S_DASH2, + AC_S_COMMENT, + AC_S_DASH3, + AC_S_DASH4, + AC_S_QUOTE1, + AC_S_IN_QUOTE, + AC_S_QUOTE2 + } state = AC_S_BANG; + + if (beg == end) + return beg; + ch = *p++; + + /* It looked like a good idea to write this as a state machine, but + now I wonder... */ + + while (state != AC_S_DONE && state != AC_S_BACKOUT) + { + if (p == end) + state = AC_S_BACKOUT; + switch (state) + { + case AC_S_DONE: + case AC_S_BACKOUT: + break; + case AC_S_BANG: + if (ch == '!') + { + ch = *p++; + state = AC_S_DEFAULT; + } + else + state = AC_S_BACKOUT; + break; + case AC_S_DEFAULT: + switch (ch) + { + case '-': + state = AC_S_DASH1; + break; + case ' ': + case '\t': + case '\r': + case '\n': + ch = *p++; + break; + case '<': + case '>': + state = AC_S_DONE; + break; + case '\'': + case '\"': + state = AC_S_QUOTE1; + break; + default: + if (NAME_CHAR_P (ch)) + state = AC_S_DCLNAME; + else + state = AC_S_BACKOUT; + break; + } + break; + case AC_S_DCLNAME: + if (ch == '-') + state = AC_S_DASH1; + else if (NAME_CHAR_P (ch)) + ch = *p++; + else + state = AC_S_DEFAULT; + break; + case AC_S_QUOTE1: + /* We must use 0x22 because broken assert macros choke on + '"' and '\"'. */ + assert (ch == '\'' || ch == 0x22); + quote_char = ch; /* cheating -- I really don't feel like + introducing more different states for + different quote characters. */ + ch = *p++; + state = AC_S_IN_QUOTE; + break; + case AC_S_IN_QUOTE: + if (ch == quote_char) + state = AC_S_QUOTE2; + else + ch = *p++; + break; + case AC_S_QUOTE2: + assert (ch == quote_char); + ch = *p++; + state = AC_S_DEFAULT; + break; + case AC_S_DASH1: + assert (ch == '-'); + ch = *p++; + state = AC_S_DASH2; + break; + case AC_S_DASH2: + switch (ch) + { + case '-': + ch = *p++; + state = AC_S_COMMENT; + break; + default: + state = AC_S_BACKOUT; + } + break; + case AC_S_COMMENT: + switch (ch) + { + case '-': + state = AC_S_DASH3; + break; + default: + ch = *p++; + break; + } + break; + case AC_S_DASH3: + assert (ch == '-'); + ch = *p++; + state = AC_S_DASH4; + break; + case AC_S_DASH4: + switch (ch) + { + case '-': + ch = *p++; + state = AC_S_DEFAULT; + break; + default: + state = AC_S_COMMENT; + break; + } + break; + } + } + + if (state == AC_S_BACKOUT) + { +#ifdef STANDALONE + ++comment_backout_count; +#endif + return beg + 1; + } + return p; +} + +/* Find the first occurrence of the substring "-->" in [BEG, END) and + return the pointer to the character after the substring. If the + substring is not found, return NULL. */ + +static const char * +find_comment_end (const char *beg, const char *end) +{ + /* Open-coded Boyer-Moore search for "-->". Examine the third char; + if it's not '>' or '-', advance by three characters. Otherwise, + look at the preceding characters and try to find a match. */ + + const char *p = beg - 1; + + while ((p += 3) < end) + switch (p[0]) + { + case '>': + if (p[-1] == '-' && p[-2] == '-') + return p + 1; + break; + case '-': + at_dash: + if (p[-1] == '-') + { + at_dash_dash: + if (++p == end) return NULL; + switch (p[0]) + { + case '>': return p + 1; + case '-': goto at_dash_dash; + } + } + else + { + if ((p += 2) >= end) return NULL; + switch (p[0]) + { + case '>': + if (p[-1] == '-') + return p + 1; + break; + case '-': + goto at_dash; + } + } + } + return NULL; +} + +/* Return true if the string containing of characters inside [b, e) is + present in hash table HT. */ + +static bool +name_allowed (const struct hash_table *ht, const char *b, const char *e) +{ + char buf[256], *copy; + size_t len = e - b; + bool ret; + + if (!ht) + return true; + + if (len < sizeof (buf)) + copy = buf; + else + copy = xmalloc (len + 1); + + memcpy (copy, b, len); + copy[len] = 0; + + ret = hash_table_get (ht, copy) != NULL; + + if (copy != buf) + xfree (copy); + + return ret; +} + +/* Advance P (a char pointer), with the explicit intent of being able + to read the next character. If this is not possible, go to finish. */ + +#define ADVANCE(p) do { \ + ++p; \ + if (p >= end) \ + goto finish; \ +} while (0) + +/* Skip whitespace, if any. */ + +#define SKIP_WS(p) do { \ + while (c_isspace (*p)) { \ + ADVANCE (p); \ + } \ +} while (0) + +#ifdef STANDALONE +static int tag_backout_count; +#endif + +/* Map MAPFUN over HTML tags in TEXT, which is SIZE characters long. + MAPFUN will be called with two arguments: pointer to an initialized + struct taginfo, and MAPARG. + + ALLOWED_TAGS and ALLOWED_ATTRIBUTES are hash tables the keys of + which are the tags and attribute names that this function should + use. If ALLOWED_TAGS is NULL, all tags are processed; if + ALLOWED_ATTRIBUTES is NULL, all attributes are returned. + + (Obviously, the caller can filter out unwanted tags and attributes + just as well, but this is just an optimization designed to avoid + unnecessary copying of tags/attributes which the caller doesn't + care about.) */ + +void +map_html_tags (const char *text, int size, + void (*mapfun) (struct taginfo *, void *), void *maparg, + int flags, + const struct hash_table *allowed_tags, + const struct hash_table *allowed_attributes) +{ + /* storage for strings passed to MAPFUN callback; if 256 bytes is + too little, POOL_APPEND allocates more with malloc. */ + char pool_initial_storage[256]; + struct pool pool; + + const char *p = text; + const char *end = text + size; + + struct attr_pair attr_pair_initial_storage[8]; + int attr_pair_size = countof (attr_pair_initial_storage); + bool attr_pair_resized = false; + struct attr_pair *pairs = attr_pair_initial_storage; + + struct tagstack_item *head = NULL; + struct tagstack_item *tail = NULL; + + if (!size) + return; + + POOL_INIT (&pool, pool_initial_storage, countof (pool_initial_storage)); + + { + int nattrs, end_tag; + const char *tag_name_begin, *tag_name_end; + const char *tag_start_position; + bool uninteresting_tag; + + look_for_tag: + POOL_REWIND (&pool); + + nattrs = 0; + end_tag = 0; + + /* Find beginning of tag. We use memchr() instead of the usual + looping with ADVANCE() for speed. */ + p = memchr (p, '<', end - p); + if (!p) + goto finish; + + tag_start_position = p; + ADVANCE (p); + + /* Establish the type of the tag (start-tag, end-tag or + declaration). */ + if (*p == '!') + { + if (!(flags & MHT_STRICT_COMMENTS) + && p + 3 < end && p[1] == '-' && p[2] == '-') + { + /* If strict comments are not enforced and if we know + we're looking at a comment, simply look for the + terminating "-->". Non-strict is the default because + it works in other browsers and most HTML writers can't + be bothered with getting the comments right. */ + const char *comment_end = find_comment_end (p + 3, end); + if (comment_end) + p = comment_end; + } + else + { + /* Either in strict comment mode or looking at a non-empty + declaration. Real declarations are much less likely to + be misused the way comments are, so advance over them + properly regardless of strictness. */ + p = advance_declaration (p, end); + } + if (p == end) + goto finish; + goto look_for_tag; + } + else if (*p == '/') + { + end_tag = 1; + ADVANCE (p); + } + tag_name_begin = p; + while (NAME_CHAR_P (*p)) + ADVANCE (p); + if (p == tag_name_begin) + goto look_for_tag; + tag_name_end = p; + SKIP_WS (p); + + if (!end_tag) + { + struct tagstack_item *ts = tagstack_push (&head, &tail); + if (ts) + { + ts->tagname_begin = tag_name_begin; + ts->tagname_end = tag_name_end; + ts->contents_begin = NULL; + } + } + + if (end_tag && *p != '>' && *p != '<') + goto backout_tag; + + if (!name_allowed (allowed_tags, tag_name_begin, tag_name_end)) + /* We can't just say "goto look_for_tag" here because we need + the loop below to properly advance over the tag's attributes. */ + uninteresting_tag = true; + else + { + uninteresting_tag = false; + convert_and_copy (&pool, tag_name_begin, tag_name_end, AP_DOWNCASE); + } + + /* Find the attributes. */ + while (1) + { + const char *attr_name_begin, *attr_name_end; + const char *attr_value_begin, *attr_value_end; + const char *attr_raw_value_begin, *attr_raw_value_end; + int operation = AP_DOWNCASE; /* stupid compiler. */ + + SKIP_WS (p); + + if (*p == '/') + { + /* A slash at this point means the tag is about to be + closed. This is legal in XML and has been popularized + in HTML via XHTML. */ + /* <foo a=b c=d /> */ + /* ^ */ + ADVANCE (p); + SKIP_WS (p); + if (*p != '<' && *p != '>') + goto backout_tag; + } + + /* Check for end of tag definition. */ + if (*p == '<' || *p == '>') + break; + + /* Establish bounds of attribute name. */ + attr_name_begin = p; /* <foo bar ...> */ + /* ^ */ + while (NAME_CHAR_P (*p)) + ADVANCE (p); + attr_name_end = p; /* <foo bar ...> */ + /* ^ */ + if (attr_name_begin == attr_name_end) + goto backout_tag; + + /* Establish bounds of attribute value. */ + SKIP_WS (p); + + if (NAME_CHAR_P (*p) || *p == '/' || *p == '<' || *p == '>') + { + /* Minimized attribute syntax allows `=' to be omitted. + For example, <UL COMPACT> is a valid shorthand for <UL + COMPACT="compact">. Even if such attributes are not + useful to Wget, we need to support them, so that the + tags containing them can be parsed correctly. */ + attr_raw_value_begin = attr_value_begin = attr_name_begin; + attr_raw_value_end = attr_value_end = attr_name_end; + } + else if (*p == '=') + { + ADVANCE (p); + SKIP_WS (p); + if (*p == '\"' || *p == '\'') + { + bool newline_seen = false; + char quote_char = *p; + attr_raw_value_begin = p; + ADVANCE (p); + attr_value_begin = p; /* <foo bar="baz"> */ + /* ^ */ + while (*p != quote_char) + { + if (!newline_seen && *p == '\n') + { + /* If a newline is seen within the quotes, it + is most likely that someone forgot to close + the quote. In that case, we back out to + the value beginning, and terminate the tag + at either `>' or the delimiter, whichever + comes first. Such a tag terminated at `>' + is discarded. */ + p = attr_value_begin; + newline_seen = true; + continue; + } + else if (newline_seen && (*p == '<' || *p == '>')) + break; + ADVANCE (p); + } + attr_value_end = p; /* <foo bar="baz"> */ + /* ^ */ + if (*p == quote_char) + ADVANCE (p); + else + goto look_for_tag; + attr_raw_value_end = p; /* <foo bar="baz"> */ + /* ^ */ + operation = AP_DECODE_ENTITIES; + if (flags & MHT_TRIM_VALUES) + operation |= AP_TRIM_BLANKS; + } + else + { + attr_value_begin = p; /* <foo bar=baz> */ + /* ^ */ + /* According to SGML, a name token should consist only + of alphanumerics, . and -. However, this is often + violated by, for instance, `%' in `width=75%'. + We'll be liberal and allow just about anything as + an attribute value. */ + while (!c_isspace (*p) && *p != '<' && *p != '>') + ADVANCE (p); + attr_value_end = p; /* <foo bar=baz qux=quix> */ + /* ^ */ + if (attr_value_begin == attr_value_end) + /* <foo bar=> */ + /* ^ */ + goto backout_tag; + attr_raw_value_begin = attr_value_begin; + attr_raw_value_end = attr_value_end; + operation = AP_DECODE_ENTITIES; + } + } + else + { + /* We skipped the whitespace and found something that is + neither `=' nor the beginning of the next attribute's + name. Back out. */ + goto backout_tag; /* <foo bar [... */ + /* ^ */ + } + + /* If we're not interested in the tag, don't bother with any + of the attributes. */ + if (uninteresting_tag) + continue; + + /* If we aren't interested in the attribute, skip it. We + cannot do this test any sooner, because our text pointer + needs to correctly advance over the attribute. */ + if (!name_allowed (allowed_attributes, attr_name_begin, attr_name_end)) + continue; + + GROW_ARRAY (pairs, attr_pair_size, nattrs + 1, attr_pair_resized, + struct attr_pair); + + pairs[nattrs].name_pool_index = pool.tail; + convert_and_copy (&pool, attr_name_begin, attr_name_end, AP_DOWNCASE); + + pairs[nattrs].value_pool_index = pool.tail; + convert_and_copy (&pool, attr_value_begin, attr_value_end, operation); + pairs[nattrs].value_raw_beginning = attr_raw_value_begin; + pairs[nattrs].value_raw_size = (attr_raw_value_end + - attr_raw_value_begin); + ++nattrs; + } + + if (!end_tag && tail && (tail->tagname_begin == tag_name_begin)) + { + tail->contents_begin = p+1; + } + + if (uninteresting_tag) + { + ADVANCE (p); + goto look_for_tag; + } + + /* By now, we have a valid tag with a name and zero or more + attributes. Fill in the data and call the mapper function. */ + { + int i; + struct taginfo taginfo; + struct tagstack_item *ts = NULL; + + taginfo.name = pool.contents; + taginfo.end_tag_p = end_tag; + taginfo.nattrs = nattrs; + /* We fill in the char pointers only now, when pool can no + longer get realloc'ed. If we did that above, we could get + hosed by reallocation. Obviously, after this point, the pool + may no longer be grown. */ + for (i = 0; i < nattrs; i++) + { + pairs[i].name = pool.contents + pairs[i].name_pool_index; + pairs[i].value = pool.contents + pairs[i].value_pool_index; + } + taginfo.attrs = pairs; + taginfo.start_position = tag_start_position; + taginfo.end_position = p + 1; + taginfo.contents_begin = NULL; + taginfo.contents_end = NULL; + + if (end_tag) + { + ts = tagstack_find (tail, tag_name_begin, tag_name_end); + if (ts) + { + if (ts->contents_begin) + { + taginfo.contents_begin = ts->contents_begin; + taginfo.contents_end = tag_start_position; + } + tagstack_pop (&head, &tail, ts); + } + } + + mapfun (&taginfo, maparg); + if (*p != '<') + ADVANCE (p); + } + goto look_for_tag; + + backout_tag: +#ifdef STANDALONE + ++tag_backout_count; +#endif + /* The tag wasn't really a tag. Treat its contents as ordinary + data characters. */ + p = tag_start_position + 1; + goto look_for_tag; + } + + finish: + POOL_FREE (&pool); + if (attr_pair_resized) + xfree (pairs); + /* pop any tag stack that's left */ + tagstack_pop (&head, &tail, head); +} + +#undef ADVANCE +#undef SKIP_WS +#undef SKIP_NON_WS + +#ifdef STANDALONE +static void +test_mapper (struct taginfo *taginfo, void *arg) +{ + int i; + + printf ("%s%s", taginfo->end_tag_p ? "/" : "", taginfo->name); + for (i = 0; i < taginfo->nattrs; i++) + printf (" %s=%s", taginfo->attrs[i].name, taginfo->attrs[i].value); + putchar ('\n'); + ++*(int *)arg; +} + +int main () +{ + int size = 256; + char *x = xmalloc (size); + int length = 0; + int read_count; + int tag_counter = 0; + +#ifdef ENABLE_NLS + /* Set the current locale. */ + setlocale (LC_ALL, ""); + /* Set the text message domain. */ + bindtextdomain ("wget", LOCALEDIR); + textdomain ("wget"); +#endif /* ENABLE_NLS */ + + while ((read_count = fread (x + length, 1, size - length, stdin))) + { + length += read_count; + size <<= 1; + x = xrealloc (x, size); + } + + map_html_tags (x, length, test_mapper, &tag_counter, 0, NULL, NULL); + printf ("TAGS: %d\n", tag_counter); + printf ("Tag backouts: %d\n", tag_backout_count); + printf ("Comment backouts: %d\n", comment_backout_count); + return 0; +} +#endif /* STANDALONE */ diff --git a/src/html-parse.h b/src/html-parse.h new file mode 100644 index 0000000..eae8e3c --- /dev/null +++ b/src/html-parse.h @@ -0,0 +1,71 @@ +/* Declarations for html-parse.c. + Copyright (C) 1998-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef HTML_PARSE_H +#define HTML_PARSE_H + +struct attr_pair { + char *name; /* attribute name */ + char *value; /* attribute value */ + + /* Needed for URL conversion; the places where the value begins and + ends, including the quotes and everything. */ + const char *value_raw_beginning; + int value_raw_size; + + /* Used internally by map_html_tags. */ + int name_pool_index, value_pool_index; +}; + +struct taginfo { + char *name; /* tag name */ + int end_tag_p; /* whether this is an end-tag */ + int nattrs; /* number of attributes */ + struct attr_pair *attrs; /* attributes */ + + const char *start_position; /* start position of tag */ + const char *end_position; /* end position of tag */ + + const char *contents_begin; /* delimiters of tag contents */ + const char *contents_end; /* only valid if end_tag_p */ +}; + +struct hash_table; /* forward declaration */ + +/* Flags for map_html_tags: */ +#define MHT_STRICT_COMMENTS 1 /* use strict comment interpretation */ +#define MHT_TRIM_VALUES 2 /* trim attribute values, e.g. interpret + <a href=" foo "> as "foo" */ + +void map_html_tags (const char *, int, + void (*) (struct taginfo *, void *), void *, int, + const struct hash_table *, const struct hash_table *); + +#endif /* HTML_PARSE_H */ diff --git a/src/html-url.c b/src/html-url.c new file mode 100644 index 0000000..b80cf26 --- /dev/null +++ b/src/html-url.c @@ -0,0 +1,977 @@ +/* Collect URLs from HTML source. + Copyright (C) 1998-2012, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <errno.h> +#include <assert.h> + +#include "exits.h" +#include "html-parse.h" +#include "url.h" +#include "utils.h" +#include "hash.h" +#include "convert.h" +#include "recur.h" +#include "html-url.h" +#include "css-url.h" +#include "c-strcase.h" + +typedef void (*tag_handler_t) (int, struct taginfo *, struct map_context *); + +#define DECLARE_TAG_HANDLER(fun) \ + static void fun (int, struct taginfo *, struct map_context *) + +DECLARE_TAG_HANDLER (tag_find_urls); +DECLARE_TAG_HANDLER (tag_handle_base); +DECLARE_TAG_HANDLER (tag_handle_form); +DECLARE_TAG_HANDLER (tag_handle_img); +DECLARE_TAG_HANDLER (tag_handle_link); +DECLARE_TAG_HANDLER (tag_handle_meta); + +enum { + TAG_A, + TAG_APPLET, + TAG_AREA, + TAG_BASE, + TAG_BGSOUND, + TAG_BODY, + TAG_EMBED, + TAG_FIG, + TAG_FORM, + TAG_FRAME, + TAG_IFRAME, + TAG_IMG, + TAG_INPUT, + TAG_LAYER, + TAG_LINK, + TAG_META, + TAG_OBJECT, + TAG_OVERLAY, + TAG_SCRIPT, + TAG_TABLE, + TAG_TD, + TAG_TH, + TAG_VIDEO, + TAG_AUDIO, + TAG_SOURCE +}; + +/* The list of known tags and functions used for handling them. Most + tags are simply harvested for URLs. */ +static struct known_tag { + int tagid; + const char *name; + tag_handler_t handler; +} known_tags[] = { + { TAG_A, "a", tag_find_urls }, + { TAG_APPLET, "applet", tag_find_urls }, + { TAG_AREA, "area", tag_find_urls }, + { TAG_BASE, "base", tag_handle_base }, + { TAG_BGSOUND, "bgsound", tag_find_urls }, + { TAG_BODY, "body", tag_find_urls }, + { TAG_EMBED, "embed", tag_find_urls }, + { TAG_FIG, "fig", tag_find_urls }, + { TAG_FORM, "form", tag_handle_form }, + { TAG_FRAME, "frame", tag_find_urls }, + { TAG_IFRAME, "iframe", tag_find_urls }, + { TAG_IMG, "img", tag_handle_img }, + { TAG_INPUT, "input", tag_find_urls }, + { TAG_LAYER, "layer", tag_find_urls }, + { TAG_LINK, "link", tag_handle_link }, + { TAG_META, "meta", tag_handle_meta }, + { TAG_OBJECT, "object", tag_find_urls }, + { TAG_OVERLAY, "overlay", tag_find_urls }, + { TAG_SCRIPT, "script", tag_find_urls }, + { TAG_TABLE, "table", tag_find_urls }, + { TAG_TD, "td", tag_find_urls }, + { TAG_TH, "th", tag_find_urls }, + { TAG_VIDEO, "video", tag_find_urls }, + { TAG_AUDIO, "audio", tag_find_urls }, + { TAG_SOURCE, "source", tag_find_urls } +}; + +/* tag_url_attributes documents which attributes of which tags contain + URLs to harvest. It is used by tag_find_urls. */ + +/* Defines for the FLAGS. */ + +/* The link is "inline", i.e. needs to be retrieved for this document + to be correctly rendered. Inline links include inlined images, + stylesheets, children frames, etc. */ +#define ATTR_INLINE 1 + +/* The link is expected to yield HTML contents. It's important not to + try to follow HTML obtained by following e.g. <img src="..."> + regardless of content-type. Doing this causes infinite loops for + "images" that return non-404 error pages with links to the same + image. */ +#define ATTR_HTML 2 + +/* For tags handled by tag_find_urls: attributes that contain URLs to + download. */ +static struct { + int tagid; + const char *attr_name; + int flags; +} tag_url_attributes[] = { + { TAG_A, "href", ATTR_HTML }, + { TAG_APPLET, "code", ATTR_INLINE }, + { TAG_AREA, "href", ATTR_HTML }, + { TAG_BGSOUND, "src", ATTR_INLINE }, + { TAG_BODY, "background", ATTR_INLINE }, + { TAG_EMBED, "href", ATTR_HTML }, + { TAG_EMBED, "src", ATTR_INLINE | ATTR_HTML }, + { TAG_FIG, "src", ATTR_INLINE }, + { TAG_FRAME, "src", ATTR_INLINE | ATTR_HTML }, + { TAG_IFRAME, "src", ATTR_INLINE | ATTR_HTML }, + { TAG_IMG, "href", ATTR_INLINE }, + { TAG_IMG, "lowsrc", ATTR_INLINE }, + { TAG_IMG, "src", ATTR_INLINE }, + { TAG_INPUT, "src", ATTR_INLINE }, + { TAG_LAYER, "src", ATTR_INLINE | ATTR_HTML }, + { TAG_OBJECT, "data", ATTR_INLINE }, + { TAG_OVERLAY, "src", ATTR_INLINE | ATTR_HTML }, + { TAG_SCRIPT, "src", ATTR_INLINE }, + { TAG_TABLE, "background", ATTR_INLINE }, + { TAG_TD, "background", ATTR_INLINE }, + { TAG_TH, "background", ATTR_INLINE }, + { TAG_VIDEO, "src", ATTR_INLINE }, + { TAG_VIDEO, "poster", ATTR_INLINE }, + { TAG_AUDIO, "src", ATTR_INLINE }, + { TAG_AUDIO, "poster", ATTR_INLINE }, + { TAG_SOURCE, "src", ATTR_INLINE } +}; + +/* The lists of interesting tags and attributes are built dynamically, + from the information above. However, some places in the code refer + to the attributes not mentioned here. We add them manually. */ +static const char *additional_attributes[] = { + "rel", /* used by tag_handle_link */ + "type", /* used by tag_handle_link */ + "http-equiv", /* used by tag_handle_meta */ + "name", /* used by tag_handle_meta */ + "content", /* used by tag_handle_meta */ + "action", /* used by tag_handle_form */ + "style", /* used by check_style_attr */ + "srcset", /* used by tag_handle_img */ +}; + +static struct hash_table *interesting_tags; +static struct hash_table *interesting_attributes; + +/* Will contains the (last) charset found in 'http-equiv=content-type' + meta tags */ +static char *meta_charset; + +static void +init_interesting (void) +{ + /* Init the variables interesting_tags and interesting_attributes + that are used by the HTML parser to know which tags and + attributes we're interested in. We initialize this only once, + for performance reasons. + + Here we also make sure that what we put in interesting_tags + matches the user's preferences as specified through --ignore-tags + and --follow-tags. */ + + size_t i; + interesting_tags = make_nocase_string_hash_table (countof (known_tags)); + + /* First, add all the tags we know hot to handle, mapped to their + respective entries in known_tags. */ + for (i = 0; i < countof (known_tags); i++) + hash_table_put (interesting_tags, known_tags[i].name, known_tags + i); + + /* Then remove the tags ignored through --ignore-tags. */ + if (opt.ignore_tags) + { + char **ignored; + for (ignored = opt.ignore_tags; *ignored; ignored++) + hash_table_remove (interesting_tags, *ignored); + } + + /* If --follow-tags is specified, use only those tags. */ + if (opt.follow_tags) + { + /* Create a new table intersecting --follow-tags and known_tags, + and use it as interesting_tags. */ + struct hash_table *intersect = make_nocase_string_hash_table (0); + char **followed; + for (followed = opt.follow_tags; *followed; followed++) + { + struct known_tag *t = hash_table_get (interesting_tags, *followed); + if (!t) + continue; /* ignore unknown --follow-tags entries. */ + hash_table_put (intersect, *followed, t); + } + hash_table_destroy (interesting_tags); + interesting_tags = intersect; + } + + /* Add the attributes we care about. */ + interesting_attributes = make_nocase_string_hash_table (10); + for (i = 0; i < countof (additional_attributes); i++) + hash_table_put (interesting_attributes, additional_attributes[i], "1"); + for (i = 0; i < countof (tag_url_attributes); i++) + hash_table_put (interesting_attributes, + tag_url_attributes[i].attr_name, "1"); +} + +/* Find the value of attribute named NAME in the taginfo TAG. If the + attribute is not present, return NULL. If ATTRIND is non-NULL, the + index of the attribute in TAG will be stored there. */ + +static char * +find_attr (struct taginfo *tag, const char *name, int *attrind) +{ + int i; + for (i = 0; i < tag->nattrs; i++) + if (!c_strcasecmp (tag->attrs[i].name, name)) + { + if (attrind) + *attrind = i; + return tag->attrs[i].value; + } + return NULL; +} + +/* used for calls to append_url */ +#define ATTR_POS(tag, attrind, ctx) \ + (tag->attrs[attrind].value_raw_beginning - ctx->text) +#define ATTR_SIZE(tag, attrind) \ + (tag->attrs[attrind].value_raw_size) + +/* Append LINK_URI to the urlpos structure that is being built. + + LINK_URI will be merged with the current document base. +*/ + +struct urlpos * +append_url (const char *link_uri, int position, int size, + struct map_context *ctx) +{ + int link_has_scheme = url_has_scheme (link_uri); + struct urlpos *newel; + const char *base = ctx->base ? ctx->base : ctx->parent_base; + struct url *url; + + struct iri *iri = iri_new (); + set_uri_encoding (iri, opt.locale, true); + iri->utf8_encode = true; + + if (!base) + { + DEBUGP (("%s: no base, merge will use \"%s\".\n", + ctx->document_file, link_uri)); + + if (!link_has_scheme) + { + /* Base URL is unavailable, and the link does not have a + location attached to it -- we have to give up. Since + this can only happen when using `--force-html -i', print + a warning. */ + logprintf (LOG_NOTQUIET, + _("%s: Cannot resolve incomplete link %s.\n"), + ctx->document_file, link_uri); + iri_free (iri); + return NULL; + } + + url = url_parse (link_uri, NULL, iri, false); + if (!url) + { + DEBUGP (("%s: link \"%s\" doesn't parse.\n", + ctx->document_file, link_uri)); + iri_free (iri); + return NULL; + } + } + else + { + /* Merge BASE with LINK_URI, but also make sure the result is + canonicalized, i.e. that "../" have been resolved. + (parse_url will do that for us.) */ + + char *complete_uri = uri_merge (base, link_uri); + + DEBUGP (("%s: merge(%s, %s) -> %s\n", + quotearg_n_style (0, escape_quoting_style, ctx->document_file), + quote_n (1, base), + quote_n (2, link_uri), + quotearg_n_style (3, escape_quoting_style, complete_uri))); + + url = url_parse (complete_uri, NULL, iri, false); + if (!url) + { + DEBUGP (("%s: merged link \"%s\" doesn't parse.\n", + ctx->document_file, complete_uri)); + xfree (complete_uri); + iri_free (iri); + return NULL; + } + xfree (complete_uri); + } + + iri_free (iri); + + DEBUGP (("appending %s to urlpos.\n", quote (url->url))); + + newel = xnew0 (struct urlpos); + newel->url = url; + newel->pos = position; + newel->size = size; + + /* A URL is relative if the host is not named, and the name does not + start with `/'. */ + if (!link_has_scheme && *link_uri != '/') + newel->link_relative_p = 1; + else if (link_has_scheme) + newel->link_complete_p = 1; + + /* Append the new URL maintaining the order by position. */ + if (ctx->head == NULL) + ctx->head = newel; + else + { + struct urlpos *it, *prev = NULL; + + it = ctx->head; + while (it && position > it->pos) + { + prev = it; + it = it->next; + } + + newel->next = it; + + if (prev) + prev->next = newel; + else + ctx->head = newel; + } + + return newel; +} + +static void +check_style_attr (struct taginfo *tag, struct map_context *ctx) +{ + int attrind; + int raw_start; + int raw_len; + char *style = find_attr (tag, "style", &attrind); + if (!style) + return; + + /* raw pos and raw size include the quotes, skip them when they are + present. */ + raw_start = ATTR_POS (tag, attrind, ctx); + raw_len = ATTR_SIZE (tag, attrind); + if( *(char *)(ctx->text + raw_start) == '\'' + || *(char *)(ctx->text + raw_start) == '"') + { + raw_start += 1; + raw_len -= 2; + } + + if(raw_len <= 0) + return; + + get_urls_css (ctx, raw_start, raw_len); +} + +/* All the tag_* functions are called from collect_tags_mapper, as + specified by KNOWN_TAGS. */ + +/* Default tag handler: collect URLs from attributes specified for + this tag by tag_url_attributes. */ + +static void +tag_find_urls (int tagid, struct taginfo *tag, struct map_context *ctx) +{ + size_t i; + int attrind; + int first = -1; + + for (i = 0; i < countof (tag_url_attributes); i++) + if (tag_url_attributes[i].tagid == tagid) + { + /* We've found the index of tag_url_attributes where the + attributes of our tag begin. */ + first = i; + break; + } + assert (first != -1); + + /* Loop over the "interesting" attributes of this tag. In this + example, it will loop over "src" and "lowsrc". + + <img src="foo.png" lowsrc="bar.png"> + + This has to be done in the outer loop so that the attributes are + processed in the same order in which they appear in the page. + This is required when converting links. */ + + for (attrind = 0; attrind < tag->nattrs; attrind++) + { + /* Find whether TAG/ATTRIND is a combination that contains a + URL. */ + char *link = tag->attrs[attrind].value; + const size_t size = countof (tag_url_attributes); + + /* If you're cringing at the inefficiency of the nested loops, + remember that they both iterate over a very small number of + items. The worst-case inner loop is for the IMG tag, which + has three attributes. */ + for (i = first; i < size && tag_url_attributes[i].tagid == tagid; i++) + { + if (0 == strcasecmp (tag->attrs[attrind].name, + tag_url_attributes[i].attr_name)) + { + struct urlpos *up = append_url (link, ATTR_POS(tag,attrind,ctx), + ATTR_SIZE(tag,attrind), ctx); + if (up) + { + int flags = tag_url_attributes[i].flags; + if (flags & ATTR_INLINE) + up->link_inline_p = 1; + if (flags & ATTR_HTML) + up->link_expect_html = 1; + } + } + } + } +} + +/* Handle the BASE tag, for <base href=...>. */ + +static void +tag_handle_base (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *ctx) +{ + struct urlpos *base_urlpos; + int attrind; + char *newbase = find_attr (tag, "href", &attrind); + if (!newbase) + return; + + base_urlpos = append_url (newbase, ATTR_POS(tag,attrind,ctx), + ATTR_SIZE(tag,attrind), ctx); + if (!base_urlpos) + return; + base_urlpos->ignore_when_downloading = 1; + base_urlpos->link_base_p = 1; + + xfree (ctx->base); + if (ctx->parent_base) + ctx->base = uri_merge (ctx->parent_base, newbase); + else + ctx->base = xstrdup (newbase); +} + +/* Mark the URL found in <form action=...> for conversion. */ + +static void +tag_handle_form (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *ctx) +{ + int attrind; + char *action = find_attr (tag, "action", &attrind); + + if (action) + { + struct urlpos *up = append_url (action, ATTR_POS(tag,attrind,ctx), + ATTR_SIZE(tag,attrind), ctx); + if (up) + up->ignore_when_downloading = 1; + } +} + +/* Handle the LINK tag. It requires special handling because how its + links will be followed in -p mode depends on the REL attribute. */ + +static void +tag_handle_link (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *ctx) +{ + int attrind; + char *href = find_attr (tag, "href", &attrind); + + /* All <link href="..."> link references are external, except those + known not to be, such as style sheet and shortcut icon: + + <link rel="stylesheet" href="..."> + <link rel="shortcut icon" href="..."> + */ + if (href) + { + struct urlpos *up = append_url (href, ATTR_POS(tag,attrind,ctx), + ATTR_SIZE(tag,attrind), ctx); + if (up) + { + char *rel = find_attr (tag, "rel", NULL); + if (rel) + { + if (0 == c_strcasecmp (rel, "stylesheet")) + { + up->link_inline_p = 1; + up->link_expect_css = 1; + } + else if (0 == c_strcasecmp (rel, "shortcut icon")) + { + up->link_inline_p = 1; + } + else + { + /* The external ones usually point to HTML pages, such as + <link rel="next" href="..."> + except when the type attribute says otherwise: + <link rel="alternate" type="application/rss+xml" href=".../?feed=rss2" /> + */ + char *type = find_attr (tag, "type", NULL); + if (!type || c_strcasecmp (type, "text/html") == 0) + up->link_expect_html = 1; + } + } + } + } +} + +/* Handle the META tag. This requires special handling because of the + refresh feature and because of robot exclusion. */ + +static void +tag_handle_meta (int tagid _GL_UNUSED, struct taginfo *tag, struct map_context *ctx) +{ + char *name = find_attr (tag, "name", NULL); + char *http_equiv = find_attr (tag, "http-equiv", NULL); + + if (http_equiv && 0 == c_strcasecmp (http_equiv, "refresh")) + { + /* Some pages use a META tag to specify that the page be + refreshed by a new page after a given number of seconds. The + general format for this is: + + <meta http-equiv=Refresh content="NUMBER; URL=index2.html"> + + So we just need to skip past the "NUMBER; URL=" garbage to + get to the URL. */ + + struct urlpos *entry; + int attrind; + int timeout = 0; + char *p; + + char *refresh = find_attr (tag, "content", &attrind); + if (!refresh) + return; + + for (p = refresh; c_isdigit (*p); p++) + timeout = 10 * timeout + *p - '0'; + if (*p++ != ';') + return; + + while (c_isspace (*p)) + ++p; + if (!( c_toupper (*p) == 'U' + && c_toupper (*(p + 1)) == 'R' + && c_toupper (*(p + 2)) == 'L' + && *(p + 3) == '=')) + return; + p += 4; + while (c_isspace (*p)) + ++p; + + entry = append_url (p, ATTR_POS(tag,attrind,ctx), + ATTR_SIZE(tag,attrind), ctx); + if (entry) + { + entry->link_refresh_p = 1; + entry->refresh_timeout = timeout; + entry->link_expect_html = 1; + } + } + else if (http_equiv && 0 == c_strcasecmp (http_equiv, "content-type")) + { + /* Handle stuff like: + <meta http-equiv="Content-Type" content="text/html; charset=CHARSET"> */ + + char *mcharset; + char *content = find_attr (tag, "content", NULL); + if (!content) + return; + + mcharset = parse_charset (content); + if (!mcharset) + return; + + xfree (meta_charset); + meta_charset = mcharset; + } + else if (name && 0 == c_strcasecmp (name, "robots")) + { + /* Handle stuff like: + <meta name="robots" content="index,nofollow"> */ + char *content = find_attr (tag, "content", NULL); + if (!content) + return; + if (!c_strcasecmp (content, "none")) + ctx->nofollow = true; + else + { + while (*content) + { + char *end; + /* Skip any initial whitespace. */ + content += strspn (content, " \f\n\r\t\v"); + /* Find the next occurrence of ',' or whitespace, + * or the end of the string. */ + end = content + strcspn (content, ", \f\n\r\t\v"); + if (!c_strncasecmp (content, "nofollow", end - content)) + ctx->nofollow = true; + /* Skip past the next comma, if any. */ + if (*end == ',') + ++end; + else + { + end = strchr (end, ','); + if (end) + ++end; + else + end = content + strlen (content); + } + content = end; + } + } + } +} + +/* Handle the IMG tag. This requires special handling for the srcset attr, + while the traditional src/lowsrc/href attributes can be handled generically. +*/ + +static void +tag_handle_img (int tagid, struct taginfo *tag, struct map_context *ctx) { + int attrind; + char *srcset; + + /* Use the generic approach for the attributes without special syntax. */ + tag_find_urls(tagid, tag, ctx); + + srcset = find_attr (tag, "srcset", &attrind); + if (srcset) + { + /* These are relative to the input text. */ + int base_ind = ATTR_POS (tag,attrind,ctx); + int size = strlen (srcset); + + /* These are relative to srcset. */ + int offset, url_start, url_end; + + /* Make sure to line up base_ind with srcset[0], not outside quotes. */ + if (ctx->text[base_ind] == '"' || ctx->text[base_ind] == '\'') + ++base_ind; + + offset = 0; + while (offset < size) + { + bool has_descriptor = true; + + /* Skip over initial whitespace and commas. Note there is no \v + in HTML5 whitespace. */ + url_start = offset + strspn (srcset + offset, " \f\n\r\t,"); + + if (url_start == size) + return; + + /* URL is any non-whitespace chars (including commas) - but with + trailing commas removed. */ + url_end = url_start + strcspn (srcset + url_start, " \f\n\r\t"); + while ((url_end - 1) > url_start && srcset[url_end - 1] == ',') + { + has_descriptor = false; + --url_end; + } + + if (url_end > url_start) + { + char *url_text = strdupdelim (srcset + url_start, + srcset + url_end); + struct urlpos *up = append_url (url_text, base_ind + url_start, + url_end - url_start, ctx); + if (up) + { + up->link_inline_p = 1; + up->link_noquote_html_p = 1; + } + xfree (url_text); + } + + /* If the URL wasn't terminated by a , there may also be a descriptor + which we just skip. */ + if (has_descriptor) + { + /* This is comma-terminated, except there may be one level of + parentheses escaping that. */ + bool in_paren = false; + for (offset = url_end; offset < size; ++offset) + { + char c = srcset[offset]; + if (c == '(') + in_paren = true; + else if (c == ')' && in_paren) + in_paren = false; + else if (c == ',' && !in_paren) + break; + } + } + else + offset = url_end; + } + } +} + +/* Dispatch the tag handler appropriate for the tag we're mapping + over. See known_tags[] for definition of tag handlers. */ + +static void +collect_tags_mapper (struct taginfo *tag, void *arg) +{ + struct map_context *ctx = (struct map_context *)arg; + + /* Find the tag in our table of tags. This must not fail because + map_html_tags only returns tags found in interesting_tags. + + I've changed this for now, I'm passing NULL as interesting_tags + to map_html_tags. This way we can check all tags for a style + attribute. + */ + struct known_tag *t = hash_table_get (interesting_tags, tag->name); + + if (t != NULL) + t->handler (t->tagid, tag, ctx); + + check_style_attr (tag, ctx); + + if (tag->end_tag_p && (0 == c_strcasecmp (tag->name, "style")) + && tag->contents_begin && tag->contents_end + && tag->contents_begin <= tag->contents_end) + { + /* parse contents */ + get_urls_css (ctx, tag->contents_begin - ctx->text, + tag->contents_end - tag->contents_begin); + } +} + +/* Analyze HTML tags FILE and construct a list of URLs referenced from + it. It merges relative links in FILE with URL. It is aware of + <base href=...> and does the right thing. */ + +struct urlpos * +get_urls_html_fm (const char *file, const struct file_memory *fm, + const char *url, bool *meta_disallow_follow, + struct iri *iri) +{ + struct map_context ctx; + int flags; + + ctx.text = fm->content; + ctx.head = NULL; + ctx.base = NULL; + ctx.parent_base = url ? url : opt.base_href; + ctx.document_file = file; + ctx.nofollow = false; + + if (!interesting_tags) + init_interesting (); + + /* Specify MHT_TRIM_VALUES because of buggy HTML generators that + generate <a href=" foo"> instead of <a href="foo"> (browsers + ignore spaces as well.) If you really mean space, use &32; or + %20. MHT_TRIM_VALUES also causes squashing of embedded newlines, + e.g. in <img src="foo.[newline]html">. Such newlines are also + ignored by IE and Mozilla and are presumably introduced by + writing HTML with editors that force word wrap. */ + flags = MHT_TRIM_VALUES; + if (opt.strict_comments) + flags |= MHT_STRICT_COMMENTS; + + /* the NULL here used to be interesting_tags */ + map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags, + NULL, interesting_attributes); + +#ifdef ENABLE_IRI + /* Meta charset is only valid if there was no HTTP header Content-Type charset. */ + /* This is true for HTTP 1.0 and 1.1. */ + if (iri && !iri->content_encoding && meta_charset) + set_content_encoding (iri, meta_charset); +#endif + xfree (meta_charset); + + if (ctx.nofollow) { + logprintf(LOG_VERBOSE, _("no-follow attribute found in %s. Will not follow any links on this page\n"), file); + } + DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow)); + + if (meta_disallow_follow) + *meta_disallow_follow = ctx.nofollow; + + xfree (ctx.base); + return ctx.head; +} + +struct urlpos * +get_urls_html (const char *file, const char *url, bool *meta_disallow_follow, + struct iri *iri) +{ + struct urlpos *urls; + struct file_memory *fm; + + fm = wget_read_file (file); + if (!fm) + { + logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno)); + return NULL; + } + DEBUGP (("Loaded %s (size %s).\n", file, number_to_static_string (fm->length))); + + urls = get_urls_html_fm (file, fm, url, meta_disallow_follow, iri); + wget_read_file_free (fm); + return urls; +} + +/* This doesn't really have anything to do with HTML, but it's similar + to get_urls_html, so we put it here. */ + +struct urlpos * +get_urls_file (const char *file) +{ + struct file_memory *fm; + struct urlpos *head, *tail; + const char *text, *text_end; + + /* Load the file. */ + fm = wget_read_file (file); + if (!fm) + { + logprintf (LOG_NOTQUIET, "%s: %s\n", file, strerror (errno)); + return NULL; + } + DEBUGP (("Loaded %s (size %s).\n", file, number_to_static_string (fm->length))); + + head = tail = NULL; + text = fm->content; + text_end = fm->content + fm->length; + while (text < text_end) + { + int up_error_code; + char *url_text; + char *new_url; + struct urlpos *entry; + struct url *url; + + const char *line_beg = text; + const char *line_end = memchr (text, '\n', text_end - text); + if (!line_end) + line_end = text_end; + else + ++line_end; + text = line_end; + + /* Strip whitespace from the beginning and end of line. */ + while (line_beg < line_end && c_isspace (*line_beg)) + ++line_beg; + while (line_end > line_beg && c_isspace (*(line_end - 1))) + --line_end; + + if (line_beg == line_end) + continue; + + /* The URL is in the [line_beg, line_end) region. */ + + /* We must copy the URL to a zero-terminated string, and we + can't use alloca because we're in a loop. *sigh*. */ + url_text = strdupdelim (line_beg, line_end); + + if (opt.base_href) + { + /* Merge opt.base_href with URL. */ + char *merged = uri_merge (opt.base_href, url_text); + xfree (url_text); + url_text = merged; + } + + new_url = rewrite_shorthand_url (url_text); + if (new_url) + { + xfree (url_text); + url_text = new_url; + } + + url = url_parse (url_text, &up_error_code, NULL, false); + if (!url) + { + char *error = url_error (url_text, up_error_code); + logprintf (LOG_NOTQUIET, _("%s: Invalid URL %s: %s\n"), + file, url_text, error); + xfree (url_text); + xfree (error); + inform_exit_status (URLERROR); + continue; + } + xfree (url_text); + + entry = xnew0 (struct urlpos); + entry->url = url; + + if (!head) + head = entry; + else + tail->next = entry; + tail = entry; + } + wget_read_file_free (fm); + return head; +} + +#if defined DEBUG_MALLOC || defined TESTING +void +cleanup_html_url (void) +{ + /* Destroy the hash tables. The hash table keys and values are not + allocated by this code, so we don't need to free them here. */ + if (interesting_tags) + hash_table_destroy (interesting_tags); + if (interesting_attributes) + hash_table_destroy (interesting_attributes); +} +#endif diff --git a/src/html-url.h b/src/html-url.h new file mode 100644 index 0000000..abc7edd --- /dev/null +++ b/src/html-url.h @@ -0,0 +1,58 @@ +/* Declarations for html-url.c. + Copyright (C) 1995-1997, 2009-2011, 2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef HTML_URL_H +#define HTML_URL_H + +#include <stdbool.h> +#include "utils.h" +#include "convert.h" +#include "iri.h" + +struct map_context { + char *text; /* HTML text. */ + char *base; /* Base URI of the document, possibly + changed through <base href=...>. */ + const char *parent_base; /* Base of the current document. */ + const char *document_file; /* File name of this document. */ + bool nofollow; /* whether NOFOLLOW was specified in a + <meta name=robots> tag. */ + + struct urlpos *head; /* List of URLs that is being built. */ +}; + +struct urlpos *get_urls_file (const char *); +struct urlpos *get_urls_html (const char *, const char *, bool *, struct iri *); +struct urlpos *get_urls_html_fm (const char *, const struct file_memory *, const char *, bool *, struct iri *); +struct urlpos *append_url (const char *, int, int, struct map_context *); +void free_urlpos (struct urlpos *); +void cleanup_html_url (void); + +#endif /* HTML_URL_H */ diff --git a/src/http-ntlm.c b/src/http-ntlm.c new file mode 100644 index 0000000..ceba3b8 --- /dev/null +++ b/src/http-ntlm.c @@ -0,0 +1,618 @@ +/* NTLM code. + Copyright (C) 2005-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + Contributed by Daniel Stenberg. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +/* NTLM details: + + http://davenport.sourceforge.net/ntlm.html + http://www.innovation.ch/java/ntlm.html + +*/ + +#include <stdio.h> +#include <string.h> +#include <stdlib.h> + +#include "utils.h" +#include "http-ntlm.h" + +#ifdef HAVE_NETTLE +# include <nettle/md4.h> +# include <nettle/des.h> +#else +# include <openssl/des.h> +# include <openssl/md4.h> +# include <openssl/opensslv.h> + +# if OPENSSL_VERSION_NUMBER < 0x00907001L +# define DES_key_schedule des_key_schedule +# define DES_cblock des_cblock +# define DES_set_odd_parity des_set_odd_parity +# define DES_set_key des_set_key +# define DES_ecb_encrypt des_ecb_encrypt + +/* This is how things were done in the old days */ +# define DESKEY(x) x +# define DESKEYARG(x) x +# else +/* Modern version */ +# define DESKEYARG(x) *x +# define DESKEY(x) &x +# endif + +#endif + +/* Define this to make the type-3 message include the NT response message */ +#define USE_NTRESPONSES 1 + + +/* Flag bits definitions available at on + http://davenport.sourceforge.net/ntlm.html */ + +#define NTLMFLAG_NEGOTIATE_OEM (1<<1) +#define NTLMFLAG_NEGOTIATE_NTLM_KEY (1<<9) + +/* + (*) = A "security buffer" is a triplet consisting of two shorts and one + long: + + 1. a 'short' containing the length of the buffer in bytes + 2. a 'short' containing the allocated space for the buffer in bytes + 3. a 'long' containing the offset to the start of the buffer from the + beginning of the NTLM message, in bytes. +*/ + +/* return true on success, false otherwise */ +bool +ntlm_input (struct ntlmdata *ntlm, const char *header) +{ + if (0 != strncmp (header, "NTLM", 4)) + return false; + + header += 4; + while (*header && c_isspace(*header)) + header++; + + if (*header) + { + /* We got a type-2 message here: + + Index Description Content + 0 NTLMSSP Signature Null-terminated ASCII "NTLMSSP" + (0x4e544c4d53535000) + 8 NTLM Message Type long (0x02000000) + 12 Target Name security buffer(*) + 20 Flags long + 24 Challenge 8 bytes + (32) Context (optional) 8 bytes (two consecutive longs) + (40) Target Information (optional) security buffer(*) + 32 (48) start of data block + */ + ssize_t size; + char buffer[48]; // decode 48 bytes needs ((48 + 2) / 3) * 4 + 1 bytes + + DEBUGP (("Received a type-2 NTLM message.\n")); + + size = wget_base64_decode (header, buffer, sizeof (buffer)); + if (size < 0) + return false; /* malformed base64 from server */ + + ntlm->state = NTLMSTATE_TYPE2; /* we got a type-2 */ + + if ((size_t) size >= sizeof (buffer)) + /* the nonce of interest is index [24 .. 31], 8 bytes */ + memcpy (ntlm->nonce, &buffer[24], 8); + + /* at index decimal 20, there's a 32bit NTLM flag field */ + } + else + { + if (ntlm->state == NTLMSTATE_LAST) + { + DEBUGP (("NTLM auth restarted.\n")); + /* no return, continue */ + } + else if (ntlm->state == NTLMSTATE_TYPE3) + { + DEBUGP (("NTLM handshake rejected.\n")); + ntlm->state = NTLMSTATE_NONE; + return false; + } + else if (ntlm->state >= NTLMSTATE_TYPE1) + { + DEBUGP (("Unexpected empty NTLM message.\n")); + return false; /* this is an error */ + } + + DEBUGP (("Empty NTLM message, (re)starting transaction.\n")); + ntlm->state = NTLMSTATE_TYPE1; /* we should sent away a type-1 */ + } + + return true; +} + +/* + * Turns a 56 bit key into the 64 bit, odd parity key and sets the key. The + * key schedule ks is also set. + */ +#ifdef HAVE_NETTLE +static void +setup_des_key(unsigned char *key_56, + struct des_ctx *des) +{ + unsigned char key[8]; + + key[0] = key_56[0]; + key[1] = ((key_56[0] << 7) & 0xFF) | (key_56[1] >> 1); + key[2] = ((key_56[1] << 6) & 0xFF) | (key_56[2] >> 2); + key[3] = ((key_56[2] << 5) & 0xFF) | (key_56[3] >> 3); + key[4] = ((key_56[3] << 4) & 0xFF) | (key_56[4] >> 4); + key[5] = ((key_56[4] << 3) & 0xFF) | (key_56[5] >> 5); + key[6] = ((key_56[5] << 2) & 0xFF) | (key_56[6] >> 6); + key[7] = (key_56[6] << 1) & 0xFF; + + nettle_des_set_key(des, key); +} +#else +static void +setup_des_key(unsigned char *key_56, + DES_key_schedule DESKEYARG(ks)) +{ + DES_cblock key; + + key[0] = key_56[0]; + key[1] = ((key_56[0] << 7) & 0xFF) | (key_56[1] >> 1); + key[2] = ((key_56[1] << 6) & 0xFF) | (key_56[2] >> 2); + key[3] = ((key_56[2] << 5) & 0xFF) | (key_56[3] >> 3); + key[4] = ((key_56[3] << 4) & 0xFF) | (key_56[4] >> 4); + key[5] = ((key_56[4] << 3) & 0xFF) | (key_56[5] >> 5); + key[6] = ((key_56[5] << 2) & 0xFF) | (key_56[6] >> 6); + key[7] = (key_56[6] << 1) & 0xFF; + + DES_set_odd_parity(&key); + DES_set_key(&key, ks); +} +#endif + + /* + * takes a 21 byte array and treats it as 3 56-bit DES keys. The + * 8 byte plaintext is encrypted with each key and the resulting 24 + * bytes are stored in the results array. + */ +static void +calc_resp(unsigned char *keys, unsigned char *plaintext, unsigned char *results) +{ +#ifdef HAVE_NETTLE + struct des_ctx des; + + setup_des_key(keys, &des); + nettle_des_encrypt(&des, 8, results, plaintext); + + setup_des_key(keys + 7, &des); + nettle_des_encrypt(&des, 8, results + 8, plaintext); + + setup_des_key(keys + 14, &des); + nettle_des_encrypt(&des, 8, results + 16, plaintext); +#else + DES_key_schedule ks; + + setup_des_key(keys, DESKEY(ks)); + DES_ecb_encrypt((DES_cblock*) plaintext, (DES_cblock*) results, + DESKEY(ks), DES_ENCRYPT); + + setup_des_key(keys+7, DESKEY(ks)); + DES_ecb_encrypt((DES_cblock*) plaintext, (DES_cblock*) (results+8), + DESKEY(ks), DES_ENCRYPT); + + setup_des_key(keys+14, DESKEY(ks)); + DES_ecb_encrypt((DES_cblock*) plaintext, (DES_cblock*) (results+16), + DESKEY(ks), DES_ENCRYPT); +#endif +} + +/* + * Set up lanmanager and nt hashed passwords + */ +static void +mkhash(const char *password, + unsigned char *nonce, /* 8 bytes */ + unsigned char *lmresp /* must fit 0x18 bytes */ +#ifdef USE_NTRESPONSES + , unsigned char *ntresp /* must fit 0x18 bytes */ +#endif + ) +{ + unsigned char lmbuffer[21]; +#ifdef USE_NTRESPONSES + unsigned char ntbuffer[21]; +#endif + unsigned char pw[14]; + static const unsigned char magic[] = { + 0x4B, 0x47, 0x53, 0x21, 0x40, 0x23, 0x24, 0x25 + }; + size_t i, len = strlen(password); + + /* make it fit at least 14 bytes */ + + if (len > sizeof (pw)) + len = sizeof (pw); + + for (i = 0; i < len; i++) + pw[i] = (unsigned char) c_toupper (password[i]); + + for (; i < sizeof (pw); i++) + pw[i] = 0; + + { + /* create LanManager hashed password */ +#ifdef HAVE_NETTLE + struct des_ctx des; + + setup_des_key(pw, &des); + nettle_des_encrypt(&des, 8, lmbuffer, magic); + + setup_des_key(pw + 7, &des); + nettle_des_encrypt(&des, 8, lmbuffer + 8, magic); +#else + DES_key_schedule ks; + + setup_des_key(pw, DESKEY (ks)); + DES_ecb_encrypt((DES_cblock *) magic, (DES_cblock *) lmbuffer, + DESKEY (ks), DES_ENCRYPT); + + setup_des_key(pw+7, DESKEY (ks)); + DES_ecb_encrypt((DES_cblock *) magic, (DES_cblock *) (lmbuffer + 8), + DESKEY (ks), DES_ENCRYPT); +#endif + + memset(lmbuffer + 16, 0, 5); + } + /* create LM responses */ + calc_resp(lmbuffer, nonce, lmresp); + +#ifdef USE_NTRESPONSES + { +#ifdef HAVE_NETTLE + struct md4_ctx MD4; +#else + MD4_CTX MD4; +#endif + + unsigned char pw4[64]; + + len = strlen (password); + + if (len > sizeof (pw4) / 2) + len = sizeof (pw4) / 2; + + for (i = 0; i < len; i++) { + pw4[2 * i] = (unsigned char) password[i]; + pw4[2 * i + 1] = 0; + } + +#ifdef HAVE_NETTLE + nettle_md4_init(&MD4); + nettle_md4_update(&MD4, (unsigned) (2 * len), pw4); + nettle_md4_digest(&MD4, MD4_DIGEST_SIZE, ntbuffer); +#else + /* create NT hashed password */ + MD4_Init(&MD4); + MD4_Update(&MD4, pw4, 2 * len); + MD4_Final(ntbuffer, &MD4); +#endif + + memset(ntbuffer + 16, 0, 5); + } + + calc_resp(ntbuffer, nonce, ntresp); +#endif +} + +#define SHORTPAIR(x) (char) ((x) & 0xff), (char) ((x) >> 8) +#define LONGQUARTET(x) ((x) & 0xff), (((x) >> 8)&0xff), \ + (((x) >>16)&0xff), ((x)>>24) + +/* this is for creating ntlm header output */ +char * +ntlm_output (struct ntlmdata *ntlm, const char *user, const char *passwd, + bool *ready) +{ + const char *domain = ""; /* empty */ + const char *host = ""; /* empty */ + size_t domlen = strlen(domain); + size_t hostlen = strlen(host); + size_t hostoff; /* host name offset */ + size_t domoff; /* domain name offset */ + size_t size; + char ntlmbuf[256]; /* enough, unless the host/domain is very long */ + + /* point to the address of the pointer that holds the string to sent to the + server, which is for a plain host or for a HTTP proxy */ + char *output = NULL; + + *ready = false; + + /* not set means empty */ + if(!user) + user=""; + + if(!passwd) + passwd=""; + + switch(ntlm->state) { + case NTLMSTATE_TYPE1: + case NTLMSTATE_NONE: + case NTLMSTATE_LAST: + hostoff = 32; + domoff = hostoff + hostlen; + + DEBUGP (("Creating a type-1 NTLM message.\n")); + + /* Create and send a type-1 message: + + Index Description Content + 0 NTLMSSP Signature Null-terminated ASCII "NTLMSSP" + (0x4e544c4d53535000) + 8 NTLM Message Type long (0x01000000) + 12 Flags long + 16 Supplied Domain security buffer(*) + 24 Supplied Workstation security buffer(*) + 32 start of data block + + */ + + snprintf (ntlmbuf, sizeof(ntlmbuf), "NTLMSSP%c" + "\x01%c%c%c" /* 32-bit type = 1 */ + "%c%c%c%c" /* 32-bit NTLM flag field */ + "%c%c" /* domain length */ + "%c%c" /* domain allocated space */ + "%c%c" /* domain name offset */ + "%c%c" /* 2 zeroes */ + "%c%c" /* host length */ + "%c%c" /* host allocated space */ + "%c%c" /* host name offset */ + "%c%c" /* 2 zeroes */ + "%s" /* host name */ + "%s", /* domain string */ + 0, /* trailing zero */ + 0,0,0, /* part of type-1 long */ + + LONGQUARTET( + NTLMFLAG_NEGOTIATE_OEM| /* 2 */ + NTLMFLAG_NEGOTIATE_NTLM_KEY /* 200 */ + /* equals 0x0202 */ + ), + SHORTPAIR(domlen), + SHORTPAIR(domlen), + SHORTPAIR(domoff), + 0,0, + SHORTPAIR(hostlen), + SHORTPAIR(hostlen), + SHORTPAIR(hostoff), + 0,0, + host, domain); + + /* initial packet length */ + size = 32 + hostlen + domlen; + + output = xmalloc(5 + BASE64_LENGTH (size) + 1); + memcpy(output, "NTLM ", 5); + wget_base64_encode (ntlmbuf, size, output + 5); + + break; + + case NTLMSTATE_TYPE2: + /* We received the type-2 already, create a type-3 message: + + Index Description Content + 0 NTLMSSP Signature Null-terminated ASCII "NTLMSSP" + (0x4e544c4d53535000) + 8 NTLM Message Type long (0x03000000) + 12 LM/LMv2 Response security buffer(*) + 20 NTLM/NTLMv2 Response security buffer(*) + 28 Domain Name security buffer(*) + 36 User Name security buffer(*) + 44 Workstation Name security buffer(*) + (52) Session Key (optional) security buffer(*) + (60) Flags (optional) long + 52 (64) start of data block + + */ + + { + size_t lmrespoff; + size_t ntrespoff; + size_t useroff; + unsigned char lmresp[0x18]; /* fixed-size */ +#ifdef USE_NTRESPONSES + unsigned char ntresp[0x18]; /* fixed-size */ +#endif + const char *usr; + size_t userlen; + + DEBUGP (("Creating a type-3 NTLM message.\n")); + + usr = strchr(user, '\\'); + if(!usr) + usr = strchr(user, '/'); + + if (usr) { + domain = user; + domlen = (size_t) (usr - domain); + usr++; + } + else + usr = user; + userlen = strlen(usr); + + mkhash(passwd, &ntlm->nonce[0], lmresp +#ifdef USE_NTRESPONSES + , ntresp +#endif + ); + + domoff = 64; /* always */ + useroff = domoff + domlen; + hostoff = useroff + userlen; + lmrespoff = hostoff + hostlen; + ntrespoff = lmrespoff + 0x18; + + /* Create the big type-3 message binary blob */ + + snprintf (ntlmbuf, sizeof (ntlmbuf), + "NTLMSSP%c" + "\x03%c%c%c" /* type-3, 32 bits */ + + "%c%c%c%c" /* LanManager length + allocated space */ + "%c%c" /* LanManager offset */ + "%c%c" /* 2 zeroes */ + + "%c%c" /* NT-response length */ + "%c%c" /* NT-response allocated space */ + "%c%c" /* NT-response offset */ + "%c%c" /* 2 zeroes */ + + "%c%c" /* domain length */ + "%c%c" /* domain allocated space */ + "%c%c" /* domain name offset */ + "%c%c" /* 2 zeroes */ + + "%c%c" /* user length */ + "%c%c" /* user allocated space */ + "%c%c" /* user offset */ + "%c%c" /* 2 zeroes */ + + "%c%c" /* host length */ + "%c%c" /* host allocated space */ + "%c%c" /* host offset */ + "%c%c%c%c%c%c" /* 6 zeroes */ + + "\xff\xff" /* message length */ + "%c%c" /* 2 zeroes */ + + "\x01\x82" /* flags */ + "%c%c" /* 2 zeroes */ + + /* domain string */ + /* user string */ + /* host string */ + /* LanManager response */ + /* NT response */ + , + 0, /* zero termination */ + 0, 0, 0, /* type-3 long, the 24 upper bits */ + + SHORTPAIR (0x18), /* LanManager response length, twice */ + SHORTPAIR (0x18), + SHORTPAIR (lmrespoff), + 0x0, 0x0, + +#ifdef USE_NTRESPONSES + SHORTPAIR (0x18), /* NT-response length, twice */ + SHORTPAIR (0x18), +#else + 0x0, 0x0, + 0x0, 0x0, +#endif + SHORTPAIR (ntrespoff), + 0x0, 0x0, + + SHORTPAIR (domlen), + SHORTPAIR (domlen), + SHORTPAIR (domoff), + 0x0, 0x0, + + SHORTPAIR (userlen), + SHORTPAIR (userlen), + SHORTPAIR (useroff), + 0x0, 0x0, + + SHORTPAIR (hostlen), + SHORTPAIR (hostlen), + SHORTPAIR (hostoff), + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + + 0x0, 0x0, + + 0x0, 0x0); + + /* size is now 64 */ + size=64; + ntlmbuf[62]=ntlmbuf[63]=0; + + /* Make sure that the user and domain strings fit in the target buffer + before we copy them there. */ + if((size + userlen + domlen) >= sizeof(ntlmbuf)) + return NULL; + + memcpy(&ntlmbuf[size], domain, domlen); + size += domlen; + + memcpy(&ntlmbuf[size], usr, userlen); + size += userlen; + + /* we append the binary hashes to the end of the blob */ + if(size < (sizeof(ntlmbuf) - 0x18)) { + memcpy(&ntlmbuf[size], lmresp, 0x18); + size += 0x18; + } + +#ifdef USE_NTRESPONSES + if(size < (sizeof(ntlmbuf) - 0x18)) { + memcpy(&ntlmbuf[size], ntresp, 0x18); + size += 0x18; + } +#endif + + ntlmbuf[56] = (char) (size & 0xff); + ntlmbuf[57] = (char) (size >> 8); + + /* convert the binary blob into base64 */ + output = xmalloc(5 + BASE64_LENGTH (size) + 1); + memcpy(output, "NTLM ", 5); + wget_base64_encode (ntlmbuf, size, output + 5); + + ntlm->state = NTLMSTATE_TYPE3; /* we sent a type-3 */ + *ready = true; + } + break; + + case NTLMSTATE_TYPE3: + /* connection is already authenticated, + * don't send a header in future requests */ + *ready = true; + output = NULL; + break; + } + + return output; +} diff --git a/src/http-ntlm.h b/src/http-ntlm.h new file mode 100644 index 0000000..de0c9ce --- /dev/null +++ b/src/http-ntlm.h @@ -0,0 +1,53 @@ +#ifndef __HTTP_NTLM_H +#define __HTTP_NTLM_H +/* Declarations for http_ntlm.c + Copyright (C) 1995-1997, 2000, 2007-2011, 2015, 2018-2020 Free + Software Foundation, Inc. + Contributed by Daniel Stenberg. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +typedef enum { + NTLMSTATE_NONE, + NTLMSTATE_TYPE1, + NTLMSTATE_TYPE2, + NTLMSTATE_TYPE3, + NTLMSTATE_LAST +} wgetntlm; + +/* Struct used for NTLM challenge-response authentication */ +struct ntlmdata { + wgetntlm state; + unsigned char nonce[8]; +}; + +/* this is for ntlm header input */ +bool ntlm_input (struct ntlmdata *, const char *); + +/* this is for creating ntlm header output */ +char *ntlm_output (struct ntlmdata *, const char *, const char *, bool *); +#endif diff --git a/src/http.c b/src/http.c new file mode 100644 index 0000000..26e10d2 --- /dev/null +++ b/src/http.c @@ -0,0 +1,5495 @@ +/* HTTP support. + Copyright (C) 1996-2012, 2014-2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <assert.h> +#include <errno.h> +#include <time.h> +#include <locale.h> +#include <fcntl.h> + +#include "hash.h" +#include "http.h" +#include "hsts.h" +#include "utils.h" +#include "url.h" +#include "host.h" +#include "retr.h" +#include "connect.h" +#include "netrc.h" +#ifdef HAVE_SSL +# include "ssl.h" +#endif +#ifdef ENABLE_NTLM +# include "http-ntlm.h" +#endif +#include "cookies.h" +#include "md5.h" +#include "convert.h" +#include "spider.h" +#include "warc.h" +#include "c-strcase.h" +#include "version.h" +#include "xstrndup.h" +#ifdef HAVE_METALINK +# include "metalink.h" +#endif +#ifdef ENABLE_XATTR +#include "xattr.h" +#endif + +#ifdef TESTING +#include "../tests/unit-tests.h" +#endif + +#ifdef __VMS +# include "vms.h" +#endif /* def __VMS */ + + +/* Forward decls. */ +struct http_stat; +static char *create_authorization_line (const char *, const char *, + const char *, const char *, + const char *, bool *, uerr_t *); +static char *basic_authentication_encode (const char *, const char *); +static bool known_authentication_scheme_p (const char *, const char *); +static void ensure_extension (struct http_stat *, const char *, int *); +static void load_cookies (void); + +static bool cookies_loaded_p; +static struct cookie_jar *wget_cookie_jar; + +#define TEXTHTML_S "text/html" +#define TEXTXHTML_S "application/xhtml+xml" +#define TEXTCSS_S "text/css" + +/* Some status code validation macros: */ +#define H_10X(x) (((x) >= 100) && ((x) < 200)) +#define H_20X(x) (((x) >= 200) && ((x) < 300)) +#define H_PARTIAL(x) ((x) == HTTP_STATUS_PARTIAL_CONTENTS) +#define H_REDIRECTED(x) ((x) == HTTP_STATUS_MOVED_PERMANENTLY \ + || (x) == HTTP_STATUS_MOVED_TEMPORARILY \ + || (x) == HTTP_STATUS_SEE_OTHER \ + || (x) == HTTP_STATUS_TEMPORARY_REDIRECT \ + || (x) == HTTP_STATUS_PERMANENT_REDIRECT) + +/* HTTP/1.0 status codes from RFC1945, provided for reference. */ +/* Successful 2xx. */ +#define HTTP_STATUS_OK 200 +#define HTTP_STATUS_CREATED 201 +#define HTTP_STATUS_ACCEPTED 202 +#define HTTP_STATUS_NO_CONTENT 204 +#define HTTP_STATUS_PARTIAL_CONTENTS 206 + +/* Redirection 3xx. */ +#define HTTP_STATUS_MULTIPLE_CHOICES 300 +#define HTTP_STATUS_MOVED_PERMANENTLY 301 +#define HTTP_STATUS_MOVED_TEMPORARILY 302 +#define HTTP_STATUS_SEE_OTHER 303 /* from HTTP/1.1 */ +#define HTTP_STATUS_NOT_MODIFIED 304 +#define HTTP_STATUS_TEMPORARY_REDIRECT 307 /* from HTTP/1.1 */ +#define HTTP_STATUS_PERMANENT_REDIRECT 308 /* from HTTP/1.1 */ + +/* Client error 4xx. */ +#define HTTP_STATUS_BAD_REQUEST 400 +#define HTTP_STATUS_UNAUTHORIZED 401 +#define HTTP_STATUS_FORBIDDEN 403 +#define HTTP_STATUS_NOT_FOUND 404 +#define HTTP_STATUS_RANGE_NOT_SATISFIABLE 416 + +/* Server errors 5xx. */ +#define HTTP_STATUS_INTERNAL 500 +#define HTTP_STATUS_NOT_IMPLEMENTED 501 +#define HTTP_STATUS_BAD_GATEWAY 502 +#define HTTP_STATUS_UNAVAILABLE 503 +#define HTTP_STATUS_GATEWAY_TIMEOUT 504 + +enum rp { + rel_none, rel_name, rel_value, rel_both +}; + +struct request { + const char *method; + char *arg; + + struct request_header { + char *name, *value; + enum rp release_policy; + } *headers; + int hcount, hcapacity; +}; + + +/* Create a new, empty request. Set the request's method and its + arguments. METHOD should be a literal string (or it should outlive + the request) because it will not be freed. ARG will be freed by + request_free. */ + +static struct request * +request_new (const char *method, char *arg) +{ + struct request *req = xnew0 (struct request); + req->hcapacity = 8; + req->headers = xnew_array (struct request_header, req->hcapacity); + req->method = method; + req->arg = arg; + return req; +} + +/* Return the method string passed with the last call to + request_set_method. */ + +static const char * +request_method (const struct request *req) +{ + return req->method; +} + +/* Free one header according to the release policy specified with + request_set_header. */ + +static void +release_header (struct request_header *hdr) +{ + switch (hdr->release_policy) + { + case rel_none: + break; + case rel_name: + xfree (hdr->name); + break; + case rel_value: + xfree (hdr->value); + break; + case rel_both: + xfree (hdr->name); + xfree (hdr->value); + break; + } +} + +/* Set the request named NAME to VALUE. Specifically, this means that + a "NAME: VALUE\r\n" header line will be used in the request. If a + header with the same name previously existed in the request, its + value will be replaced by this one. A NULL value means do nothing. + + RELEASE_POLICY determines whether NAME and VALUE should be released + (freed) with request_free. Allowed values are: + + - rel_none - don't free NAME or VALUE + - rel_name - free NAME when done + - rel_value - free VALUE when done + - rel_both - free both NAME and VALUE when done + + Setting release policy is useful when arguments come from different + sources. For example: + + // Don't free literal strings! + request_set_header (req, "Pragma", "no-cache", rel_none); + + // Don't free a global variable, we'll need it later. + request_set_header (req, "Referer", opt.referer, rel_none); + + // Value freshly allocated, free it when done. + request_set_header (req, "Range", + aprintf ("bytes=%s-", number_to_static_string (hs->restval)), + rel_value); + */ + +static void +request_set_header (struct request *req, const char *name, const char *value, + enum rp release_policy) +{ + struct request_header *hdr; + int i; + + if (!value) + { + /* A NULL value is a no-op; if freeing the name is requested, + free it now to avoid leaks. */ + if (release_policy == rel_name || release_policy == rel_both) + xfree (name); + return; + } + + for (i = 0; i < req->hcount; i++) + { + hdr = &req->headers[i]; + if (0 == c_strcasecmp (name, hdr->name)) + { + /* Replace existing header. */ + release_header (hdr); + hdr->name = (void *)name; + hdr->value = (void *)value; + hdr->release_policy = release_policy; + return; + } + } + + /* Install new header. */ + + if (req->hcount >= req->hcapacity) + { + req->hcapacity <<= 1; + req->headers = xrealloc (req->headers, req->hcapacity * sizeof (*hdr)); + } + hdr = &req->headers[req->hcount++]; + hdr->name = (void *)name; + hdr->value = (void *)value; + hdr->release_policy = release_policy; +} + +/* Like request_set_header, but sets the whole header line, as + provided by the user using the `--header' option. For example, + request_set_user_header (req, "Foo: bar") works just like + request_set_header (req, "Foo", "bar"). */ + +static void +request_set_user_header (struct request *req, const char *header) +{ + const char *name, *p; + + if (!(p = strchr (header, ':'))) + return; + + name = xstrndup(header, p - header); + + ++p; + while (c_isspace (*p)) + ++p; + + request_set_header (req, name, p, rel_name); +} + +/* Remove the header with specified name from REQ. Returns true if + the header was actually removed, false otherwise. */ + +static bool +request_remove_header (struct request *req, const char *name) +{ + int i; + for (i = 0; i < req->hcount; i++) + { + struct request_header *hdr = &req->headers[i]; + if (0 == c_strcasecmp (name, hdr->name)) + { + release_header (hdr); + /* Move the remaining headers by one. */ + if (i < req->hcount - 1) + memmove (hdr, hdr + 1, (req->hcount - i - 1) * sizeof (*hdr)); + --req->hcount; + return true; + } + } + return false; +} + +#define APPEND(p, str) do { \ + int A_len = strlen (str); \ + memcpy (p, str, A_len); \ + p += A_len; \ +} while (0) + +/* Construct the request and write it to FD using fd_write. + If warc_tmp is set to a file pointer, the request string will + also be written to that file. */ + +static int +request_send (const struct request *req, int fd, FILE *warc_tmp) +{ + char *request_string, *p; + int i, size, write_error; + + /* Count the request size. */ + size = 0; + + /* METHOD " " ARG " " "HTTP/1.0" "\r\n" */ + size += strlen (req->method) + 1 + strlen (req->arg) + 1 + 8 + 2; + + for (i = 0; i < req->hcount; i++) + { + struct request_header *hdr = &req->headers[i]; + /* NAME ": " VALUE "\r\n" */ + size += strlen (hdr->name) + 2 + strlen (hdr->value) + 2; + } + + /* "\r\n\0" */ + size += 3; + + p = request_string = xmalloc (size); + + /* Generate the request. */ + + APPEND (p, req->method); *p++ = ' '; + APPEND (p, req->arg); *p++ = ' '; + memcpy (p, "HTTP/1.1\r\n", 10); p += 10; + + for (i = 0; i < req->hcount; i++) + { + struct request_header *hdr = &req->headers[i]; + APPEND (p, hdr->name); + *p++ = ':', *p++ = ' '; + APPEND (p, hdr->value); + *p++ = '\r', *p++ = '\n'; + } + + *p++ = '\r', *p++ = '\n', *p++ = '\0'; + assert (p - request_string == size); + +#undef APPEND + + DEBUGP (("\n---request begin---\n%s---request end---\n", request_string)); + + /* Send the request to the server. */ + + write_error = fd_write (fd, request_string, size - 1, -1); + if (write_error < 0) + logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"), + fd_errstr (fd)); + else if (warc_tmp != NULL) + { + /* Write a copy of the data to the WARC record. */ + int warc_tmp_written = fwrite (request_string, 1, size - 1, warc_tmp); + if (warc_tmp_written != size - 1) + write_error = -2; + } + xfree (request_string); + return write_error; +} + +/* Release the resources used by REQ. + It is safe to call it with a valid pointer to a NULL pointer. + It is not safe to call it with an invalid or NULL pointer. */ + +static void +request_free (struct request **req_ref) +{ + int i; + struct request *req = *req_ref; + + if (!req) + return; + + xfree (req->arg); + for (i = 0; i < req->hcount; i++) + release_header (&req->headers[i]); + xfree (req->headers); + xfree (req); + *req_ref = NULL; +} + +static struct hash_table *basic_authed_hosts; + +/* Find out if this host has issued a Basic challenge yet; if so, give + * it the username, password. A temporary measure until we can get + * proper authentication in place. */ + +static bool +maybe_send_basic_creds (const char *hostname, const char *user, + const char *passwd, struct request *req) +{ + bool do_challenge = false; + + if (opt.auth_without_challenge) + { + DEBUGP (("Auth-without-challenge set, sending Basic credentials.\n")); + do_challenge = true; + } + else if (basic_authed_hosts + && hash_table_contains (basic_authed_hosts, hostname)) + { + DEBUGP (("Found %s in basic_authed_hosts.\n", quote (hostname))); + do_challenge = true; + } + else + { + DEBUGP (("Host %s has not issued a general basic challenge.\n", + quote (hostname))); + } + if (do_challenge) + { + request_set_header (req, "Authorization", + basic_authentication_encode (user, passwd), + rel_value); + } + return do_challenge; +} + +static void +register_basic_auth_host (const char *hostname) +{ + if (!basic_authed_hosts) + { + basic_authed_hosts = make_nocase_string_hash_table (1); + } + if (!hash_table_contains (basic_authed_hosts, hostname)) + { + hash_table_put (basic_authed_hosts, xstrdup (hostname), NULL); + DEBUGP (("Inserted %s into basic_authed_hosts\n", quote (hostname))); + } +} + +/* Send the contents of FILE_NAME to SOCK. Make sure that exactly + PROMISED_SIZE bytes are sent over the wire -- if the file is + longer, read only that much; if the file is shorter, report an error. + If warc_tmp is set to a file pointer, the post data will + also be written to that file. */ + +static int +body_file_send (int sock, const char *file_name, wgint promised_size, FILE *warc_tmp) +{ + static char chunk[8192]; + wgint written = 0; + int write_error; + FILE *fp; + + DEBUGP (("[writing BODY file %s ... ", file_name)); + + fp = fopen (file_name, "rb"); + if (!fp) + return -1; + while (!feof (fp) && written < promised_size) + { + int towrite; + int length = fread (chunk, 1, sizeof (chunk), fp); + if (length == 0) + break; + towrite = MIN (promised_size - written, length); + write_error = fd_write (sock, chunk, towrite, -1); + if (write_error < 0) + { + fclose (fp); + return -1; + } + if (warc_tmp != NULL) + { + /* Write a copy of the data to the WARC record. */ + int warc_tmp_written = fwrite (chunk, 1, towrite, warc_tmp); + if (warc_tmp_written != towrite) + { + fclose (fp); + return -2; + } + } + written += towrite; + } + fclose (fp); + + /* If we've written less than was promised, report a (probably + nonsensical) error rather than break the promise. */ + if (written < promised_size) + { + errno = EINVAL; + return -1; + } + + assert (written == promised_size); + DEBUGP (("done]\n")); + return 0; +} + +/* Determine whether [START, PEEKED + PEEKLEN) contains an empty line. + If so, return the pointer to the position after the line, otherwise + return NULL. This is used as callback to fd_read_hunk. The data + between START and PEEKED has been read and cannot be "unread"; the + data after PEEKED has only been peeked. */ + +static const char * +response_head_terminator (const char *start, const char *peeked, int peeklen) +{ + const char *p, *end; + + /* If at first peek, verify whether HUNK starts with "HTTP". If + not, this is a HTTP/0.9 request and we must bail out without + reading anything. */ + if (start == peeked && 0 != memcmp (start, "HTTP", MIN (peeklen, 4))) + return start; + + /* Look for "\n[\r]\n", and return the following position if found. + Start two chars before the current to cover the possibility that + part of the terminator (e.g. "\n\r") arrived in the previous + batch. */ + p = peeked - start < 2 ? start : peeked - 2; + end = peeked + peeklen; + + /* Check for \n\r\n or \n\n anywhere in [p, end-2). */ + for (; p < end - 2; p++) + if (*p == '\n') + { + if (p[1] == '\r' && p[2] == '\n') + return p + 3; + else if (p[1] == '\n') + return p + 2; + } + /* p==end-2: check for \n\n directly preceding END. */ + if (peeklen >= 2 && p[0] == '\n' && p[1] == '\n') + return p + 2; + + return NULL; +} + +/* The maximum size of a single HTTP response we care to read. Rather + than being a limit of the reader implementation, this limit + prevents Wget from slurping all available memory upon encountering + malicious or buggy server output, thus protecting the user. Define + it to 0 to remove the limit. */ + +#define HTTP_RESPONSE_MAX_SIZE 65536 + +/* Read the HTTP request head from FD and return it. The error + conditions are the same as with fd_read_hunk. + + To support HTTP/0.9 responses, this function tries to make sure + that the data begins with "HTTP". If this is not the case, no data + is read and an empty request is returned, so that the remaining + data can be treated as body. */ + +static char * +read_http_response_head (int fd) +{ + return fd_read_hunk (fd, response_head_terminator, 512, + HTTP_RESPONSE_MAX_SIZE); +} + +struct response { + /* The response data. */ + const char *data; + + /* The array of pointers that indicate where each header starts. + For example, given this HTTP response: + + HTTP/1.0 200 Ok + Description: some + text + Etag: x + + The headers are located like this: + + "HTTP/1.0 200 Ok\r\nDescription: some\r\n text\r\nEtag: x\r\n\r\n" + ^ ^ ^ ^ + headers[0] headers[1] headers[2] headers[3] + + I.e. headers[0] points to the beginning of the request, + headers[1] points to the end of the first header and the + beginning of the second one, etc. */ + + const char **headers; +}; + +/* Create a new response object from the text of the HTTP response, + available in HEAD. That text is automatically split into + constituent header lines for fast retrieval using + resp_header_*. */ + +static struct response * +resp_new (char *head) +{ + char *hdr; + int count, size; + + struct response *resp = xnew0 (struct response); + resp->data = head; + + if (*head == '\0') + { + /* Empty head means that we're dealing with a headerless + (HTTP/0.9) response. In that case, don't set HEADERS at + all. */ + return resp; + } + + /* Split HEAD into header lines, so that resp_header_* functions + don't need to do this over and over again. */ + + size = count = 0; + hdr = head; + while (1) + { + DO_REALLOC (resp->headers, size, count + 1, const char *); + resp->headers[count++] = hdr; + + /* Break upon encountering an empty line. */ + if (!hdr[0] || (hdr[0] == '\r' && hdr[1] == '\n') || hdr[0] == '\n') + break; + + /* Find the end of HDR, including continuations. */ + for (;;) + { + char *end = strchr (hdr, '\n'); + + if (!end) + { + hdr += strlen (hdr); + break; + } + + hdr = end + 1; + + if (*hdr != ' ' && *hdr != '\t') + break; + + // continuation, transform \r and \n into spaces + *end = ' '; + if (end > head && end[-1] == '\r') + end[-1] = ' '; + } + } + DO_REALLOC (resp->headers, size, count + 1, const char *); + resp->headers[count] = NULL; + + return resp; +} + +/* Locate the header named NAME in the request data, starting with + position START. This allows the code to loop through the request + data, filtering for all requests of a given name. Returns the + found position, or -1 for failure. The code that uses this + function typically looks like this: + + for (pos = 0; (pos = resp_header_locate (...)) != -1; pos++) + ... do something with header ... + + If you only care about one header, use resp_header_get instead of + this function. */ + +static int +resp_header_locate (const struct response *resp, const char *name, int start, + const char **begptr, const char **endptr) +{ + int i; + const char **headers = resp->headers; + int name_len; + + if (!headers || !headers[1]) + return -1; + + name_len = strlen (name); + if (start > 0) + i = start; + else + i = 1; + + for (; headers[i + 1]; i++) + { + const char *b = headers[i]; + const char *e = headers[i + 1]; + if (e - b > name_len + && b[name_len] == ':' + && 0 == c_strncasecmp (b, name, name_len)) + { + b += name_len + 1; + while (b < e && c_isspace (*b)) + ++b; + while (b < e && c_isspace (e[-1])) + --e; + *begptr = b; + *endptr = e; + return i; + } + } + return -1; +} + +/* Find and retrieve the header named NAME in the request data. If + found, set *BEGPTR to its starting, and *ENDPTR to its ending + position, and return true. Otherwise return false. + + This function is used as a building block for resp_header_copy + and resp_header_strdup. */ + +static bool +resp_header_get (const struct response *resp, const char *name, + const char **begptr, const char **endptr) +{ + int pos = resp_header_locate (resp, name, 0, begptr, endptr); + return pos != -1; +} + +/* Copy the response header named NAME to buffer BUF, no longer than + BUFSIZE (BUFSIZE includes the terminating 0). If the header + exists, true is returned, false otherwise. If there should be no + limit on the size of the header, use resp_header_strdup instead. + + If BUFSIZE is 0, no data is copied, but the boolean indication of + whether the header is present is still returned. */ + +static bool +resp_header_copy (const struct response *resp, const char *name, + char *buf, int bufsize) +{ + const char *b, *e; + if (!resp_header_get (resp, name, &b, &e)) + return false; + if (bufsize) + { + int len = MIN (e - b, bufsize - 1); + memcpy (buf, b, len); + buf[len] = '\0'; + } + return true; +} + +/* Return the value of header named NAME in RESP, allocated with + malloc. If such a header does not exist in RESP, return NULL. */ + +static char * +resp_header_strdup (const struct response *resp, const char *name) +{ + const char *b, *e; + if (!resp_header_get (resp, name, &b, &e)) + return NULL; + return strdupdelim (b, e); +} + +/* Parse the HTTP status line, which is of format: + + HTTP-Version SP Status-Code SP Reason-Phrase + + The function returns the status-code, or -1 if the status line + appears malformed. The pointer to "reason-phrase" message is + returned in *MESSAGE. */ + +static int +resp_status (const struct response *resp, char **message) +{ + int status; + const char *p, *end; + + if (!resp->headers) + { + /* For a HTTP/0.9 response, assume status 200. */ + if (message) + *message = xstrdup (_("No headers, assuming HTTP/0.9")); + return 200; + } + + p = resp->headers[0]; + end = resp->headers[1]; + + if (!end) + return -1; + + /* "HTTP" */ + if (end - p < 4 || 0 != strncmp (p, "HTTP", 4)) + return -1; + p += 4; + + /* Match the HTTP version. This is optional because Gnutella + servers have been reported to not specify HTTP version. */ + if (p < end && *p == '/') + { + ++p; + while (p < end && c_isdigit (*p)) + ++p; + if (p < end && *p == '.') + ++p; + while (p < end && c_isdigit (*p)) + ++p; + } + + while (p < end && c_isspace (*p)) + ++p; + if (end - p < 3 || !c_isdigit (p[0]) || !c_isdigit (p[1]) || !c_isdigit (p[2])) + return -1; + + status = 100 * (p[0] - '0') + 10 * (p[1] - '0') + (p[2] - '0'); + p += 3; + + if (message) + { + while (p < end && c_isspace (*p)) + ++p; + while (p < end && c_isspace (end[-1])) + --end; + *message = strdupdelim (p, end); + } + + return status; +} + +/* Release the resources used by RESP. + It is safe to call it with a valid pointer to a NULL pointer. + It is not safe to call it with a invalid or NULL pointer. */ + +static void +resp_free (struct response **resp_ref) +{ + struct response *resp = *resp_ref; + + if (!resp) + return; + + xfree (resp->headers); + xfree (resp); + + *resp_ref = NULL; +} + +/* Print a single line of response, the characters [b, e). We tried + getting away with + logprintf (LOG_VERBOSE, "%s%.*s\n", prefix, (int) (e - b), b); + but that failed to escape the non-printable characters and, in fact, + caused crashes in UTF-8 locales. */ + +static void +print_response_line (const char *prefix, const char *b, const char *e) +{ + char buf[1024], *copy; + size_t len = e - b; + + if (len < sizeof (buf)) + copy = buf; + else + copy = xmalloc(len + 1); + + memcpy(copy, b, len); + copy[len] = 0; + + logprintf (LOG_ALWAYS, "%s%s\n", prefix, + quotearg_style (escape_quoting_style, copy)); + + if (copy != buf) + xfree (copy); +} + +/* Print the server response, line by line, omitting the trailing CRLF + from individual header lines, and prefixed with PREFIX. */ + +static void +print_server_response (const struct response *resp, const char *prefix) +{ + int i; + if (!resp->headers) + return; + for (i = 0; resp->headers[i + 1]; i++) + { + const char *b = resp->headers[i]; + const char *e = resp->headers[i + 1]; + /* Skip CRLF */ + if (b < e && e[-1] == '\n') + --e; + if (b < e && e[-1] == '\r') + --e; + print_response_line (prefix, b, e); + } +} + +/* Parse the `Content-Range' header and extract the information it + contains. Returns true if successful, false otherwise. */ +static bool +parse_content_range (const char *hdr, wgint *first_byte_ptr, + wgint *last_byte_ptr, wgint *entity_length_ptr) +{ + wgint num; + + /* Ancient versions of Netscape proxy server, presumably predating + rfc2068, sent out `Content-Range' without the "bytes" + specifier. */ + if (0 == strncasecmp (hdr, "bytes", 5)) + { + hdr += 5; + /* "JavaWebServer/1.1.1" sends "bytes: x-y/z", contrary to the + HTTP spec. */ + if (*hdr == ':') + ++hdr; + while (c_isspace (*hdr)) + ++hdr; + if (!*hdr) + return false; + } + if (!c_isdigit (*hdr)) + return false; + for (num = 0; c_isdigit (*hdr); hdr++) + num = 10 * num + (*hdr - '0'); + if (*hdr != '-' || !c_isdigit (*(hdr + 1))) + return false; + *first_byte_ptr = num; + ++hdr; + for (num = 0; c_isdigit (*hdr); hdr++) + num = 10 * num + (*hdr - '0'); + if (*hdr != '/') + return false; + *last_byte_ptr = num; + if (!(c_isdigit (*(hdr + 1)) || *(hdr + 1) == '*')) + return false; + if (*last_byte_ptr < *first_byte_ptr) + return false; + ++hdr; + if (*hdr == '*') + num = -1; + else + for (num = 0; c_isdigit (*hdr); hdr++) + num = 10 * num + (*hdr - '0'); + *entity_length_ptr = num; + if ((*entity_length_ptr <= *last_byte_ptr) && *entity_length_ptr != -1) + return false; + return true; +} + +/* Read the body of the request, but don't store it anywhere and don't + display a progress gauge. This is useful for reading the bodies of + administrative responses to which we will soon issue another + request. The response is not useful to the user, but reading it + allows us to continue using the same connection to the server. + + If reading fails, false is returned, true otherwise. In debug + mode, the body is displayed for debugging purposes. */ + +static bool +skip_short_body (int fd, wgint contlen, bool chunked) +{ + enum { + SKIP_SIZE = 512, /* size of the download buffer */ + SKIP_THRESHOLD = 4096 /* the largest size we read */ + }; + wgint remaining_chunk_size = 0; + char dlbuf[SKIP_SIZE + 1]; + dlbuf[SKIP_SIZE] = '\0'; /* so DEBUGP can safely print it */ + + /* If the body is too large, it makes more sense to simply close the + connection than to try to read the body. */ + if (contlen > SKIP_THRESHOLD) + return false; + + while (contlen > 0 || chunked) + { + int ret; + if (chunked) + { + if (remaining_chunk_size == 0) + { + char *line = fd_read_line (fd); + char *endl; + if (line == NULL) + break; + + remaining_chunk_size = strtol (line, &endl, 16); + xfree (line); + + if (remaining_chunk_size < 0) + return false; + + if (remaining_chunk_size == 0) + { + line = fd_read_line (fd); + xfree (line); + break; + } + } + + contlen = MIN (remaining_chunk_size, SKIP_SIZE); + } + + DEBUGP (("Skipping %s bytes of body: [", number_to_static_string (contlen))); + + ret = fd_read (fd, dlbuf, MIN (contlen, SKIP_SIZE), -1); + if (ret <= 0) + { + /* Don't normally report the error since this is an + optimization that should be invisible to the user. */ + DEBUGP (("] aborting (%s).\n", + ret < 0 ? fd_errstr (fd) : "EOF received")); + return false; + } + contlen -= ret; + + if (chunked) + { + remaining_chunk_size -= ret; + if (remaining_chunk_size == 0) + { + char *line = fd_read_line (fd); + if (line == NULL) + return false; + else + xfree (line); + } + } + + /* Safe even if %.*s bogusly expects terminating \0 because + we've zero-terminated dlbuf above. */ + DEBUGP (("%.*s", ret, dlbuf)); + } + + DEBUGP (("] done.\n")); + return true; +} + +#define NOT_RFC2231 0 +#define RFC2231_NOENCODING 1 +#define RFC2231_ENCODING 2 + +/* extract_param extracts the parameter name into NAME. + However, if the parameter name is in RFC2231 format then + this function adjusts NAME by stripping of the trailing + characters that are not part of the name but are present to + indicate the presence of encoding information in the value + or a fragment of a long parameter value +*/ +static int +modify_param_name (param_token *name) +{ + const char *delim1 = memchr (name->b, '*', name->e - name->b); + const char *delim2 = memrchr (name->b, '*', name->e - name->b); + + int result; + + if (delim1 == NULL) + { + result = NOT_RFC2231; + } + else if (delim1 == delim2) + { + if ((name->e - 1) == delim1) + { + result = RFC2231_ENCODING; + } + else + { + result = RFC2231_NOENCODING; + } + name->e = delim1; + } + else + { + name->e = delim1; + result = RFC2231_ENCODING; + } + return result; +} + +/* extract_param extract the parameter value into VALUE. + Like modify_param_name this function modifies VALUE by + stripping off the encoding information from the actual value +*/ +static void +modify_param_value (param_token *value, int encoding_type ) +{ + if (encoding_type == RFC2231_ENCODING) + { + const char *delim = memrchr (value->b, '\'', value->e - value->b); + if (delim != NULL) + { + value->b = (delim+1); + } + } +} + +/* Extract a parameter from the string (typically an HTTP header) at + **SOURCE and advance SOURCE to the next parameter. Return false + when there are no more parameters to extract. The name of the + parameter is returned in NAME, and the value in VALUE. If the + parameter has no value, the token's value is zeroed out. + + For example, if *SOURCE points to the string "attachment; + filename=\"foo bar\"", the first call to this function will return + the token named "attachment" and no value, and the second call will + return the token named "filename" and value "foo bar". The third + call will return false, indicating no more valid tokens. + + is_url_encoded is an out parameter. If not NULL, a boolean value will be + stored into it, letting the caller know whether or not the extracted value is + URL-encoded. The caller can then decode it with url_unescape(), which however + performs decoding in-place. URL-encoding is used by RFC 2231 to support + non-US-ASCII characters in HTTP header values. */ + +bool +extract_param (const char **source, param_token *name, param_token *value, + char separator, bool *is_url_encoded) +{ + const char *p = *source; + int param_type; + if (is_url_encoded) + *is_url_encoded = false; /* initializing the out parameter */ + + while (c_isspace (*p)) ++p; + if (!*p) + { + *source = p; + return false; /* no error; nothing more to extract */ + } + + /* Extract name. */ + name->b = p; + while (*p && !c_isspace (*p) && *p != '=' && *p != separator) ++p; + name->e = p; + if (name->b == name->e) + return false; /* empty name: error */ + while (c_isspace (*p)) ++p; + if (*p == separator || !*p) /* no value */ + { + xzero (*value); + if (*p == separator) ++p; + *source = p; + return true; + } + if (*p != '=') + return false; /* error */ + + /* *p is '=', extract value */ + ++p; + while (c_isspace (*p)) ++p; + if (*p == '"') /* quoted */ + { + value->b = ++p; + while (*p && *p != '"') ++p; + if (!*p) + return false; + value->e = p++; + /* Currently at closing quote; find the end of param. */ + while (c_isspace (*p)) ++p; + while (*p && *p != separator) ++p; + if (*p == separator) + ++p; + else if (*p) + /* garbage after closed quote, e.g. foo="bar"baz */ + return false; + } + else /* unquoted */ + { + value->b = p; + while (*p && *p != separator) ++p; + value->e = p; + while (value->e != value->b && c_isspace (value->e[-1])) + --value->e; + if (*p == separator) ++p; + } + *source = p; + + param_type = modify_param_name (name); + if (param_type != NOT_RFC2231) + { + if (param_type == RFC2231_ENCODING && is_url_encoded) + *is_url_encoded = true; + modify_param_value (value, param_type); + } + return true; +} + +#undef NOT_RFC2231 +#undef RFC2231_NOENCODING +#undef RFC2231_ENCODING + +/* Appends the string represented by VALUE to FILENAME */ + +static void +append_value_to_filename (char **filename, param_token const * const value, + bool is_url_encoded) +{ + int original_length = strlen (*filename); + int new_length = strlen (*filename) + (value->e - value->b); + *filename = xrealloc (*filename, new_length+1); + memcpy (*filename + original_length, value->b, (value->e - value->b)); + (*filename)[new_length] = '\0'; + if (is_url_encoded) + url_unescape (*filename + original_length); +} + +/* Parse the contents of the `Content-Disposition' header, extracting + the information useful to Wget. Content-Disposition is a header + borrowed from MIME; when used in HTTP, it typically serves for + specifying the desired file name of the resource. For example: + + Content-Disposition: attachment; filename="flora.jpg" + + Wget will skip the tokens it doesn't care about, such as + "attachment" in the previous example; it will also skip other + unrecognized params. If the header is syntactically correct and + contains a file name, a copy of the file name is stored in + *filename and true is returned. Otherwise, the function returns + false. + + The file name is stripped of directory components and must not be + empty. + + Historically, this function returned filename prefixed with opt.dir_prefix, + now that logic is handled by the caller, new code should pay attention, + changed by crq, Sep 2010. + +*/ +static bool +parse_content_disposition (const char *hdr, char **filename) +{ + param_token name, value; + bool is_url_encoded = false; + + char *encodedFilename = NULL; + char *unencodedFilename = NULL; + for ( ; extract_param (&hdr, &name, &value, ';', &is_url_encoded); + is_url_encoded = false) + { + int isFilename = BOUNDED_EQUAL_NO_CASE (name.b, name.e, "filename"); + if ( isFilename && value.b != NULL) + { + /* Make the file name begin at the last slash or backslash. */ + bool isEncodedFilename; + char **outFilename; + const char *last_slash = memrchr (value.b, '/', value.e - value.b); + const char *last_bs = memrchr (value.b, '\\', value.e - value.b); + if (last_slash && last_bs) + value.b = 1 + MAX (last_slash, last_bs); + else if (last_slash || last_bs) + value.b = 1 + (last_slash ? last_slash : last_bs); + if (value.b == value.e) + continue; + + /* Check if the name is "filename*" as specified in RFC 6266. + * Since "filename" could be broken up as "filename*N" (RFC 2231), + * a check is needed to make sure this is not the case */ + isEncodedFilename = *name.e == '*' && !c_isdigit (*(name.e + 1)); + outFilename = isEncodedFilename ? &encodedFilename + : &unencodedFilename; + if (*outFilename) + append_value_to_filename (outFilename, &value, is_url_encoded); + else + { + *outFilename = strdupdelim (value.b, value.e); + if (is_url_encoded) + url_unescape (*outFilename); + } + } + } + if (encodedFilename) + { + xfree (unencodedFilename); + *filename = encodedFilename; + } + else + { + xfree (encodedFilename); + *filename = unencodedFilename; + } + if (*filename) + return true; + else + return false; +} + +#ifdef HAVE_HSTS +static bool +parse_strict_transport_security (const char *header, time_t *max_age, bool *include_subdomains) +{ + param_token name, value; + const char *c_max_age = NULL; + bool is = false; /* includeSubDomains */ + bool is_url_encoded = false; + bool success = false; + + if (header) + { + /* Process the STS header. Keys should be matched case-insensitively. */ + for (; extract_param (&header, &name, &value, ';', &is_url_encoded); is_url_encoded = false) + { + if (BOUNDED_EQUAL_NO_CASE (name.b, name.e, "max-age")) + { + xfree (c_max_age); + c_max_age = strdupdelim (value.b, value.e); + } + else if (BOUNDED_EQUAL_NO_CASE (name.b, name.e, "includeSubDomains")) + is = true; + } + + /* pass the parsed values over */ + if (c_max_age) + { + /* If the string value goes out of a long's bounds, strtol() will return LONG_MIN or LONG_MAX. + * In theory, the HSTS engine should be able to handle it. + * Also, time_t is normally defined as a long, so this should not break. + */ + if (max_age) + *max_age = (time_t) strtol (c_max_age, NULL, 10); + if (include_subdomains) + *include_subdomains = is; + + DEBUGP (("Parsed Strict-Transport-Security max-age = %s, includeSubDomains = %s\n", + c_max_age, (is ? "true" : "false"))); + + xfree (c_max_age); + success = true; + } + else + { + /* something weird happened */ + logprintf (LOG_VERBOSE, "Could not parse String-Transport-Security header\n"); + success = false; + } + } + + return success; +} +#endif + +/* Persistent connections. Currently, we cache the most recently used + connection as persistent, provided that the HTTP server agrees to + make it such. The persistence data is stored in the variables + below. Ideally, it should be possible to cache an arbitrary fixed + number of these connections. */ + +/* Whether a persistent connection is active. */ +static bool pconn_active; + +static struct { + /* The socket of the connection. */ + int socket; + + /* Host and port of the currently active persistent connection. */ + char *host; + int port; + + /* Whether a ssl handshake has occurred on this connection. */ + bool ssl; + + /* Whether the connection was authorized. This is only done by + NTLM, which authorizes *connections* rather than individual + requests. (That practice is peculiar for HTTP, but it is a + useful optimization.) */ + bool authorized; + +#ifdef ENABLE_NTLM + /* NTLM data of the current connection. */ + struct ntlmdata ntlm; +#endif +} pconn; + +/* Mark the persistent connection as invalid and free the resources it + uses. This is used by the CLOSE_* macros after they forcefully + close a registered persistent connection. */ + +static void +invalidate_persistent (void) +{ + DEBUGP (("Disabling further reuse of socket %d.\n", pconn.socket)); + pconn_active = false; + fd_close (pconn.socket); + xfree (pconn.host); + xzero (pconn); +} + +/* Register FD, which should be a TCP/IP connection to HOST:PORT, as + persistent. This will enable someone to use the same connection + later. In the context of HTTP, this must be called only AFTER the + response has been received and the server has promised that the + connection will remain alive. + + If a previous connection was persistent, it is closed. */ + +static void +register_persistent (const char *host, int port, int fd, bool ssl) +{ + if (pconn_active) + { + if (pconn.socket == fd) + { + /* The connection FD is already registered. */ + return; + } + else + { + /* The old persistent connection is still active; close it + first. This situation arises whenever a persistent + connection exists, but we then connect to a different + host, and try to register a persistent connection to that + one. */ + invalidate_persistent (); + } + } + + pconn_active = true; + pconn.socket = fd; + pconn.host = xstrdup (host); + pconn.port = port; + pconn.ssl = ssl; + pconn.authorized = false; + + DEBUGP (("Registered socket %d for persistent reuse.\n", fd)); +} + +/* Return true if a persistent connection is available for connecting + to HOST:PORT. */ + +static bool +persistent_available_p (const char *host, int port, bool ssl, + bool *host_lookup_failed) +{ + /* First, check whether a persistent connection is active at all. */ + if (!pconn_active) + return false; + + /* If we want SSL and the last connection wasn't or vice versa, + don't use it. Checking for host and port is not enough because + HTTP and HTTPS can apparently coexist on the same port. */ + if (ssl != pconn.ssl) + return false; + + /* If we're not connecting to the same port, we're not interested. */ + if (port != pconn.port) + return false; + + /* If the host is the same, we're in business. If not, there is + still hope -- read below. */ + if (0 != strcasecmp (host, pconn.host)) + { + /* Check if pconn.socket is talking to HOST under another name. + This happens often when both sites are virtual hosts + distinguished only by name and served by the same network + interface, and hence the same web server (possibly set up by + the ISP and serving many different web sites). This + admittedly unconventional optimization does not contradict + HTTP and works well with popular server software. */ + + bool found; + ip_address ip; + struct address_list *al; + + if (ssl) + /* Don't try to talk to two different SSL sites over the same + secure connection! (Besides, it's not clear that + name-based virtual hosting is even possible with SSL.) */ + return false; + + /* If pconn.socket's peer is one of the IP addresses HOST + resolves to, pconn.socket is for all intents and purposes + already talking to HOST. */ + + if (!socket_ip_address (pconn.socket, &ip, ENDPOINT_PEER)) + { + /* Can't get the peer's address -- something must be very + wrong with the connection. */ + invalidate_persistent (); + return false; + } + al = lookup_host (host, 0); + if (!al) + { + *host_lookup_failed = true; + return false; + } + + found = address_list_contains (al, &ip); + address_list_release (al); + + if (!found) + return false; + + /* The persistent connection's peer address was found among the + addresses HOST resolved to; therefore, pconn.sock is in fact + already talking to HOST -- no need to reconnect. */ + } + + /* Finally, check whether the connection is still open. This is + important because most servers implement liberal (short) timeout + on persistent connections. Wget can of course always reconnect + if the connection doesn't work out, but it's nicer to know in + advance. This test is a logical followup of the first test, but + is "expensive" and therefore placed at the end of the list. + + (Current implementation of test_socket_open has a nice side + effect that it treats sockets with pending data as "closed". + This is exactly what we want: if a broken server sends message + body in response to HEAD, or if it sends more than conent-length + data, we won't reuse the corrupted connection.) */ + + if (!test_socket_open (pconn.socket)) + { + /* Oops, the socket is no longer open. Now that we know that, + let's invalidate the persistent connection before returning + 0. */ + invalidate_persistent (); + return false; + } + + return true; +} + +/* The idea behind these two CLOSE macros is to distinguish between + two cases: one when the job we've been doing is finished, and we + want to close the connection and leave, and two when something is + seriously wrong and we're closing the connection as part of + cleanup. + + In case of keep_alive, CLOSE_FINISH should leave the connection + open, while CLOSE_INVALIDATE should still close it. + + Note that the semantics of the flag `keep_alive' is "this + connection *will* be reused (the server has promised not to close + the connection once we're done)", while the semantics of + `pc_active_p && (fd) == pc_last_fd' is "we're *now* using an + active, registered connection". */ + +#define CLOSE_FINISH(fd) do { \ + if (!keep_alive) \ + { \ + if (pconn_active && (fd) == pconn.socket) \ + invalidate_persistent (); \ + else \ + fd_close (fd); \ + fd = -1; \ + } \ +} while (0) + +#define CLOSE_INVALIDATE(fd) do { \ + if (pconn_active && (fd) == pconn.socket) \ + invalidate_persistent (); \ + else \ + fd_close (fd); \ + fd = -1; \ +} while (0) + +typedef enum +{ + ENC_INVALID = -1, /* invalid encoding */ + ENC_NONE = 0, /* no special encoding */ + ENC_GZIP, /* gzip compression */ + ENC_DEFLATE, /* deflate compression */ + ENC_COMPRESS, /* compress compression */ + ENC_BROTLI /* brotli compression */ +} encoding_t; + +struct http_stat +{ + wgint len; /* received length */ + wgint contlen; /* expected length */ + wgint restval; /* the restart value */ + int res; /* the result of last read */ + char *rderrmsg; /* error message from read error */ + char *newloc; /* new location (redirection) */ + char *remote_time; /* remote time-stamp string */ + char *error; /* textual HTTP error */ + int statcode; /* status code */ + char *message; /* status message */ + wgint rd_size; /* amount of data read from socket */ + double dltime; /* time it took to download the data */ + const char *referer; /* value of the referer header. */ + char *local_file; /* local file name. */ + bool existence_checked; /* true if we already checked for a file's + existence after having begun to download + (needed in gethttp for when connection is + interrupted/restarted. */ + bool timestamp_checked; /* true if pre-download time-stamping checks + * have already been performed */ + char *orig_file_name; /* name of file to compare for time-stamping + * (might be != local_file if -K is set) */ + wgint orig_file_size; /* size of file to compare for time-stamping */ + time_t orig_file_tstamp; /* time-stamp of file to compare for + * time-stamping */ +#ifdef HAVE_METALINK + metalink_t *metalink; +#endif + + encoding_t local_encoding; /* the encoding of the local file */ + encoding_t remote_encoding; /* the encoding of the remote file */ + + bool temporary; /* downloading a temporary file */ +}; + +static void +free_hstat (struct http_stat *hs) +{ + xfree (hs->newloc); + xfree (hs->remote_time); + xfree (hs->error); + xfree (hs->rderrmsg); + xfree (hs->local_file); + xfree (hs->orig_file_name); + xfree (hs->message); +#ifdef HAVE_METALINK + metalink_delete (hs->metalink); + hs->metalink = NULL; +#endif +} + +static void +get_file_flags (const char *filename, int *dt) +{ + logprintf (LOG_VERBOSE, _("\ +File %s already there; not retrieving.\n\n"), quote (filename)); + /* If the file is there, we suppose it's retrieved OK. */ + *dt |= RETROKF; + + /* #### Bogusness alert. */ + /* If its suffix is "html" or "htm" or similar, assume text/html. */ + if (has_html_suffix_p (filename)) + *dt |= TEXTHTML; +} + +/* Download the response body from the socket and writes it to + an output file. The headers have already been read from the + socket. If WARC is enabled, the response body will also be + written to a WARC response record. + + hs, contlen, contrange, chunked_transfer_encoding and url are + parameters from the gethttp method. fp is a pointer to the + output file. + + url, warc_timestamp_str, warc_request_uuid, warc_ip, type + and statcode will be saved in the headers of the WARC record. + The head parameter contains the HTTP headers of the response. + + If fp is NULL and WARC is enabled, the response body will be + written only to the WARC file. If WARC is disabled and fp + is a file pointer, the data will be written to the file. + If fp is a file pointer and WARC is enabled, the body will + be written to both destinations. + + Returns the error code. */ +static int +read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen, + wgint contrange, bool chunked_transfer_encoding, + char *url, char *warc_timestamp_str, char *warc_request_uuid, + ip_address *warc_ip, char *type, int statcode, char *head) +{ + int warc_payload_offset = 0; + FILE *warc_tmp = NULL; + int warcerr = 0; + int flags = 0; + + if (opt.warc_filename != NULL) + { + /* Open a temporary file where we can write the response before we + add it to the WARC record. */ + warc_tmp = warc_tempfile (); + if (warc_tmp == NULL) + warcerr = WARC_TMP_FOPENERR; + + if (warcerr == 0) + { + /* We should keep the response headers for the WARC record. */ + int head_len = strlen (head); + int warc_tmp_written = fwrite (head, 1, head_len, warc_tmp); + if (warc_tmp_written != head_len) + warcerr = WARC_TMP_FWRITEERR; + warc_payload_offset = head_len; + } + + if (warcerr != 0) + { + if (warc_tmp != NULL) + fclose (warc_tmp); + return warcerr; + } + } + + if (fp != NULL) + { + /* This confuses the timestamping code that checks for file size. + #### The timestamping code should be smarter about file size. */ + if (opt.save_headers && hs->restval == 0) + fwrite (head, 1, strlen (head), fp); + } + + /* Read the response body. */ + if (contlen != -1) + /* If content-length is present, read that much; otherwise, read + until EOF. The HTTP spec doesn't require the server to + actually close the connection when it's done sending data. */ + flags |= rb_read_exactly; + if (fp != NULL && hs->restval > 0 && contrange == 0) + /* If the server ignored our range request, instruct fd_read_body + to skip the first RESTVAL bytes of body. */ + flags |= rb_skip_startpos; + if (chunked_transfer_encoding) + flags |= rb_chunked_transfer_encoding; + + if (hs->remote_encoding == ENC_GZIP) + flags |= rb_compressed_gzip; + + hs->len = hs->restval; + hs->rd_size = 0; + /* Download the response body and write it to fp. + If we are working on a WARC file, we simultaneously write the + response body to warc_tmp. */ + hs->res = fd_read_body (hs->local_file, sock, fp, contlen != -1 ? contlen : 0, + hs->restval, &hs->rd_size, &hs->len, &hs->dltime, + flags, warc_tmp); + if (hs->res >= 0) + { + if (warc_tmp != NULL) + { + /* Create a response record and write it to the WARC file. + Note: per the WARC standard, the request and response should share + the same date header. We re-use the timestamp of the request. + The response record should also refer to the uuid of the request. */ + bool r = warc_write_response_record (url, warc_timestamp_str, + warc_request_uuid, warc_ip, + warc_tmp, warc_payload_offset, + type, statcode, hs->newloc); + + /* warc_write_response_record has closed warc_tmp. */ + + if (! r) + return WARC_ERR; + } + + return RETRFINISHED; + } + + if (warc_tmp != NULL) + fclose (warc_tmp); + + if (hs->res == -2) + { + /* Error while writing to fd. */ + return FWRITEERR; + } + else if (hs->res == -3) + { + /* Error while writing to warc_tmp. */ + return WARC_TMP_FWRITEERR; + } + else + { + /* A read error! */ + xfree (hs->rderrmsg); + hs->rderrmsg = xstrdup (fd_errstr (sock)); + return RETRFINISHED; + } +} + +#define BEGINS_WITH(line, string_constant) \ + (!c_strncasecmp (line, string_constant, sizeof (string_constant) - 1) \ + && (c_isspace (line[sizeof (string_constant) - 1]) \ + || !line[sizeof (string_constant) - 1])) + +#define SET_USER_AGENT(req) do { \ + if (!opt.useragent) \ + request_set_header (req, "User-Agent", \ + aprintf ("Wget/%s", \ + version_string), \ + rel_value); \ + else if (*opt.useragent) \ + request_set_header (req, "User-Agent", opt.useragent, rel_none); \ +} while (0) + +/* + Convert time_t to one of valid HTTP date formats + ie. rfc1123-date. + + HTTP-date = rfc1123-date | rfc850-date | asctime-date + rfc1123-date = wkday "," SP date1 SP time SP "GMT" + rfc850-date = weekday "," SP date2 SP time SP "GMT" + asctime-date = wkday SP date3 SP time SP 4DIGIT + date1 = 2DIGIT SP month SP 4DIGIT + ; day month year (e.g., 02 Jun 1982) + date2 = 2DIGIT "-" month "-" 2DIGIT + ; day-month-year (e.g., 02-Jun-82) + date3 = month SP ( 2DIGIT | ( SP 1DIGIT )) + ; month day (e.g., Jun 2) + time = 2DIGIT ":" 2DIGIT ":" 2DIGIT + ; 00:00:00 - 23:59:59 + wkday = "Mon" | "Tue" | "Wed" + | "Thu" | "Fri" | "Sat" | "Sun" + weekday = "Monday" | "Tuesday" | "Wednesday" + | "Thursday" | "Friday" | "Saturday" | "Sunday" + month = "Jan" | "Feb" | "Mar" | "Apr" + | "May" | "Jun" | "Jul" | "Aug" + | "Sep" | "Oct" | "Nov" | "Dec" + + source: RFC2616 */ +static uerr_t +time_to_rfc1123 (time_t time, char *buf, size_t bufsize) +{ + static const char *wkday[] = { "Sun", "Mon", "Tue", "Wed", + "Thu", "Fri", "Sat" }; + static const char *month[] = { "Jan", "Feb", "Mar", "Apr", + "May", "Jun", "Jul", "Aug", + "Sep", "Oct", "Nov", "Dec" }; + /* rfc1123 example: Thu, 01 Jan 1998 22:12:57 GMT */ + static const char *time_format = "%s, %02d %s %04d %02d:%02d:%02d GMT"; + + struct tm *gtm = gmtime (&time); + if (!gtm) + { + logprintf (LOG_NOTQUIET, + _("gmtime failed. This is probably a bug.\n")); + return TIMECONV_ERR; + } + + snprintf (buf, bufsize, time_format, wkday[gtm->tm_wday], + gtm->tm_mday, month[gtm->tm_mon], + gtm->tm_year + 1900, gtm->tm_hour, + gtm->tm_min, gtm->tm_sec); + + return RETROK; +} + +static struct request * +initialize_request (const struct url *u, struct http_stat *hs, int *dt, struct url *proxy, + bool inhibit_keep_alive, bool *basic_auth_finished, + wgint *body_data_size, char **user, char **passwd, uerr_t *ret) +{ + bool head_only = !!(*dt & HEAD_ONLY); + struct request *req; + + /* Prepare the request to send. */ + { + char *meth_arg; + const char *meth = "GET"; + if (head_only) + meth = "HEAD"; + else if (opt.method) + meth = opt.method; + /* Use the full path, i.e. one that includes the leading slash and + the query string. E.g. if u->path is "foo/bar" and u->query is + "param=value", full_path will be "/foo/bar?param=value". */ + if (proxy +#ifdef HAVE_SSL + /* When using SSL over proxy, CONNECT establishes a direct + connection to the HTTPS server. Therefore use the same + argument as when talking to the server directly. */ + && u->scheme != SCHEME_HTTPS +#endif + ) + meth_arg = xstrdup (u->url); + else + meth_arg = url_full_path (u); + req = request_new (meth, meth_arg); + } + + request_set_header (req, "Referer", hs->referer, rel_none); + if (*dt & SEND_NOCACHE) + { + /* Cache-Control MUST be obeyed by all HTTP/1.1 caching mechanisms... */ + request_set_header (req, "Cache-Control", "no-cache", rel_none); + + /* ... but some HTTP/1.0 caches doesn't implement Cache-Control. */ + request_set_header (req, "Pragma", "no-cache", rel_none); + } + if (*dt & IF_MODIFIED_SINCE) + { + char strtime[32]; + uerr_t err = time_to_rfc1123 (hs->orig_file_tstamp, strtime, countof (strtime)); + + if (err != RETROK) + { + logputs (LOG_VERBOSE, _("Cannot convert timestamp to http format. " + "Falling back to time 0 as last modification " + "time.\n")); + strcpy (strtime, "Thu, 01 Jan 1970 00:00:00 GMT"); + } + request_set_header (req, "If-Modified-Since", xstrdup (strtime), rel_value); + } + if (hs->restval) + request_set_header (req, "Range", + aprintf ("bytes=%s-", + number_to_static_string (hs->restval)), + rel_value); + SET_USER_AGENT (req); + request_set_header (req, "Accept", "*/*", rel_none); +#ifdef HAVE_LIBZ + if (opt.compression != compression_none) + request_set_header (req, "Accept-Encoding", "gzip", rel_none); + else +#endif + request_set_header (req, "Accept-Encoding", "identity", rel_none); + + /* Find the username with priority */ + if (u->user) + *user = u->user; + else if (opt.user && (opt.use_askpass || opt.ask_passwd)) + *user = opt.user; + else if (opt.http_user) + *user = opt.http_user; + else if (opt.user) + *user = opt.user; + else + *user = NULL; + + /* Find the password with priority */ + if (u->passwd) + *passwd = u->passwd; + else if (opt.passwd && (opt.use_askpass || opt.ask_passwd)) + *passwd = opt.passwd; + else if (opt.http_passwd) + *passwd = opt.http_passwd; + else if (opt.passwd) + *passwd = opt.passwd; + else + *passwd = NULL; + + /* Check for ~/.netrc if none of the above match */ + if (opt.netrc && (!*user || !*passwd)) + search_netrc (u->host, (const char **) user, (const char **) passwd, 0, NULL); + + /* We only do "site-wide" authentication with "global" user/password + * values unless --auth-no-challenge has been requested; URL user/password + * info overrides. */ + if (*user && *passwd && (!u->user || opt.auth_without_challenge)) + { + /* If this is a host for which we've already received a Basic + * challenge, we'll go ahead and send Basic authentication creds. */ + *basic_auth_finished = maybe_send_basic_creds (u->host, *user, *passwd, req); + } + + /* Generate the Host header, HOST:PORT. Take into account that: + + - Broken server-side software often doesn't recognize the PORT + argument, so we must generate "Host: www.server.com" instead of + "Host: www.server.com:80" (and likewise for https port). + + - IPv6 addresses contain ":", so "Host: 3ffe:8100:200:2::2:1234" + becomes ambiguous and needs to be rewritten as "Host: + [3ffe:8100:200:2::2]:1234". */ + { + /* Formats arranged for hfmt[add_port][add_squares]. */ + static const char *hfmt[][2] = { + { "%s", "[%s]" }, { "%s:%d", "[%s]:%d" } + }; + int add_port = u->port != scheme_default_port (u->scheme); + int add_squares = strchr (u->host, ':') != NULL; + request_set_header (req, "Host", + aprintf (hfmt[add_port][add_squares], u->host, u->port), + rel_value); + } + + if (inhibit_keep_alive) + request_set_header (req, "Connection", "Close", rel_none); + else + { + request_set_header (req, "Connection", "Keep-Alive", rel_none); + if (proxy) + request_set_header (req, "Proxy-Connection", "Keep-Alive", rel_none); + } + + if (opt.method) + { + + if (opt.body_data || opt.body_file) + { + request_set_header (req, "Content-Type", + "application/x-www-form-urlencoded", rel_none); + + if (opt.body_data) + *body_data_size = strlen (opt.body_data); + else + { + *body_data_size = file_size (opt.body_file); + if (*body_data_size == -1) + { + logprintf (LOG_NOTQUIET, _("BODY data file %s missing: %s\n"), + quote (opt.body_file), strerror (errno)); + request_free (&req); + *ret = FILEBADFILE; + return NULL; + } + } + request_set_header (req, "Content-Length", + xstrdup (number_to_static_string (*body_data_size)), + rel_value); + } + else if (c_strcasecmp (opt.method, "post") == 0 + || c_strcasecmp (opt.method, "put") == 0 + || c_strcasecmp (opt.method, "patch") == 0) + request_set_header (req, "Content-Length", "0", rel_none); + } + return req; +} + +static void +initialize_proxy_configuration (const struct url *u, struct request *req, + struct url *proxy, char **proxyauth) +{ + char *proxy_user, *proxy_passwd; + /* For normal username and password, URL components override + command-line/wgetrc parameters. With proxy + authentication, it's the reverse, because proxy URLs are + normally the "permanent" ones, so command-line args + should take precedence. */ + if (opt.proxy_user && opt.proxy_passwd) + { + proxy_user = opt.proxy_user; + proxy_passwd = opt.proxy_passwd; + } + else + { + proxy_user = proxy->user; + proxy_passwd = proxy->passwd; + } + /* #### This does not appear right. Can't the proxy request, + say, `Digest' authentication? */ + if (proxy_user && proxy_passwd) + *proxyauth = basic_authentication_encode (proxy_user, proxy_passwd); + + /* Proxy authorization over SSL is handled below. */ +#ifdef HAVE_SSL + if (u->scheme != SCHEME_HTTPS) +#endif + request_set_header (req, "Proxy-Authorization", *proxyauth, rel_value); +} + +static uerr_t +establish_connection (const struct url *u, const struct url **conn_ref, + struct http_stat *hs, struct url *proxy, + char **proxyauth, + struct request **req_ref, bool *using_ssl, + bool inhibit_keep_alive, + int *sock_ref) +{ + bool host_lookup_failed = false; + int sock = *sock_ref; + struct request *req = *req_ref; + const struct url *conn = *conn_ref; + struct response *resp; + int write_error; + int statcode; + + if (! inhibit_keep_alive) + { + /* Look for a persistent connection to target host, unless a + proxy is used. The exception is when SSL is in use, in which + case the proxy is nothing but a passthrough to the target + host, registered as a connection to the latter. */ + const struct url *relevant = conn; +#ifdef HAVE_SSL + if (u->scheme == SCHEME_HTTPS) + relevant = u; +#endif + + if (persistent_available_p (relevant->host, relevant->port, +#ifdef HAVE_SSL + relevant->scheme == SCHEME_HTTPS, +#else + 0, +#endif + &host_lookup_failed)) + { + int family = socket_family (pconn.socket, ENDPOINT_PEER); + sock = pconn.socket; + *using_ssl = pconn.ssl; +#if ENABLE_IPV6 + if (family == AF_INET6) + logprintf (LOG_VERBOSE, _("Reusing existing connection to [%s]:%d.\n"), + quotearg_style (escape_quoting_style, pconn.host), + pconn.port); + else +#endif + logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"), + quotearg_style (escape_quoting_style, pconn.host), + pconn.port); + DEBUGP (("Reusing fd %d.\n", sock)); + if (pconn.authorized) + /* If the connection is already authorized, the "Basic" + authorization added by code above is unnecessary and + only hurts us. */ + request_remove_header (req, "Authorization"); + } + else if (host_lookup_failed) + { + logprintf(LOG_NOTQUIET, + _("%s: unable to resolve host address %s\n"), + exec_name, quote (relevant->host)); + return HOSTERR; + } + else if (sock != -1) + { + sock = -1; + } + } + + if (sock < 0) + { + sock = connect_to_host (conn->host, conn->port); + if (sock == E_HOST) + return HOSTERR; + else if (sock < 0) + return (retryable_socket_connect_error (errno) + ? CONERROR : CONIMPOSSIBLE); + +#ifdef HAVE_SSL + if (proxy && u->scheme == SCHEME_HTTPS) + { + char *head; + char *message; + /* When requesting SSL URLs through proxies, use the + CONNECT method to request passthrough. */ + struct request *connreq = request_new ("CONNECT", + aprintf ("%s:%d", u->host, u->port)); + SET_USER_AGENT (connreq); + if (proxyauth) + { + request_set_header (connreq, "Proxy-Authorization", + *proxyauth, rel_value); + /* Now that PROXYAUTH is part of the CONNECT request, + zero it out so we don't send proxy authorization with + the regular request below. */ + *proxyauth = NULL; + } + request_set_header (connreq, "Host", + aprintf ("%s:%d", u->host, u->port), + rel_value); + + write_error = request_send (connreq, sock, 0); + request_free (&connreq); + if (write_error < 0) + { + CLOSE_INVALIDATE (sock); + return WRITEFAILED; + } + + head = read_http_response_head (sock); + if (!head) + { + logprintf (LOG_VERBOSE, _("Failed reading proxy response: %s\n"), + fd_errstr (sock)); + CLOSE_INVALIDATE (sock); + return HERR; + } + message = NULL; + if (!*head) + { + xfree (head); + goto failed_tunnel; + } + DEBUGP (("proxy responded with: [%s]\n", head)); + + resp = resp_new (head); + statcode = resp_status (resp, &message); + if (statcode < 0) + { + char *tms = datetime_str (time (NULL)); + logprintf (LOG_VERBOSE, "%d\n", statcode); + logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"), tms, statcode, + quotearg_style (escape_quoting_style, + _("Malformed status line"))); + xfree (head); + return HERR; + } + xfree (hs->message); + hs->message = xstrdup (message); + resp_free (&resp); + xfree (head); + if (statcode != 200) + { + failed_tunnel: + logprintf (LOG_NOTQUIET, _("Proxy tunneling failed: %s"), + message ? quotearg_style (escape_quoting_style, message) : "?"); + xfree (message); + return CONSSLERR; + } + xfree (message); + + /* SOCK is now *really* connected to u->host, so update CONN + to reflect this. That way register_persistent will + register SOCK as being connected to u->host:u->port. */ + conn = u; + } + + if (conn->scheme == SCHEME_HTTPS) + { + if (!ssl_connect_wget (sock, u->host, NULL)) + { + CLOSE_INVALIDATE (sock); + return CONSSLERR; + } + else if (!ssl_check_certificate (sock, u->host)) + { + CLOSE_INVALIDATE (sock); + return VERIFCERTERR; + } + *using_ssl = true; + } +#endif /* HAVE_SSL */ + } + *conn_ref = conn; + *req_ref = req; + *sock_ref = sock; + return RETROK; +} + +static uerr_t +set_file_timestamp (struct http_stat *hs) +{ + bool local_dot_orig_file_exists = false; + char *local_filename = NULL; + struct stat st; + char buf[1024]; + + if (opt.backup_converted) + /* If -K is specified, we'll act on the assumption that it was specified + last time these files were downloaded as well, and instead of just + comparing local file X against server file X, we'll compare local + file X.orig (if extant, else X) against server file X. If -K + _wasn't_ specified last time, or the server contains files called + *.orig, -N will be back to not operating correctly with -k. */ + { + size_t filename_len = strlen (hs->local_file); + char *filename_plus_orig_suffix; + + if (filename_len + sizeof (ORIG_SFX) > sizeof (buf)) + filename_plus_orig_suffix = xmalloc (filename_len + sizeof (ORIG_SFX)); + else + filename_plus_orig_suffix = buf; + + /* Would a single s[n]printf() call be faster? --dan + + Definitely not. sprintf() is horribly slow. It's a + different question whether the difference between the two + affects a program. Usually I'd say "no", but at one + point I profiled Wget, and found that a measurable and + non-negligible amount of time was lost calling sprintf() + in url.c. Replacing sprintf with inline calls to + strcpy() and number_to_string() made a difference. + --hniksic */ + memcpy (filename_plus_orig_suffix, hs->local_file, filename_len); + memcpy (filename_plus_orig_suffix + filename_len, + ORIG_SFX, sizeof (ORIG_SFX)); + + /* Try to stat() the .orig file. */ + if (stat (filename_plus_orig_suffix, &st) == 0) + { + local_dot_orig_file_exists = true; + local_filename = filename_plus_orig_suffix; + } + } + + if (!local_dot_orig_file_exists) + /* Couldn't stat() <file>.orig, so try to stat() <file>. */ + if (stat (hs->local_file, &st) == 0) + { + if (local_filename != buf) + xfree (local_filename); + local_filename = hs->local_file; + } + + if (local_filename != NULL) + /* There was a local file, so we'll check later to see if the version + the server has is the same version we already have, allowing us to + skip a download. */ + { + if (local_filename == buf || local_filename == hs->local_file) + hs->orig_file_name = xstrdup (local_filename); // on stack or a copy, make a heap copy + else + hs->orig_file_name = local_filename; // was previously malloc'ed + hs->orig_file_size = st.st_size; + hs->orig_file_tstamp = st.st_mtime; +#ifdef WINDOWS + /* Modification time granularity is 2 seconds for Windows, so + increase local time by 1 second for later comparison. */ + ++hs->orig_file_tstamp; +#endif + hs->timestamp_checked = true; + } + + return RETROK; +} + +static uerr_t +check_file_output (const struct url *u, struct http_stat *hs, + struct response *resp, char *hdrval, size_t hdrsize) +{ + /* Determine the local filename if needed. Notice that if -O is used + * hstat.local_file is set by http_loop to the argument of -O. */ + if (!hs->local_file) + { + char *local_file = NULL; + + /* Honor Content-Disposition whether possible. */ + if (!opt.content_disposition + || !resp_header_copy (resp, "Content-Disposition", + hdrval, hdrsize) + || !parse_content_disposition (hdrval, &local_file)) + { + /* The Content-Disposition header is missing or broken. + * Choose unique file name according to given URL. */ + hs->local_file = url_file_name (u, NULL); + } + else + { + DEBUGP (("Parsed filename from Content-Disposition: %s\n", + local_file)); + hs->local_file = url_file_name (u, local_file); + } + + xfree (local_file); + } + + hs->temporary = opt.delete_after || opt.spider || !acceptable (hs->local_file); + if (hs->temporary) + { + char *tmp = aprintf ("%s.tmp", hs->local_file); + xfree (hs->local_file); + hs->local_file = tmp; + } + + /* TODO: perform this check only once. */ + if (!hs->existence_checked && file_exists_p (hs->local_file, NULL)) + { + if (opt.noclobber && !opt.output_document) + { + /* If opt.noclobber is turned on and file already exists, do not + retrieve the file. But if the output_document was given, then this + test was already done and the file didn't exist. Hence the !opt.output_document */ + return RETRUNNEEDED; + } + else if (!ALLOW_CLOBBER) + { + char *unique = unique_name_passthrough (hs->local_file); + if (unique != hs->local_file) + xfree (hs->local_file); + hs->local_file = unique; + } + } + hs->existence_checked = true; + + /* Support timestamping */ + if (opt.timestamping && !hs->timestamp_checked) + { + uerr_t timestamp_err = set_file_timestamp (hs); + if (timestamp_err != RETROK) + return timestamp_err; + } + return RETROK; +} + +static uerr_t +check_auth (const struct url *u, char *user, char *passwd, struct response *resp, + struct request *req, bool *ntlm_seen_ref, bool *retry, + bool *basic_auth_finished_ref, bool *auth_finished_ref) +{ + uerr_t auth_err = RETROK; + bool basic_auth_finished = *basic_auth_finished_ref; + bool auth_finished = *auth_finished_ref; + bool ntlm_seen = *ntlm_seen_ref; + char buf[256], *tmp = NULL; + + *retry = false; + + if (!auth_finished && (user && passwd)) + { + /* IIS sends multiple copies of WWW-Authenticate, one with + the value "negotiate", and other(s) with data. Loop over + all the occurrences and pick the one we recognize. */ + int wapos; + const char *www_authenticate = NULL; + const char *wabeg, *waend; + const char *digest = NULL, *basic = NULL, *ntlm = NULL; + + for (wapos = 0; !ntlm + && (wapos = resp_header_locate (resp, "WWW-Authenticate", wapos, + &wabeg, &waend)) != -1; + ++wapos) + { + param_token name, value; + size_t len = waend - wabeg; + + if (tmp != buf) + xfree (tmp); + + if (len < sizeof (buf)) + tmp = buf; + else + tmp = xmalloc (len + 1); + + memcpy (tmp, wabeg, len); + tmp[len] = 0; + + www_authenticate = tmp; + + for (;!ntlm;) + { + /* extract the auth-scheme */ + while (c_isspace (*www_authenticate)) www_authenticate++; + name.e = name.b = www_authenticate; + while (*name.e && !c_isspace (*name.e)) name.e++; + + if (name.b == name.e) + break; + + DEBUGP (("Auth scheme found '%.*s'\n", (int) (name.e - name.b), name.b)); + + if (known_authentication_scheme_p (name.b, name.e)) + { + if (BEGINS_WITH (name.b, "NTLM")) + { + ntlm = name.b; + break; /* this is the most secure challenge, stop here */ + } + else if (!digest && BEGINS_WITH (name.b, "Digest")) + digest = name.b; + else if (!basic && BEGINS_WITH (name.b, "Basic")) + basic = name.b; + } + + /* now advance over the auth-params */ + www_authenticate = name.e; + DEBUGP (("Auth param list '%s'\n", www_authenticate)); + while (extract_param (&www_authenticate, &name, &value, ',', NULL) && name.b && value.b) + { + DEBUGP (("Auth param %.*s=%.*s\n", + (int) (name.e - name.b), name.b, (int) (value.e - value.b), value.b)); + } + } + } + + if (!basic && !digest && !ntlm) + { + /* If the authentication header is missing or + unrecognized, there's no sense in retrying. */ + logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n")); + } + else if (!basic_auth_finished + || !basic) + { + char *pth = url_full_path (u); + const char *value; + uerr_t *auth_stat; + auth_stat = xmalloc (sizeof (uerr_t)); + *auth_stat = RETROK; + + if (ntlm) + www_authenticate = ntlm; + else if (digest) + www_authenticate = digest; + else + www_authenticate = basic; + + logprintf (LOG_NOTQUIET, _("Authentication selected: %s\n"), www_authenticate); + + value = create_authorization_line (www_authenticate, + user, passwd, + request_method (req), + pth, + &auth_finished, + auth_stat); + + auth_err = *auth_stat; + xfree (auth_stat); + xfree (pth); + if (auth_err == RETROK) + { + request_set_header (req, "Authorization", value, rel_value); + + if (BEGINS_WITH (www_authenticate, "NTLM")) + ntlm_seen = true; + else if (!u->user && BEGINS_WITH (www_authenticate, "Basic")) + { + /* Need to register this host as using basic auth, + * so we automatically send creds next time. */ + register_basic_auth_host (u->host); + } + + *retry = true; + goto cleanup; + } + else + { + /* Creating the Authorization header went wrong */ + xfree (value); + } + } + else + { + /* We already did Basic auth, and it failed. Gotta + * give up. */ + } + } + + cleanup: + if (tmp != buf) + xfree (tmp); + *ntlm_seen_ref = ntlm_seen; + *basic_auth_finished_ref = basic_auth_finished; + *auth_finished_ref = auth_finished; + return auth_err; +} + +static uerr_t +open_output_stream (struct http_stat *hs, int count, FILE **fp) +{ +/* 2005-06-17 SMS. + For VMS, define common fopen() optional arguments. +*/ +#ifdef __VMS +# define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id +# define FOPEN_BIN_FLAG 3 +#else /* def __VMS */ +# define FOPEN_BIN_FLAG true +#endif /* def __VMS [else] */ + + /* Open the local file. */ + if (!output_stream) + { + mkalldirs (hs->local_file); + if (opt.backups) + rotate_backups (hs->local_file); + if (hs->restval) + { +#ifdef __VMS + int open_id; + + open_id = 21; + *fp = fopen (hs->local_file, "ab", FOPEN_OPT_ARGS); +#else /* def __VMS */ + *fp = fopen (hs->local_file, "ab"); +#endif /* def __VMS [else] */ + } + else if (ALLOW_CLOBBER || count > 0) + { + if (opt.unlink_requested && file_exists_p (hs->local_file, NULL)) + { + if (unlink (hs->local_file) < 0) + { + logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file, + strerror (errno)); + return UNLINKERR; + } + } + +#ifdef __VMS + int open_id; + + open_id = 22; + *fp = fopen (hs->local_file, "wb", FOPEN_OPT_ARGS); +#else /* def __VMS */ + if (hs->temporary) + { + *fp = fdopen (open (hs->local_file, O_BINARY | O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR), "wb"); + } + else + { + *fp = fopen (hs->local_file, "wb"); + } + +#endif /* def __VMS [else] */ + } + else + { + *fp = fopen_excl (hs->local_file, FOPEN_BIN_FLAG); + if (!*fp && errno == EEXIST) + { + /* We cannot just invent a new name and use it (which is + what functions like unique_create typically do) + because we told the user we'd use this name. + Instead, return and retry the download. */ + logprintf (LOG_NOTQUIET, + _("%s has sprung into existence.\n"), + hs->local_file); + return FOPEN_EXCL_ERR; + } + } + if (!*fp) + { + logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file, strerror (errno)); + return FOPENERR; + } + } + else + *fp = output_stream; + + /* Print fetch message, if opt.verbose. */ + logprintf (LOG_VERBOSE, _("Saving to: %s\n"), + HYPHENP (hs->local_file) ? quote ("STDOUT") : quote (hs->local_file)); + + return RETROK; +} + +/* Set proper type flags based on type string. */ +static void +set_content_type (int *dt, const char *type) +{ + /* If content-type is not given, assume text/html. This is because + of the multitude of broken CGI's that "forget" to generate the + content-type. */ + if (!type || + 0 == c_strcasecmp (type, TEXTHTML_S) || + 0 == c_strcasecmp (type, TEXTXHTML_S)) + *dt |= TEXTHTML; + else + *dt &= ~TEXTHTML; + + if (type && + 0 == c_strcasecmp (type, TEXTCSS_S)) + *dt |= TEXTCSS; + else + *dt &= ~TEXTCSS; +} + +#ifdef HAVE_METALINK +/* Will return proper metalink_t structure if enough data was found in + http response resp. Otherwise returns NULL. + Two exit points: one for success and one for failure. */ +static metalink_t * +metalink_from_http (const struct response *resp, const struct http_stat *hs, + const struct url *u) +{ + metalink_t *metalink = NULL; + metalink_file_t *mfile = xnew0 (metalink_file_t); + const char *val_beg, *val_end; + int res_count = 0, meta_count = 0, hash_count = 0, sig_count = 0, i; + + DEBUGP (("Checking for Metalink in HTTP response\n")); + + /* Initialize metalink file for our simple use case. */ + if (hs->local_file) + mfile->name = xstrdup (hs->local_file); + else + mfile->name = url_file_name (u, NULL); + + /* Begin with 1-element array (for 0-termination). */ + mfile->checksums = xnew0 (metalink_checksum_t *); + mfile->resources = xnew0 (metalink_resource_t *); + mfile->metaurls = xnew0 (metalink_metaurl_t *); + + /* Process the Content-Type header. */ + if (resp_header_locate (resp, "Content-Type", 0, &val_beg, &val_end) != -1) + { + metalink_metaurl_t murl = {0}; + + const char *type_beg, *type_end; + char *typestr = NULL; + char *namestr = NULL; + size_t type_len; + + DEBUGP (("Processing Content-Type header...\n")); + + /* Find beginning of type. */ + type_beg = val_beg; + while (type_beg < val_end && c_isspace (*type_beg)) + type_beg++; + + /* Find end of type. */ + type_end = type_beg + 1; + while (type_end < val_end && + *type_end != ';' && + *type_end != ' ' && + *type_end != '\r' && + *type_end != '\n') + type_end++; + + if (type_beg >= val_end || type_end > val_end) + { + DEBUGP (("Invalid Content-Type header. Ignoring.\n")); + goto skip_content_type; + } + + type_len = type_end - type_beg; + typestr = xstrndup (type_beg, type_len); + + DEBUGP (("Content-Type: %s\n", typestr)); + + if (strcmp (typestr, "application/metalink4+xml")) + { + xfree (typestr); + goto skip_content_type; + } + + /* + Valid ranges for the "pri" attribute are from + 1 to 999999. Mirror servers with a lower value of the "pri" + attribute have a higher priority, while mirrors with an undefined + "pri" attribute are considered to have a value of 999999, which is + the lowest priority. + + rfc6249 section 3.1 + */ + murl.priority = DEFAULT_PRI; + + murl.mediatype = typestr; + typestr = NULL; + + if (opt.content_disposition + && resp_header_locate (resp, "Content-Disposition", 0, &val_beg, &val_end) != -1) + { + find_key_value (val_beg, val_end, "filename", &namestr); + murl.name = namestr; + namestr = NULL; + } + + murl.url = xstrdup (u->url); + + DEBUGP (("URL=%s\n", murl.url)); + DEBUGP (("MEDIATYPE=%s\n", murl.mediatype)); + DEBUGP (("NAME=%s\n", murl.name ? murl.name : "")); + DEBUGP (("PRIORITY=%d\n", murl.priority)); + + /* 1 slot from new resource, 1 slot for null-termination. */ + mfile->metaurls = xrealloc (mfile->metaurls, + sizeof (metalink_metaurl_t *) * (meta_count + 2)); + mfile->metaurls[meta_count] = xnew0 (metalink_metaurl_t); + *mfile->metaurls[meta_count] = murl; + meta_count++; + } +skip_content_type: + + /* Find all Link headers. */ + for (i = 0; + (i = resp_header_locate (resp, "Link", i, &val_beg, &val_end)) != -1; + i++) + { + char *rel = NULL, *reltype = NULL; + char *urlstr = NULL; + const char *url_beg, *url_end, *attrs_beg; + size_t url_len; + + /* Sample Metalink Link headers: + + Link: <http://www2.example.com/dir1/dir2/dir3/dir4/dir5/example.ext>; + rel=duplicate; pri=1; pref; geo=gb; depth=4 + + Link: <http://example.com/example.ext.asc>; rel=describedby; + type="application/pgp-signature" + */ + + /* Find beginning of URL. */ + url_beg = val_beg; + while (url_beg < val_end - 1 && c_isspace (*url_beg)) + url_beg++; + + /* Find end of URL. */ + /* The convention here is that end ptr points to one element after + end of string. In this case, it should be pointing to the '>', which + is one element after end of actual URL. Therefore, it should never point + to val_end, which is one element after entire header value string. */ + url_end = url_beg + 1; + while (url_end < val_end - 1 && *url_end != '>') + url_end++; + + if (url_beg >= val_end || url_end >= val_end || + *url_beg != '<' || *url_end != '>') + { + DEBUGP (("This is not a valid Link header. Ignoring.\n")); + continue; + } + + /* Skip <. */ + url_beg++; + url_len = url_end - url_beg; + + /* URL found. Now handle the attributes. */ + attrs_beg = url_end + 1; + + /* First we need to find out what type of link it is. Currently, we + support rel=duplicate and rel=describedby. */ + if (!find_key_value (attrs_beg, val_end, "rel", &rel)) + { + DEBUGP (("No rel value in Link header, skipping.\n")); + continue; + } + + urlstr = xstrndup (url_beg, url_len); + DEBUGP (("URL=%s\n", urlstr)); + DEBUGP (("rel=%s\n", rel)); + + if (!strcmp (rel, "describedby")) + find_key_value (attrs_beg, val_end, "type", &reltype); + + /* Handle signatures. + Libmetalink only supports one signature per file. Therefore we stop + as soon as we successfully get first supported signature. */ + if (sig_count == 0 && + reltype && !strcmp (reltype, "application/pgp-signature")) + { + /* Download the signature to a temporary file. */ + FILE *_output_stream = output_stream; + bool _output_stream_regular = output_stream_regular; + + output_stream = tmpfile (); + if (output_stream) + { + struct iri *iri = iri_new (); + struct url *url; + int url_err; + + set_uri_encoding (iri, opt.locale, true); + url = url_parse (urlstr, &url_err, iri, false); + + if (!url) + { + char *error = url_error (urlstr, url_err); + logprintf (LOG_NOTQUIET, _("When downloading signature:\n" + "%s: %s.\n"), urlstr, error); + xfree (error); + iri_free (iri); + } + else + { + /* Avoid recursive Metalink from HTTP headers. */ + bool _metalink_http = opt.metalink_over_http; + uerr_t retr_err; + + opt.metalink_over_http = false; + retr_err = retrieve_url (url, urlstr, NULL, NULL, + NULL, NULL, false, iri, false); + opt.metalink_over_http = _metalink_http; + + url_free (url); + iri_free (iri); + + if (retr_err == RETROK) + { + /* Signature is in the temporary file. Read it into + metalink resource structure. */ + metalink_signature_t msig; + size_t siglen; + + fseek (output_stream, 0, SEEK_END); + siglen = ftell (output_stream); + fseek (output_stream, 0, SEEK_SET); + + DEBUGP (("siglen=%lu\n", siglen)); + + msig.signature = xmalloc (siglen + 1); + if (fread (msig.signature, siglen, 1, output_stream) != 1) + { + logputs (LOG_NOTQUIET, + _("Unable to read signature content from " + "temporary file. Skipping.\n")); + xfree (msig.signature); + } + else + { + msig.signature[siglen] = '\0'; /* Just in case. */ + msig.mediatype = xstrdup ("application/pgp-signature"); + + DEBUGP (("Signature (%s):\n%s\n", + msig.mediatype, msig.signature)); + + mfile->signature = xnew (metalink_signature_t); + *mfile->signature = msig; + + sig_count++; + } + } + } + fclose (output_stream); + } + else + { + logputs (LOG_NOTQUIET, _("Could not create temporary file. " + "Skipping signature download.\n")); + } + output_stream_regular = _output_stream_regular; + output_stream = _output_stream; + } /* Iterate over signatures. */ + + /* Handle Metalink resources. */ + else if (!strcmp (rel, "duplicate")) + { + metalink_resource_t mres = {0}; + char *pristr; + + /* + Valid ranges for the "pri" attribute are from + 1 to 999999. Mirror servers with a lower value of the "pri" + attribute have a higher priority, while mirrors with an undefined + "pri" attribute are considered to have a value of 999999, which is + the lowest priority. + + rfc6249 section 3.1 + */ + mres.priority = DEFAULT_PRI; + if (find_key_value (url_end, val_end, "pri", &pristr)) + { + long pri; + char *end_pristr; + /* Do not care for errno since 0 is error in this case. */ + pri = strtol (pristr, &end_pristr, 10); + if (end_pristr != pristr + strlen (pristr) || + !VALID_PRI_RANGE (pri)) + { + /* This is against the specification, so let's inform the user. */ + logprintf (LOG_NOTQUIET, + _("Invalid pri value. Assuming %d.\n"), + DEFAULT_PRI); + } + else + mres.priority = pri; + xfree (pristr); + } + + switch (url_scheme (urlstr)) + { + case SCHEME_HTTP: + mres.type = xstrdup ("http"); + break; +#ifdef HAVE_SSL + case SCHEME_HTTPS: + mres.type = xstrdup ("https"); + break; + case SCHEME_FTPS: + mres.type = xstrdup ("ftps"); + break; +#endif + case SCHEME_FTP: + mres.type = xstrdup ("ftp"); + break; + default: + DEBUGP (("Unsupported url scheme in %s. Skipping resource.\n", urlstr)); + } + + if (mres.type) + { + DEBUGP (("TYPE=%s\n", mres.type)); + + /* At this point we have validated the new resource. */ + + find_key_value (url_end, val_end, "geo", &mres.location); + + mres.url = urlstr; + urlstr = NULL; + + mres.preference = 0; + if (has_key (url_end, val_end, "pref")) + { + DEBUGP (("This resource has preference\n")); + mres.preference = 1; + } + + /* 1 slot from new resource, 1 slot for null-termination. */ + mfile->resources = xrealloc (mfile->resources, + sizeof (metalink_resource_t *) * (res_count + 2)); + mfile->resources[res_count] = xnew0 (metalink_resource_t); + *mfile->resources[res_count] = mres; + res_count++; + } + } /* Handle resource link (rel=duplicate). */ + + /* Handle Metalink/XML resources. */ + else if (reltype && !strcmp (reltype, "application/metalink4+xml")) + { + metalink_metaurl_t murl = {0}; + char *pristr; + + /* + Valid ranges for the "pri" attribute are from + 1 to 999999. Mirror servers with a lower value of the "pri" + attribute have a higher priority, while mirrors with an undefined + "pri" attribute are considered to have a value of 999999, which is + the lowest priority. + + rfc6249 section 3.1 + */ + murl.priority = DEFAULT_PRI; + if (find_key_value (url_end, val_end, "pri", &pristr)) + { + long pri; + char *end_pristr; + /* Do not care for errno since 0 is error in this case. */ + pri = strtol (pristr, &end_pristr, 10); + if (end_pristr != pristr + strlen (pristr) || + !VALID_PRI_RANGE (pri)) + { + /* This is against the specification, so let's inform the user. */ + logprintf (LOG_NOTQUIET, + _("Invalid pri value. Assuming %d.\n"), + DEFAULT_PRI); + } + else + murl.priority = pri; + xfree (pristr); + } + + murl.mediatype = xstrdup (reltype); + + DEBUGP (("MEDIATYPE=%s\n", murl.mediatype)); + + /* At this point we have validated the new resource. */ + + find_key_value (url_end, val_end, "name", &murl.name); + + murl.url = urlstr; + urlstr = NULL; + + /* 1 slot from new resource, 1 slot for null-termination. */ + mfile->metaurls = xrealloc (mfile->metaurls, + sizeof (metalink_metaurl_t *) * (meta_count + 2)); + mfile->metaurls[meta_count] = xnew0 (metalink_metaurl_t); + *mfile->metaurls[meta_count] = murl; + meta_count++; + } /* Handle resource link (rel=describedby). */ + else + DEBUGP (("This link header was not used for Metalink\n")); + + xfree (urlstr); + xfree (reltype); + xfree (rel); + } /* Iterate over link headers. */ + + /* Null-terminate resources array. */ + mfile->resources[res_count] = 0; + mfile->metaurls[meta_count] = 0; + + if (res_count == 0 && meta_count == 0) + { + DEBUGP (("No valid metalink references found.\n")); + goto fail; + } + + /* Find all Digest headers. */ + for (i = 0; + (i = resp_header_locate (resp, "Digest", i, &val_beg, &val_end)) != -1; + i++) + { + const char *dig_pos; + char *dig_type, *dig_hash; + + /* Each Digest header can include multiple hashes. Example: + Digest: SHA=thvDyvhfIqlvFe+A9MYgxAfm1q5=,unixsum=30637 + Digest: md5=HUXZLQLMuI/KZ5KDcJPcOA== + */ + for (dig_pos = val_beg; + (dig_pos = find_key_values (dig_pos, val_end, &dig_type, &dig_hash)); + dig_pos++) + { + /* The hash here is assumed to be base64. We need the hash in hex. + Therefore we convert: base64 -> binary -> hex. */ + const size_t dig_hash_str_len = strlen (dig_hash); + char bin_hash[256]; + ssize_t hash_bin_len; + + // there is no hash with that size + if (dig_hash_str_len >= sizeof (bin_hash)) + { + DEBUGP (("Hash too long, ignored.\n")); + xfree (dig_type); + xfree (dig_hash); + continue; + } + + hash_bin_len = wget_base64_decode (dig_hash, bin_hash, dig_hash_str_len * 3 / 4 + 1); + + /* Detect malformed base64 input. */ + if (hash_bin_len < 0) + { + DEBUGP (("Malformed base64 input, ignored.\n")); + xfree (dig_type); + xfree (dig_hash); + continue; + } + + /* One slot for me, one for zero-termination. */ + mfile->checksums = + xrealloc (mfile->checksums, + sizeof (metalink_checksum_t *) * (hash_count + 2)); + mfile->checksums[hash_count] = xnew (metalink_checksum_t); + mfile->checksums[hash_count]->type = dig_type; + + mfile->checksums[hash_count]->hash = xmalloc ((size_t)hash_bin_len * 2 + 1); + wg_hex_to_string (mfile->checksums[hash_count]->hash, bin_hash, (size_t)hash_bin_len); + + xfree (dig_hash); + + hash_count++; + } + } + + /* Zero-terminate checksums array. */ + mfile->checksums[hash_count] = 0; + + /* + If Instance Digests are not provided by the Metalink servers, the + Link header fields pertaining to this specification MUST be ignored. + + rfc6249 section 6 + */ + if (res_count && hash_count == 0) + { + logputs (LOG_VERBOSE, + _("Could not find acceptable digest for Metalink resources.\n" + "Ignoring them.\n")); + goto fail; + } + + /* Metalink data is OK. Now we just need to sort the resources based + on their priorities, preference, and perhaps location. */ + stable_sort (mfile->resources, res_count, sizeof (metalink_resource_t *), metalink_res_cmp); + stable_sort (mfile->metaurls, meta_count, sizeof (metalink_metaurl_t *), metalink_meta_cmp); + + /* Restore sensible preference values (in case someone cares to look). */ + for (i = 0; i < res_count; ++i) + mfile->resources[i]->preference = 1000000 - mfile->resources[i]->priority; + + metalink = xnew0 (metalink_t); + metalink->files = xmalloc (sizeof (metalink_file_t *) * 2); + metalink->files[0] = mfile; + metalink->files[1] = 0; + metalink->origin = xstrdup (u->url); + metalink->version = METALINK_VERSION_4; + /* Leave other fields set to 0. */ + + return metalink; + +fail: + /* Free all allocated memory. */ + if (metalink) + metalink_delete (metalink); + else + metalink_file_delete (mfile); + return NULL; +} +#endif /* HAVE_METALINK */ + +/* Retrieve a document through HTTP protocol. It recognizes status + code, and correctly handles redirections. It closes the network + socket. If it receives an error from the functions below it, it + will print it if there is enough information to do so (almost + always), returning the error to the caller (i.e. http_loop). + + Various HTTP parameters are stored to hs. + + If PROXY is non-NULL, the connection will be made to the proxy + server, and u->url will be requested. */ +static uerr_t +gethttp (const struct url *u, struct url *original_url, struct http_stat *hs, + int *dt, struct url *proxy, struct iri *iri, int count) +{ + struct request *req = NULL; + + char *type = NULL; + char *user, *passwd; + char *proxyauth; + int statcode; + int write_error; + wgint contlen, contrange; + const struct url *conn; + FILE *fp; + int err; + uerr_t retval; +#ifdef HAVE_HSTS +#ifdef TESTING + /* we don't link against main.o when we're testing */ + hsts_store_t hsts_store = NULL; +#else + extern hsts_store_t hsts_store; +#endif + const char *hsts_params; + time_t max_age; + bool include_subdomains; +#endif + + int sock = -1; + + /* Set to 1 when the authorization has already been sent and should + not be tried again. */ + bool auth_finished = false; + + /* Set to 1 when just globally-set Basic authorization has been sent; + * should prevent further Basic negotiations, but not other + * mechanisms. */ + bool basic_auth_finished = false; + + /* Whether NTLM authentication is used for this request. */ + bool ntlm_seen = false; + + /* Whether our connection to the remote host is through SSL. */ + bool using_ssl = false; + + /* Whether a HEAD request will be issued (as opposed to GET or + POST). */ + bool head_only = !!(*dt & HEAD_ONLY); + + /* Whether conditional get request will be issued. */ + bool cond_get = !!(*dt & IF_MODIFIED_SINCE); + +#ifdef HAVE_METALINK + /* Are we looking for metalink info in HTTP headers? */ + bool metalink = !!(*dt & METALINK_METADATA); +#endif + + char *head = NULL; + struct response *resp = NULL; + char hdrval[512]; + char *message = NULL; + + /* Declare WARC variables. */ + bool warc_enabled = (opt.warc_filename != NULL); + FILE *warc_tmp = NULL; + char warc_timestamp_str [21]; + char warc_request_uuid [48]; + ip_address warc_ip_buf, *warc_ip = NULL; + off_t warc_payload_offset = -1; + + /* Whether this connection will be kept alive after the HTTP request + is done. */ + bool keep_alive; + + /* Is the server using the chunked transfer encoding? */ + bool chunked_transfer_encoding = false; + + /* Whether keep-alive should be inhibited. */ + bool inhibit_keep_alive = + !opt.http_keep_alive || opt.ignore_length; + + /* Headers sent when using POST. */ + wgint body_data_size = 0; + +#ifdef HAVE_SSL + if (u->scheme == SCHEME_HTTPS) + { + /* Initialize the SSL context. After this has once been done, + it becomes a no-op. */ + if (!ssl_init ()) + { + scheme_disable (SCHEME_HTTPS); + logprintf (LOG_NOTQUIET, + _("Disabling SSL due to encountered errors.\n")); + retval = SSLINITFAILED; + goto cleanup; + } + } +#endif /* HAVE_SSL */ + + /* Initialize certain elements of struct http_stat. + * Since this function is called in a loop, we have to xfree certain + * members. */ + hs->len = 0; + hs->contlen = -1; + hs->res = -1; + xfree (hs->rderrmsg); + xfree (hs->newloc); + xfree (hs->remote_time); + xfree (hs->error); + xfree (hs->message); + hs->local_encoding = ENC_NONE; + hs->remote_encoding = ENC_NONE; + + conn = u; + + { + uerr_t ret; + req = initialize_request (u, hs, dt, proxy, inhibit_keep_alive, + &basic_auth_finished, &body_data_size, + &user, &passwd, &ret); + if (req == NULL) + { + retval = ret; + goto cleanup; + } + } + retry_with_auth: + /* We need to come back here when the initial attempt to retrieve + without authorization header fails. (Expected to happen at least + for the Digest authorization scheme.) */ + + if (opt.cookies) + request_set_header (req, "Cookie", + cookie_header (wget_cookie_jar, + u->host, u->port, u->path, +#ifdef HAVE_SSL + u->scheme == SCHEME_HTTPS +#else + 0 +#endif + ), + rel_value); + + /* Add the user headers. */ + if (opt.user_headers) + { + int i; + for (i = 0; opt.user_headers[i]; i++) + request_set_user_header (req, opt.user_headers[i]); + } + + proxyauth = NULL; + if (proxy) + { + conn = proxy; + initialize_proxy_configuration (u, req, proxy, &proxyauth); + } + keep_alive = true; + + /* Establish the connection. */ + if (inhibit_keep_alive) + keep_alive = false; + + { + uerr_t conn_err = establish_connection (u, &conn, hs, proxy, &proxyauth, &req, + &using_ssl, inhibit_keep_alive, &sock); + if (conn_err != RETROK) + { + retval = conn_err; + goto cleanup; + } + } + + /* Open the temporary file where we will write the request. */ + if (warc_enabled) + { + warc_tmp = warc_tempfile (); + if (warc_tmp == NULL) + { + CLOSE_INVALIDATE (sock); + retval = WARC_TMP_FOPENERR; + goto cleanup; + } + + if (! proxy) + { + warc_ip = &warc_ip_buf; + socket_ip_address (sock, warc_ip, ENDPOINT_PEER); + } + } + + /* Send the request to server. */ + write_error = request_send (req, sock, warc_tmp); + + if (write_error >= 0) + { + if (opt.body_data) + { + DEBUGP (("[BODY data: %s]\n", opt.body_data)); + write_error = fd_write (sock, opt.body_data, body_data_size, -1); + if (write_error >= 0 && warc_tmp != NULL) + { + int warc_tmp_written; + + /* Remember end of headers / start of payload. */ + warc_payload_offset = ftello (warc_tmp); + + /* Write a copy of the data to the WARC record. */ + warc_tmp_written = fwrite (opt.body_data, 1, body_data_size, warc_tmp); + if (warc_tmp_written != body_data_size) + write_error = -2; + } + } + else if (opt.body_file && body_data_size != 0) + { + if (warc_tmp != NULL) + /* Remember end of headers / start of payload */ + warc_payload_offset = ftello (warc_tmp); + + write_error = body_file_send (sock, opt.body_file, body_data_size, warc_tmp); + } + } + + if (write_error < 0) + { + CLOSE_INVALIDATE (sock); + + if (warc_tmp != NULL) + fclose (warc_tmp); + + if (write_error == -2) + retval = WARC_TMP_FWRITEERR; + else + retval = WRITEFAILED; + goto cleanup; + } + logprintf (LOG_VERBOSE, _("%s request sent, awaiting response... "), + proxy ? "Proxy" : "HTTP"); + contlen = -1; + contrange = 0; + *dt &= ~RETROKF; + + + if (warc_enabled) + { + bool warc_result; + + /* Generate a timestamp and uuid for this request. */ + warc_timestamp (warc_timestamp_str, sizeof (warc_timestamp_str)); + warc_uuid_str (warc_request_uuid, sizeof (warc_request_uuid)); + + /* Create a request record and store it in the WARC file. */ + warc_result = warc_write_request_record (u->url, warc_timestamp_str, + warc_request_uuid, warc_ip, + warc_tmp, warc_payload_offset); + if (! warc_result) + { + CLOSE_INVALIDATE (sock); + retval = WARC_ERR; + goto cleanup; + } + + /* warc_write_request_record has also closed warc_tmp. */ + } + + /* Repeat while we receive a 10x response code. */ + { + bool _repeat; + + do + { + head = read_http_response_head (sock); + if (!head) + { + if (errno == 0) + { + logputs (LOG_NOTQUIET, _("No data received.\n")); + CLOSE_INVALIDATE (sock); + retval = HEOF; + } + else + { + logprintf (LOG_NOTQUIET, _("Read error (%s) in headers.\n"), + fd_errstr (sock)); + CLOSE_INVALIDATE (sock); + retval = HERR; + } + goto cleanup; + } + DEBUGP (("\n---response begin---\n%s---response end---\n", head)); + + resp = resp_new (head); + + /* Check for status line. */ + xfree (message); + statcode = resp_status (resp, &message); + if (statcode < 0) + { + char *tms = datetime_str (time (NULL)); + logprintf (LOG_VERBOSE, "%d\n", statcode); + logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"), tms, statcode, + quotearg_style (escape_quoting_style, + _("Malformed status line"))); + CLOSE_INVALIDATE (sock); + retval = HERR; + goto cleanup; + } + + if (H_10X (statcode)) + { + xfree (head); + resp_free (&resp); + _repeat = true; + DEBUGP (("Ignoring response\n")); + } + else + { + _repeat = false; + } + } + while (_repeat); + } + + xfree (hs->message); + hs->message = xstrdup (message); + if (!opt.server_response) + logprintf (LOG_VERBOSE, "%2d %s\n", statcode, + message ? quotearg_style (escape_quoting_style, message) : ""); + else + { + logprintf (LOG_VERBOSE, "\n"); + print_server_response (resp, " "); + } + + if (!opt.ignore_length + && resp_header_copy (resp, "Content-Length", hdrval, sizeof (hdrval))) + { + wgint parsed; + errno = 0; + parsed = str_to_wgint (hdrval, NULL, 10); + if (parsed == WGINT_MAX && errno == ERANGE) + { + /* Out of range. + #### If Content-Length is out of range, it most likely + means that the file is larger than 2G and that we're + compiled without LFS. In that case we should probably + refuse to even attempt to download the file. */ + contlen = -1; + } + else if (parsed < 0) + { + /* Negative Content-Length; nonsensical, so we can't + assume any information about the content to receive. */ + contlen = -1; + } + else + contlen = parsed; + } + + /* Check for keep-alive related responses. */ + if (!inhibit_keep_alive) + { + if (resp_header_copy (resp, "Connection", hdrval, sizeof (hdrval))) + { + if (0 == c_strcasecmp (hdrval, "Close")) + keep_alive = false; + } + } + + chunked_transfer_encoding = false; + if (resp_header_copy (resp, "Transfer-Encoding", hdrval, sizeof (hdrval)) + && 0 == c_strcasecmp (hdrval, "chunked")) + chunked_transfer_encoding = true; + + /* Handle (possibly multiple instances of) the Set-Cookie header. */ + if (opt.cookies) + { + int scpos; + const char *scbeg, *scend; + /* The jar should have been created by now. */ + assert (wget_cookie_jar != NULL); + for (scpos = 0; + (scpos = resp_header_locate (resp, "Set-Cookie", scpos, + &scbeg, &scend)) != -1; + ++scpos) + { + char buf[1024], *set_cookie; + size_t len = scend - scbeg; + + if (len < sizeof (buf)) + set_cookie = buf; + else + set_cookie = xmalloc (len + 1); + + memcpy (set_cookie, scbeg, len); + set_cookie[len] = 0; + + cookie_handle_set_cookie (wget_cookie_jar, u->host, u->port, + u->path, set_cookie); + + if (set_cookie != buf) + xfree (set_cookie); + } + } + + if (keep_alive) + /* The server has promised that it will not close the connection + when we're done. This means that we can register it. */ + register_persistent (conn->host, conn->port, sock, using_ssl); + +#ifdef HAVE_METALINK + /* We need to check for the Metalink data in the very first response + we get from the server (before redirections, authorization, etc.). */ + if (metalink) + { + hs->metalink = metalink_from_http (resp, hs, u); + /* Bugfix: hs->local_file is NULL (opt.content_disposition). */ + if (!hs->local_file && hs->metalink && hs->metalink->origin) + hs->local_file = xstrdup (hs->metalink->origin); + xfree (hs->message); + retval = RETR_WITH_METALINK; + CLOSE_FINISH (sock); + goto cleanup; + } +#endif + + if (statcode == HTTP_STATUS_UNAUTHORIZED) + { + /* Authorization is required. */ + uerr_t auth_err = RETROK; + bool retry; + /* Normally we are not interested in the response body. + But if we are writing a WARC file we are: we like to keep everything. */ + if (warc_enabled) + { + int _err; + type = resp_header_strdup (resp, "Content-Type"); + _err = read_response_body (hs, sock, NULL, contlen, 0, + chunked_transfer_encoding, + u->url, warc_timestamp_str, + warc_request_uuid, warc_ip, type, + statcode, head); + xfree (type); + + if (_err != RETRFINISHED || hs->res < 0) + { + CLOSE_INVALIDATE (sock); + retval = _err; + goto cleanup; + } + else + CLOSE_FINISH (sock); + } + else + { + /* Since WARC is disabled, we are not interested in the response body. */ + if (keep_alive && !head_only + && skip_short_body (sock, contlen, chunked_transfer_encoding)) + CLOSE_FINISH (sock); + else + CLOSE_INVALIDATE (sock); + } + + pconn.authorized = false; + + { + auth_err = check_auth (u, user, passwd, resp, req, + &ntlm_seen, &retry, + &basic_auth_finished, + &auth_finished); + if (auth_err == RETROK && retry) + { + resp_free (&resp); + xfree (message); + xfree (head); + goto retry_with_auth; + } + } + if (auth_err == RETROK) + retval = AUTHFAILED; + else + retval = auth_err; + goto cleanup; + } + else /* statcode != HTTP_STATUS_UNAUTHORIZED */ + { + /* Kludge: if NTLM is used, mark the TCP connection as authorized. */ + if (ntlm_seen) + pconn.authorized = true; + } + + { + uerr_t ret = check_file_output (u, hs, resp, hdrval, sizeof hdrval); + if (ret != RETROK) + { + retval = ret; + goto cleanup; + } + } + + hs->statcode = statcode; + xfree (hs->error); + if (statcode == -1) + hs->error = xstrdup (_("Malformed status line")); + else if (!*message) + hs->error = xstrdup (_("(no description)")); + else + hs->error = xstrdup (message); + +#ifdef HAVE_HSTS + if (opt.hsts && hsts_store) + { + hsts_params = resp_header_strdup (resp, "Strict-Transport-Security"); + if (parse_strict_transport_security (hsts_params, &max_age, &include_subdomains)) + { + /* process strict transport security */ + if (hsts_store_entry (hsts_store, u->scheme, u->host, u->port, max_age, include_subdomains)) + DEBUGP(("Added new HSTS host: %s:%u (max-age: %lu, includeSubdomains: %s)\n", + u->host, + (unsigned) u->port, + (unsigned long) max_age, + (include_subdomains ? "true" : "false"))); + else + DEBUGP(("Updated HSTS host: %s:%u (max-age: %lu, includeSubdomains: %s)\n", + u->host, + (unsigned) u->port, + (unsigned long) max_age, + (include_subdomains ? "true" : "false"))); + } + xfree (hsts_params); + } +#endif + + type = resp_header_strdup (resp, "Content-Type"); + if (type) + { + char *tmp = strchr (type, ';'); + if (tmp) + { +#ifdef ENABLE_IRI + /* sXXXav: only needed if IRI support is enabled */ + char *tmp2 = tmp + 1; +#endif + + while (tmp > type && c_isspace (tmp[-1])) + --tmp; + *tmp = '\0'; + +#ifdef ENABLE_IRI + /* Try to get remote encoding if needed */ + if (opt.enable_iri && !opt.encoding_remote) + { + tmp = parse_charset (tmp2); + if (tmp) + set_content_encoding (iri, tmp); + xfree (tmp); + } +#endif + } + } + xfree (hs->newloc); + hs->newloc = resp_header_strdup (resp, "Location"); + xfree (hs->remote_time); + hs->remote_time = resp_header_strdup (resp, "Last-Modified"); + if (!hs->remote_time) // now look for the Wayback Machine's timestamp + hs->remote_time = resp_header_strdup (resp, "X-Archive-Orig-last-modified"); + + if (resp_header_copy (resp, "Content-Range", hdrval, sizeof (hdrval))) + { + wgint first_byte_pos, last_byte_pos, entity_length; + if (parse_content_range (hdrval, &first_byte_pos, &last_byte_pos, + &entity_length)) + { + contrange = first_byte_pos; + contlen = last_byte_pos - first_byte_pos + 1; + } + } + + if (resp_header_copy (resp, "Content-Encoding", hdrval, sizeof (hdrval))) + { + hs->local_encoding = ENC_INVALID; + + switch (hdrval[0]) + { + case 'b': case 'B': + if (0 == c_strcasecmp(hdrval, "br")) + hs->local_encoding = ENC_BROTLI; + break; + case 'c': case 'C': + if (0 == c_strcasecmp(hdrval, "compress")) + hs->local_encoding = ENC_COMPRESS; + break; + case 'd': case 'D': + if (0 == c_strcasecmp(hdrval, "deflate")) + hs->local_encoding = ENC_DEFLATE; + break; + case 'g': case 'G': + if (0 == c_strcasecmp(hdrval, "gzip")) + hs->local_encoding = ENC_GZIP; + break; + case 'i': case 'I': + if (0 == c_strcasecmp(hdrval, "identity")) + hs->local_encoding = ENC_NONE; + break; + case 'x': case 'X': + if (0 == c_strcasecmp(hdrval, "x-compress")) + hs->local_encoding = ENC_COMPRESS; + else if (0 == c_strcasecmp(hdrval, "x-gzip")) + hs->local_encoding = ENC_GZIP; + break; + case '\0': + hs->local_encoding = ENC_NONE; + } + + if (hs->local_encoding == ENC_INVALID) + { + DEBUGP (("Unrecognized Content-Encoding: %s\n", hdrval)); + hs->local_encoding = ENC_NONE; + } +#ifdef HAVE_LIBZ + else if (hs->local_encoding == ENC_GZIP + && opt.compression != compression_none) + { + const char *p; + + /* Make sure the Content-Type is not gzip before decompressing */ + if (type) + { + p = strchr (type, '/'); + if (p == NULL) + { + hs->remote_encoding = ENC_GZIP; + hs->local_encoding = ENC_NONE; + } + else + { + p++; + if (c_tolower(p[0]) == 'x' && p[1] == '-') + p += 2; + if (0 != c_strcasecmp (p, "gzip")) + { + hs->remote_encoding = ENC_GZIP; + hs->local_encoding = ENC_NONE; + } + } + } + else + { + hs->remote_encoding = ENC_GZIP; + hs->local_encoding = ENC_NONE; + } + + /* don't uncompress if a file ends with '.gz' or '.tgz' */ + if (hs->remote_encoding == ENC_GZIP + && (p = strrchr(u->file, '.')) + && (c_strcasecmp(p, ".gz") == 0 || c_strcasecmp(p, ".tgz") == 0)) + { + DEBUGP (("Enabling broken server workaround. Will not decompress this GZip file.\n")); + hs->remote_encoding = ENC_NONE; + } + } +#endif + } + + /* 20x responses are counted among successful by default. */ + if (H_20X (statcode)) + *dt |= RETROKF; + + if (statcode == HTTP_STATUS_NO_CONTENT) + { + /* 204 response has no body (RFC 2616, 4.3) */ + + /* In case the caller cares to look... */ + hs->len = 0; + hs->res = 0; + hs->restval = 0; + + CLOSE_FINISH (sock); + + retval = RETRFINISHED; + goto cleanup; + } + + /* Return if redirected. */ + if (H_REDIRECTED (statcode) || statcode == HTTP_STATUS_MULTIPLE_CHOICES) + { + /* RFC2068 says that in case of the 300 (multiple choices) + response, the server can output a preferred URL through + `Location' header; otherwise, the request should be treated + like GET. So, if the location is set, it will be a + redirection; otherwise, just proceed normally. */ + if (statcode == HTTP_STATUS_MULTIPLE_CHOICES && !hs->newloc) + *dt |= RETROKF; + else + { + logprintf (LOG_VERBOSE, + _("Location: %s%s\n"), + hs->newloc ? escnonprint_uri (hs->newloc) : _("unspecified"), + hs->newloc ? _(" [following]") : ""); + + /* In case the caller cares to look... */ + hs->len = 0; + hs->res = 0; + hs->restval = 0; + + /* Normally we are not interested in the response body of a redirect. + But if we are writing a WARC file we are: we like to keep everything. */ + if (warc_enabled) + { + int _err = read_response_body (hs, sock, NULL, contlen, 0, + chunked_transfer_encoding, + u->url, warc_timestamp_str, + warc_request_uuid, warc_ip, type, + statcode, head); + + if (_err != RETRFINISHED || hs->res < 0) + { + CLOSE_INVALIDATE (sock); + retval = _err; + goto cleanup; + } + else + CLOSE_FINISH (sock); + } + else + { + /* Since WARC is disabled, we are not interested in the response body. */ + if (keep_alive && !head_only + && skip_short_body (sock, contlen, chunked_transfer_encoding)) + CLOSE_FINISH (sock); + else + CLOSE_INVALIDATE (sock); + } + + /* From RFC2616: The status codes 303 and 307 have + been added for servers that wish to make unambiguously + clear which kind of reaction is expected of the client. + + A 307 should be redirected using the same method, + in other words, a POST should be preserved and not + converted to a GET in that case. + + With strict adherence to RFC2616, POST requests are not + converted to a GET request on 301 Permanent Redirect + or 302 Temporary Redirect. + + A switch may be provided later based on the HTTPbis draft + that allows clients to convert POST requests to GET + requests on 301 and 302 response codes. */ + switch (statcode) + { + case HTTP_STATUS_TEMPORARY_REDIRECT: + case HTTP_STATUS_PERMANENT_REDIRECT: + retval = NEWLOCATION_KEEP_POST; + goto cleanup; + case HTTP_STATUS_MOVED_PERMANENTLY: + if (opt.method && c_strcasecmp (opt.method, "post") != 0) + { + retval = NEWLOCATION_KEEP_POST; + goto cleanup; + } + break; + case HTTP_STATUS_MOVED_TEMPORARILY: + if (opt.method && c_strcasecmp (opt.method, "post") != 0) + { + retval = NEWLOCATION_KEEP_POST; + goto cleanup; + } + break; + } + retval = NEWLOCATION; + goto cleanup; + } + } + + if (cond_get) + { + if (statcode == HTTP_STATUS_NOT_MODIFIED) + { + logprintf (LOG_VERBOSE, + _ ("File %s not modified on server. Omitting download.\n\n"), + quote (hs->local_file)); + *dt |= RETROKF; + CLOSE_FINISH (sock); + retval = RETRUNNEEDED; + goto cleanup; + } + } + + set_content_type (dt, type); + + if (opt.adjust_extension) + { + const char *encoding_ext = NULL; + switch (hs->local_encoding) + { + case ENC_INVALID: + case ENC_NONE: + break; + case ENC_BROTLI: + encoding_ext = ".br"; + break; + case ENC_COMPRESS: + encoding_ext = ".Z"; + break; + case ENC_DEFLATE: + encoding_ext = ".zlib"; + break; + case ENC_GZIP: + encoding_ext = ".gz"; + break; + default: + DEBUGP (("No extension found for encoding %d\n", + hs->local_encoding)); + } + if (encoding_ext != NULL) + { + char *file_ext = strrchr (hs->local_file, '.'); + /* strip Content-Encoding extension (it will be re-added later) */ + if (file_ext != NULL && 0 == strcasecmp (file_ext, encoding_ext)) + *file_ext = '\0'; + } + if (*dt & TEXTHTML) + /* -E / --adjust-extension / adjust_extension = on was specified, + and this is a text/html file. If some case-insensitive + variation on ".htm[l]" isn't already the file's suffix, + tack on ".html". */ + { + ensure_extension (hs, ".html", dt); + } + else if (*dt & TEXTCSS) + { + ensure_extension (hs, ".css", dt); + } + if (encoding_ext != NULL) + { + ensure_extension (hs, encoding_ext, dt); + } + } + + if (cond_get) + { + /* Handle the case when server ignores If-Modified-Since header. */ + if (statcode == HTTP_STATUS_OK && hs->remote_time) + { + time_t tmr = http_atotm (hs->remote_time); + + /* Check if the local file is up-to-date based on Last-Modified header + and content length. */ + if (tmr != (time_t) - 1 && tmr <= hs->orig_file_tstamp + && (contlen == -1 || contlen == hs->orig_file_size)) + { + logprintf (LOG_VERBOSE, + _("Server ignored If-Modified-Since header for file %s.\n" + "You might want to add --no-if-modified-since option." + "\n\n"), + quote (hs->local_file)); + *dt |= RETROKF; + CLOSE_INVALIDATE (sock); + retval = RETRUNNEEDED; + goto cleanup; + } + } + } + + if (statcode == HTTP_STATUS_RANGE_NOT_SATISFIABLE + || (!opt.timestamping && hs->restval > 0 && statcode == HTTP_STATUS_OK + && contrange == 0 && contlen >= 0 && hs->restval >= contlen)) + { + /* If `-c' is in use and the file has been fully downloaded (or + the remote file has shrunk), Wget effectively requests bytes + after the end of file and the server response with 416 + (or 200 with a <= Content-Length. */ + logputs (LOG_VERBOSE, _("\ +\n The file is already fully retrieved; nothing to do.\n\n")); + /* In case the caller inspects. */ + hs->len = contlen; + hs->res = 0; + /* Mark as successfully retrieved. */ + *dt |= RETROKF; + + /* Try to maintain the keep-alive connection. It is often cheaper to + * consume some bytes which have already been sent than to negotiate + * a new connection. However, if the body is too large, or we don't + * care about keep-alive, then simply terminate the connection */ + if (keep_alive && + skip_short_body (sock, contlen, chunked_transfer_encoding)) + CLOSE_FINISH (sock); + else + CLOSE_INVALIDATE (sock); + retval = RETRUNNEEDED; + goto cleanup; + } + if ((contrange != 0 && contrange != hs->restval) + || (H_PARTIAL (statcode) && !contrange && hs->restval)) + { + /* The Range request was somehow misunderstood by the server. + Bail out. */ + CLOSE_INVALIDATE (sock); + retval = RANGEERR; + goto cleanup; + } + if (contlen == -1) + hs->contlen = -1; + /* If the response is gzipped, the uncompressed size is unknown. */ + else if (hs->remote_encoding == ENC_GZIP) + hs->contlen = -1; + else + hs->contlen = contlen + contrange; + + if (opt.verbose) + { + if (*dt & RETROKF) + { + /* No need to print this output if the body won't be + downloaded at all, or if the original server response is + printed. */ + logputs (LOG_VERBOSE, _("Length: ")); + if (contlen != -1) + { + logputs (LOG_VERBOSE, number_to_static_string (contlen + contrange)); + if (contlen + contrange >= 1024) + logprintf (LOG_VERBOSE, " (%s)", + human_readable (contlen + contrange, 10, 1)); + if (contrange) + { + if (contlen >= 1024) + logprintf (LOG_VERBOSE, _(", %s (%s) remaining"), + number_to_static_string (contlen), + human_readable (contlen, 10, 1)); + else + logprintf (LOG_VERBOSE, _(", %s remaining"), + number_to_static_string (contlen)); + } + } + else + logputs (LOG_VERBOSE, + opt.ignore_length ? _("ignored") : _("unspecified")); + if (type) + logprintf (LOG_VERBOSE, " [%s]\n", quotearg_style (escape_quoting_style, type)); + else + logputs (LOG_VERBOSE, "\n"); + } + } + + /* Return if we have no intention of further downloading. */ + if ((!(*dt & RETROKF) && !opt.content_on_error) || head_only || (opt.spider && !opt.recursive)) + { + /* In case the caller cares to look... */ + hs->len = 0; + hs->res = 0; + hs->restval = 0; + + /* Normally we are not interested in the response body of a error responses. + But if we are writing a WARC file we are: we like to keep everything. */ + if (warc_enabled) + { + int _err = read_response_body (hs, sock, NULL, contlen, 0, + chunked_transfer_encoding, + u->url, warc_timestamp_str, + warc_request_uuid, warc_ip, type, + statcode, head); + + if (_err != RETRFINISHED || hs->res < 0) + { + CLOSE_INVALIDATE (sock); + retval = _err; + goto cleanup; + } + + CLOSE_FINISH (sock); + } + else + { + /* Since WARC is disabled, we are not interested in the response body. */ + if (head_only) + /* Pre-1.10 Wget used CLOSE_INVALIDATE here. Now we trust the + servers not to send body in response to a HEAD request, and + those that do will likely be caught by test_socket_open. + If not, they can be worked around using + `--no-http-keep-alive'. */ + CLOSE_FINISH (sock); + else if (opt.spider && !opt.recursive) + /* we just want to see if the page exists - no downloading required */ + CLOSE_INVALIDATE (sock); + else if (keep_alive + && skip_short_body (sock, contlen, chunked_transfer_encoding)) + /* Successfully skipped the body; also keep using the socket. */ + CLOSE_FINISH (sock); + else + CLOSE_INVALIDATE (sock); + } + + if (statcode == HTTP_STATUS_GATEWAY_TIMEOUT) + retval = GATEWAYTIMEOUT; + else + retval = RETRFINISHED; + + goto cleanup; + } + + err = open_output_stream (hs, count, &fp); + if (err != RETROK) + { + CLOSE_INVALIDATE (sock); + retval = err; + goto cleanup; + } + +#ifdef ENABLE_XATTR + if (opt.enable_xattr) + { + if (original_url != u) + set_file_metadata (u, original_url, fp); + else + set_file_metadata (u, NULL, fp); + } +#endif + + err = read_response_body (hs, sock, fp, contlen, contrange, + chunked_transfer_encoding, + u->url, warc_timestamp_str, + warc_request_uuid, warc_ip, type, + statcode, head); + + if (hs->res >= 0) + CLOSE_FINISH (sock); + else + CLOSE_INVALIDATE (sock); + + if (!output_stream) + fclose (fp); + + retval = err; + + cleanup: + xfree (head); + xfree (type); + xfree (message); + resp_free (&resp); + request_free (&req); + + return retval; +} + +/* Check whether the supplied HTTP status code is among those + listed for the --retry-on-http-error option. */ +static bool +check_retry_on_http_error (const int statcode) +{ + const char *tok = opt.retry_on_http_error; + while (tok && *tok) + { + if (atoi (tok) == statcode) + return true; + if ((tok = strchr (tok, ','))) + ++tok; + } + return false; +} + +/* The genuine HTTP loop! This is the part where the retrieval is + retried, and retried, and retried, and... */ +uerr_t +http_loop (const struct url *u, struct url *original_url, char **newloc, + char **local_file, const char *referer, int *dt, struct url *proxy, + struct iri *iri) +{ + int count; + bool got_head = false; /* used for time-stamping and filename detection */ + bool time_came_from_head = false; + bool got_name = false; + char *tms; + const char *tmrate; + uerr_t err, ret = TRYLIMEXC; + time_t tmr = -1; /* remote time-stamp */ + struct http_stat hstat; /* HTTP status */ + struct stat st; + bool send_head_first = true; + bool force_full_retrieve = false; + + + /* If we are writing to a WARC file: always retrieve the whole file. */ + if (opt.warc_filename != NULL) + force_full_retrieve = true; + + + /* Assert that no value for *LOCAL_FILE was passed. */ + assert (local_file == NULL || *local_file == NULL); + + /* Set LOCAL_FILE parameter. */ + if (local_file && opt.output_document) + *local_file = HYPHENP (opt.output_document) ? NULL : xstrdup (opt.output_document); + + /* Reset NEWLOC parameter. */ + *newloc = NULL; + + /* This used to be done in main, but it's a better idea to do it + here so that we don't go through the hoops if we're just using + FTP or whatever. */ + if (opt.cookies) + load_cookies (); + + /* Warn on (likely bogus) wildcard usage in HTTP. */ + if (opt.ftp_glob && has_wildcards_p (u->path)) + logputs (LOG_VERBOSE, _("Warning: wildcards not supported in HTTP.\n")); + + /* Setup hstat struct. */ + xzero (hstat); + hstat.referer = referer; + + if (opt.output_document) + { + hstat.local_file = xstrdup (opt.output_document); + got_name = true; + } + else if (!opt.content_disposition) + { + hstat.local_file = + url_file_name (opt.trustservernames ? u : original_url, NULL); + got_name = true; + } + + if (got_name && file_exists_p (hstat.local_file, NULL) && opt.noclobber && !opt.output_document) + { + /* If opt.noclobber is turned on and file already exists, do not + retrieve the file. But if the output_document was given, then this + test was already done and the file didn't exist. Hence the !opt.output_document */ + get_file_flags (hstat.local_file, dt); + ret = RETROK; + goto exit; + } + + /* Reset the counter. */ + count = 0; + + /* Reset the document type. */ + *dt = 0; + + /* Skip preliminary HEAD request if we're not in spider mode. */ + if (!opt.spider) + send_head_first = false; + + /* Send preliminary HEAD request if --content-disposition and -c are used + together. */ + if (opt.content_disposition && opt.always_rest) + send_head_first = true; + +#ifdef HAVE_METALINK + if (opt.metalink_over_http) + { + *dt |= METALINK_METADATA; + send_head_first = true; + } +#endif + + if (opt.timestamping) + { + /* Use conditional get request if requested + * and if timestamp is known at this moment. */ + if (opt.if_modified_since && !send_head_first && got_name && file_exists_p (hstat.local_file, NULL)) + { + *dt |= IF_MODIFIED_SINCE; + { + uerr_t timestamp_err = set_file_timestamp (&hstat); + if (timestamp_err != RETROK) + return timestamp_err; + } + } + /* Send preliminary HEAD request if -N is given and we have existing + * destination file or content disposition is enabled. */ + else if (opt.content_disposition || file_exists_p (hstat.local_file, NULL)) + send_head_first = true; + } + + /* THE loop */ + do + { + /* Increment the pass counter. */ + ++count; + sleep_between_retrievals (count); + + /* Get the current time string. */ + tms = datetime_str (time (NULL)); + + if (opt.spider && !got_head) + logprintf (LOG_VERBOSE, + _("Spider mode enabled. Check if remote file exists.\n")); + + /* Print fetch message, if opt.verbose. */ + if (opt.verbose) + { + char *hurl = url_string (u, URL_AUTH_HIDE_PASSWD); + + if (count > 1) + { + char tmp[256]; + sprintf (tmp, _("(try:%2d)"), count); + logprintf (LOG_NOTQUIET, "--%s-- %s %s\n", + tms, tmp, hurl); + } + else + { + logprintf (LOG_NOTQUIET, "--%s-- %s\n", + tms, hurl); + } + +#ifdef WINDOWS + ws_changetitle (hurl); +#endif + xfree (hurl); + } + + /* Default document type is empty. However, if spider mode is + on or time-stamping is employed, HEAD_ONLY commands is + encoded within *dt. */ + if (send_head_first && !got_head) + *dt |= HEAD_ONLY; + else + *dt &= ~HEAD_ONLY; + + /* Decide whether or not to restart. */ + if (force_full_retrieve) + hstat.restval = hstat.len; + else if (opt.start_pos >= 0) + hstat.restval = opt.start_pos; + else if (opt.always_rest + && got_name + && stat (hstat.local_file, &st) == 0 + && S_ISREG (st.st_mode)) + /* When -c is used, continue from on-disk size. (Can't use + hstat.len even if count>1 because we don't want a failed + first attempt to clobber existing data.) */ + hstat.restval = st.st_size; + else if (count > 1) + { + /* otherwise, continue where the previous try left off */ + if (hstat.len < hstat.restval) + hstat.restval -= hstat.len; + else + hstat.restval = hstat.len; + } + else + hstat.restval = 0; + + /* Decide whether to send the no-cache directive. We send it in + two cases: + a) we're using a proxy, and we're past our first retrieval. + Some proxies are notorious for caching incomplete data, so + we require a fresh get. + b) caching is explicitly inhibited. */ + if ((proxy && count > 1) /* a */ + || !opt.allow_cache) /* b */ + *dt |= SEND_NOCACHE; + else + *dt &= ~SEND_NOCACHE; + + /* Try fetching the document, or at least its head. */ + err = gethttp (u, original_url, &hstat, dt, proxy, iri, count); + + /* Time? */ + tms = datetime_str (time (NULL)); + + /* Get the new location (with or without the redirection). */ + if (hstat.newloc) + *newloc = xstrdup (hstat.newloc); + + switch (err) + { + case HERR: case HEOF: case CONSOCKERR: + case CONERROR: case READERR: case WRITEFAILED: + case RANGEERR: case FOPEN_EXCL_ERR: case GATEWAYTIMEOUT: + /* Non-fatal errors continue executing the loop, which will + bring them to "while" statement at the end, to judge + whether the number of tries was exceeded. */ + printwhat (count, opt.ntry); + continue; + case FWRITEERR: case FOPENERR: + /* Another fatal error. */ + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("Cannot write to %s (%s).\n"), + quote (hstat.local_file), strerror (errno)); + ret = err; + goto exit; + case HOSTERR: + /* Fatal unless option set otherwise. */ + if ( opt.retry_on_host_error ) + { + printwhat (count, opt.ntry); + continue; + } + ret = err; + goto exit; + case CONIMPOSSIBLE: case PROXERR: case SSLINITFAILED: + case CONTNOTSUPPORTED: case VERIFCERTERR: case FILEBADFILE: + case UNKNOWNATTR: + /* Fatal errors just return from the function. */ + ret = err; + goto exit; + case ATTRMISSING: + /* A missing attribute in a Header is a fatal Protocol error. */ + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("Required attribute missing from Header received.\n")); + ret = err; + goto exit; + case AUTHFAILED: + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("Username/Password Authentication Failed.\n")); + ret = err; + goto exit; + case WARC_ERR: + /* A fatal WARC error. */ + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("Cannot write to WARC file.\n")); + ret = err; + goto exit; + case WARC_TMP_FOPENERR: case WARC_TMP_FWRITEERR: + /* A fatal WARC error. */ + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("Cannot write to temporary WARC file.\n")); + ret = err; + goto exit; + case CONSSLERR: + /* Another fatal error. */ + logprintf (LOG_NOTQUIET, _("Unable to establish SSL connection.\n")); + ret = err; + goto exit; + case UNLINKERR: + /* Another fatal error. */ + logputs (LOG_VERBOSE, "\n"); + logprintf (LOG_NOTQUIET, _("Cannot unlink %s (%s).\n"), + quote (hstat.local_file), strerror (errno)); + ret = err; + goto exit; + case NEWLOCATION: + case NEWLOCATION_KEEP_POST: + /* Return the new location to the caller. */ + if (!*newloc) + { + logprintf (LOG_NOTQUIET, + _("ERROR: Redirection (%d) without location.\n"), + hstat.statcode); + ret = WRONGCODE; + } + else + { + ret = err; + } + goto exit; + case RETRUNNEEDED: + /* The file was already fully retrieved. */ + ret = RETROK; + goto exit; + case RETRFINISHED: + /* Deal with you later. */ + break; +#ifdef HAVE_METALINK + case RETR_WITH_METALINK: + { + if (hstat.metalink == NULL) + { + logputs (LOG_NOTQUIET, + _("Could not find Metalink data in HTTP response. " + "Downloading file using HTTP GET.\n")); + *dt &= ~METALINK_METADATA; + *dt &= ~HEAD_ONLY; + got_head = true; + continue; + } + + logputs (LOG_VERBOSE, + _("Metalink headers found. " + "Switching to Metalink mode.\n")); + + ret = retrieve_from_metalink (hstat.metalink); + goto exit; + } + break; +#endif + default: + /* All possibilities should have been exhausted. */ + abort (); + } + + if (!(*dt & RETROKF)) + { + char *hurl = NULL; + if (!opt.verbose) + { + /* #### Ugly ugly ugly! */ + hurl = url_string (u, URL_AUTH_HIDE_PASSWD); + logprintf (LOG_NONVERBOSE, "%s:\n", hurl); + } + + /* Fall back to GET if HEAD fails with a 500 or 501 error code. */ + if (*dt & HEAD_ONLY + && (hstat.statcode == 500 || hstat.statcode == 501)) + { + got_head = true; + xfree (hurl); + continue; + } + /* Maybe we should always keep track of broken links, not just in + * spider mode. + * Don't log error if it was UTF-8 encoded because we will try + * once unencoded. */ + else if (opt.spider && !iri->utf8_encode) + { + /* #### Again: ugly ugly ugly! */ + if (!hurl) + hurl = url_string (u, URL_AUTH_HIDE_PASSWD); + nonexisting_url (hurl); + logprintf (LOG_NOTQUIET, _("\ +Remote file does not exist -- broken link!!!\n")); + } + else if (check_retry_on_http_error (hstat.statcode)) + { + printwhat (count, opt.ntry); + xfree (hurl); + continue; + } + else + { + logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"), + tms, hstat.statcode, + quotearg_style (escape_quoting_style, hstat.error)); + } + logputs (LOG_VERBOSE, "\n"); + ret = WRONGCODE; + xfree (hurl); + goto exit; + } + + /* Did we get the time-stamp? */ + if (!got_head || (opt.spider && !opt.recursive)) + { + got_head = true; /* no more time-stamping */ + + if (opt.timestamping && !hstat.remote_time) + { + logputs (LOG_NOTQUIET, _("\ +Last-modified header missing -- time-stamps turned off.\n")); + } + else if (hstat.remote_time) + { + /* Convert the date-string into struct tm. */ + tmr = http_atotm (hstat.remote_time); + if (tmr == (time_t) (-1)) + logputs (LOG_VERBOSE, _("\ +Last-modified header invalid -- time-stamp ignored.\n")); + if (*dt & HEAD_ONLY) + time_came_from_head = true; + } + + if (send_head_first) + { + /* The time-stamping section. */ + if (opt.timestamping) + { + if (hstat.orig_file_name) /* Perform the following + checks only if the file + we're supposed to + download already exists. */ + { + if (hstat.remote_time && + tmr != (time_t) (-1)) + { + /* Now time-stamping can be used validly. + Time-stamping means that if the sizes of + the local and remote file match, and local + file is newer than the remote file, it will + not be retrieved. Otherwise, the normal + download procedure is resumed. */ + if (hstat.orig_file_tstamp >= tmr) + { + if (hstat.contlen == -1 + || hstat.orig_file_size == hstat.contlen) + { + logprintf (LOG_VERBOSE, _("\ +Server file no newer than local file %s -- not retrieving.\n\n"), + quote (hstat.orig_file_name)); + ret = RETROK; + goto exit; + } + else + { + logprintf (LOG_VERBOSE, _("\ +The sizes do not match (local %s) -- retrieving.\n"), + number_to_static_string (hstat.orig_file_size)); + } + } + else + { + force_full_retrieve = true; + logputs (LOG_VERBOSE, + _("Remote file is newer, retrieving.\n")); + } + + logputs (LOG_VERBOSE, "\n"); + } + } + + /* free_hstat (&hstat); */ + hstat.timestamp_checked = true; + } + + if (opt.spider) + { + bool finished = true; + if (opt.recursive) + { + if ((*dt & TEXTHTML) || (*dt & TEXTCSS)) + { + logputs (LOG_VERBOSE, _("\ +Remote file exists and could contain links to other resources -- retrieving.\n\n")); + finished = false; + } + else + { + logprintf (LOG_VERBOSE, _("\ +Remote file exists but does not contain any link -- not retrieving.\n\n")); + ret = RETROK; /* RETRUNNEEDED is not for caller. */ + } + } + else + { + if ((*dt & TEXTHTML) || (*dt & TEXTCSS)) + { + logprintf (LOG_VERBOSE, _("\ +Remote file exists and could contain further links,\n\ +but recursion is disabled -- not retrieving.\n\n")); + } + else + { + logprintf (LOG_VERBOSE, _("\ +Remote file exists.\n\n")); + } + ret = RETROK; /* RETRUNNEEDED is not for caller. */ + } + + if (finished) + { + logprintf (LOG_NONVERBOSE, + _("%s URL: %s %2d %s\n"), + tms, u->url, hstat.statcode, + hstat.message ? quotearg_style (escape_quoting_style, hstat.message) : ""); + goto exit; + } + } + + got_name = true; + *dt &= ~HEAD_ONLY; + count = 0; /* the retrieve count for HEAD is reset */ + continue; + } /* send_head_first */ + } /* !got_head */ + + if (opt.useservertimestamps + && (tmr != (time_t) (-1)) + && ((hstat.len == hstat.contlen) || + ((hstat.res == 0) && (hstat.contlen == -1)))) + { + const char *fl = NULL; + set_local_file (&fl, hstat.local_file); + if (fl) + { + time_t newtmr = -1; + /* Reparse time header, in case it's changed. */ + if (time_came_from_head + && hstat.remote_time && hstat.remote_time[0]) + { + newtmr = http_atotm (hstat.remote_time); + if (newtmr != (time_t)-1) + tmr = newtmr; + } + touch (fl, tmr); + } + } + /* End of time-stamping section. */ + + tmrate = retr_rate (hstat.rd_size, hstat.dltime); + total_download_time += hstat.dltime; + + if (hstat.len == hstat.contlen) + { + if (*dt & RETROKF || opt.content_on_error) + { + bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document)); + + logprintf (LOG_VERBOSE, + write_to_stdout + ? _("%s (%s) - written to stdout %s[%s/%s]\n\n") + : _("%s (%s) - %s saved [%s/%s]\n\n"), + tms, tmrate, + write_to_stdout ? "" : quote (hstat.local_file), + number_to_static_string (hstat.len), + number_to_static_string (hstat.contlen)); + logprintf (LOG_NONVERBOSE, + "%s URL:%s [%s/%s] -> \"%s\" [%d]\n", + tms, u->url, + number_to_static_string (hstat.len), + number_to_static_string (hstat.contlen), + hstat.local_file, count); + } + ++numurls; + total_downloaded_bytes += hstat.rd_size; + + /* Remember that we downloaded the file for later ".orig" code. */ + if (*dt & ADDED_HTML_EXTENSION) + downloaded_file (FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, hstat.local_file); + else + downloaded_file (FILE_DOWNLOADED_NORMALLY, hstat.local_file); + + ret = RETROK; + goto exit; + } + else if (hstat.res == 0) /* No read error */ + { + if (hstat.contlen == -1) /* We don't know how much we were supposed + to get, so assume we succeeded. */ + { + if (*dt & RETROKF || opt.content_on_error) + { + bool write_to_stdout = (opt.output_document && HYPHENP (opt.output_document)); + + logprintf (LOG_VERBOSE, + write_to_stdout + ? _("%s (%s) - written to stdout %s[%s]\n\n") + : _("%s (%s) - %s saved [%s]\n\n"), + tms, tmrate, + write_to_stdout ? "" : quote (hstat.local_file), + number_to_static_string (hstat.len)); + logprintf (LOG_NONVERBOSE, + "%s URL:%s [%s] -> \"%s\" [%d]\n", + tms, u->url, number_to_static_string (hstat.len), + hstat.local_file, count); + } + ++numurls; + total_downloaded_bytes += hstat.rd_size; + + /* Remember that we downloaded the file for later ".orig" code. */ + if (*dt & ADDED_HTML_EXTENSION) + downloaded_file (FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED, hstat.local_file); + else + downloaded_file (FILE_DOWNLOADED_NORMALLY, hstat.local_file); + + ret = RETROK; + goto exit; + } + else if (hstat.len < hstat.contlen) /* meaning we lost the + connection too soon */ + { + logprintf (LOG_VERBOSE, + _("%s (%s) - Connection closed at byte %s. "), + tms, tmrate, number_to_static_string (hstat.len)); + printwhat (count, opt.ntry); + continue; + } + else if (hstat.len != hstat.restval) + /* Getting here would mean reading more data than + requested with content-length, which we never do. */ + abort (); + else + { + /* Getting here probably means that the content-length was + * _less_ than the original, local size. We should probably + * truncate or re-read, or something. FIXME */ + ret = RETROK; + goto exit; + } + } + else /* from now on hstat.res can only be -1 */ + { + if (hstat.contlen == -1) + { + logprintf (LOG_VERBOSE, + _("%s (%s) - Read error at byte %s (%s)."), + tms, tmrate, number_to_static_string (hstat.len), + hstat.rderrmsg); + printwhat (count, opt.ntry); + continue; + } + else /* hstat.res == -1 and contlen is given */ + { + logprintf (LOG_VERBOSE, + _("%s (%s) - Read error at byte %s/%s (%s). "), + tms, tmrate, + number_to_static_string (hstat.len), + number_to_static_string (hstat.contlen), + hstat.rderrmsg); + printwhat (count, opt.ntry); + continue; + } + } + /* not reached */ + } + while (!opt.ntry || (count < opt.ntry)); + +exit: + if ((ret == RETROK || opt.content_on_error) && local_file) + { + xfree (*local_file); + /* Bugfix: Prevent SIGSEGV when hstat.local_file was left NULL + (i.e. due to opt.content_disposition). */ + if (hstat.local_file) + { + *local_file = hstat.local_file; + hstat.local_file = NULL; + } + } + free_hstat (&hstat); + + return ret; +} + +/* Check whether the result of strptime() indicates success. + strptime() returns the pointer to how far it got to in the string. + The processing has been successful if the string is at `GMT' or + `+X', or at the end of the string. + + In extended regexp parlance, the function returns 1 if P matches + "^ *(GMT|[+-][0-9]|$)", 0 otherwise. P being NULL (which strptime + can return) is considered a failure and 0 is returned. */ +static bool +check_end (const char *p) +{ + if (!p) + return false; + while (c_isspace (*p)) + ++p; + if (!*p + || (p[0] == 'G' && p[1] == 'M' && p[2] == 'T') + || ((p[0] == '+' || p[0] == '-') && c_isdigit (p[1]))) + return true; + else + return false; +} + +/* Convert the textual specification of time in TIME_STRING to the + number of seconds since the Epoch. + + TIME_STRING can be in any of the three formats RFC2616 allows the + HTTP servers to emit -- RFC1123-date, RFC850-date or asctime-date, + as well as the time format used in the Set-Cookie header. + Timezones are ignored, and should be GMT. + + Return the computed time_t representation, or -1 if the conversion + fails. + + This function uses strptime with various string formats for parsing + TIME_STRING. This results in a parser that is not as lenient in + interpreting TIME_STRING as I would like it to be. Being based on + strptime, it always allows shortened months, one-digit days, etc., + but due to the multitude of formats in which time can be + represented, an ideal HTTP time parser would be even more + forgiving. It should completely ignore things like week days and + concentrate only on the various forms of representing years, + months, days, hours, minutes, and seconds. For example, it would + be nice if it accepted ISO 8601 out of the box. + + I've investigated free and PD code for this purpose, but none was + usable. getdate was big and unwieldy, and had potential copyright + issues, or so I was informed. Dr. Marcus Hennecke's atotm(), + distributed with phttpd, is excellent, but we cannot use it because + it is not assigned to the FSF. So I stuck it with strptime. */ + +time_t +http_atotm (const char *time_string) +{ + /* NOTE: Solaris strptime man page claims that %n and %t match white + space, but that's not universally available. Instead, we simply + use ` ' to mean "skip all WS", which works under all strptime + implementations I've tested. */ + + static const char *time_formats[] = { + "%a, %d %b %Y %T", /* rfc1123: Thu, 29 Jan 1998 22:12:57 */ + "%A, %d-%b-%y %T", /* rfc850: Thursday, 29-Jan-98 22:12:57 */ + "%a %b %d %T %Y", /* asctime: Thu Jan 29 22:12:57 1998 */ + "%a, %d-%b-%Y %T" /* cookies: Thu, 29-Jan-1998 22:12:57 + (used in Set-Cookie, defined in the + Netscape cookie specification.) */ + }; + const char *oldlocale; + char savedlocale[256]; + size_t i; + time_t ret = (time_t) -1; + + /* Solaris strptime fails to recognize English month names in + non-English locales, which we work around by temporarily setting + locale to C before invoking strptime. */ + oldlocale = setlocale (LC_TIME, NULL); + if (oldlocale) + { + size_t l = strlen (oldlocale) + 1; + if (l >= sizeof savedlocale) + savedlocale[0] = '\0'; + else + memcpy (savedlocale, oldlocale, l); + } + else savedlocale[0] = '\0'; + + setlocale (LC_TIME, "C"); + + for (i = 0; i < countof (time_formats); i++) + { + struct tm t; + + /* Some versions of strptime use the existing contents of struct + tm to recalculate the date according to format. Zero it out + to prevent stack garbage from influencing strptime. */ + xzero (t); + + if (check_end (strptime (time_string, time_formats[i], &t))) + { + ret = timegm (&t); + break; + } + } + + /* Restore the previous locale. */ + if (savedlocale[0]) + setlocale (LC_TIME, savedlocale); + + return ret; +} + +/* Authorization support: We support three authorization schemes: + + * `Basic' scheme, consisting of base64-ing USER:PASSWORD string; + + * `Digest' scheme, added by Junio Hamano <junio@twinsun.com>, + consisting of answering to the server's challenge with the proper + MD5 digests. + + * `NTLM' ("NT Lan Manager") scheme, based on code written by Daniel + Stenberg for libcurl. Like digest, NTLM is based on a + challenge-response mechanism, but unlike digest, it is non-standard + (authenticates TCP connections rather than requests), undocumented + and Microsoft-specific. */ + +/* Create the authentication header contents for the `Basic' scheme. + This is done by encoding the string "USER:PASS" to base64 and + prepending the string "Basic " in front of it. */ + +static char * +basic_authentication_encode (const char *user, const char *passwd) +{ + char buf_t1[256], buf_t2[256]; + char *t1, *t2, *ret; + size_t len1 = strlen (user) + 1 + strlen (passwd); + + if (len1 < sizeof (buf_t1)) + t1 = buf_t1; + else + t1 = xmalloc(len1 + 1); + + if (BASE64_LENGTH (len1) < sizeof (buf_t2)) + t2 = buf_t2; + else + t2 = xmalloc (BASE64_LENGTH (len1) + 1); + + sprintf (t1, "%s:%s", user, passwd); + wget_base64_encode (t1, len1, t2); + + ret = concat_strings ("Basic ", t2, (char *) 0); + + if (t2 != buf_t2) + xfree (t2); + + if (t1 != buf_t1) + xfree (t1); + + return ret; +} + +#define SKIP_WS(x) do { \ + while (c_isspace (*(x))) \ + ++(x); \ +} while (0) + +#ifdef ENABLE_DIGEST +/* Dump the hexadecimal representation of HASH to BUF. HASH should be + an array of 16 bytes containing the hash keys, and BUF should be a + buffer of 33 writable characters (32 for hex digits plus one for + zero termination). */ +static void +dump_hash (char *buf, const unsigned char *hash) +{ + int i; + + for (i = 0; i < MD5_DIGEST_SIZE; i++, hash++) + { + *buf++ = XNUM_TO_digit (*hash >> 4); + *buf++ = XNUM_TO_digit (*hash & 0xf); + } + *buf = '\0'; +} + +/* Take the line apart to find the challenge, and compose a digest + authorization header. See RFC2069 section 2.1.2. */ +static char * +digest_authentication_encode (const char *au, const char *user, + const char *passwd, const char *method, + const char *path, uerr_t *auth_err) +{ + static char *realm, *opaque, *nonce, *qop, *algorithm; + static struct { + const char *name; + char **variable; + } options[] = { + { "realm", &realm }, + { "opaque", &opaque }, + { "nonce", &nonce }, + { "qop", &qop }, + { "algorithm", &algorithm } + }; + char cnonce[16] = ""; + char *res = NULL; + int res_len; + size_t res_size; + param_token name, value; + + + realm = opaque = nonce = algorithm = qop = NULL; + + au += 6; /* skip over `Digest' */ + while (extract_param (&au, &name, &value, ',', NULL)) + { + size_t i; + size_t namelen = name.e - name.b; + for (i = 0; i < countof (options); i++) + if (namelen == strlen (options[i].name) + && 0 == strncmp (name.b, options[i].name, + namelen)) + { + *options[i].variable = strdupdelim (value.b, value.e); + break; + } + } + + if (qop && strcmp (qop, "auth")) + { + logprintf (LOG_NOTQUIET, _("Unsupported quality of protection '%s'.\n"), qop); + xfree (qop); /* force freeing mem and continue */ + } + else if (algorithm && strcmp (algorithm,"MD5") && strcmp (algorithm,"MD5-sess")) + { + logprintf (LOG_NOTQUIET, _("Unsupported algorithm '%s'.\n"), algorithm); + xfree (algorithm); /* force freeing mem and continue */ + } + + if (!realm || !nonce || !user || !passwd || !path || !method) + { + *auth_err = ATTRMISSING; + goto cleanup; + } + + /* Calculate the digest value. */ + { + struct md5_ctx ctx; + unsigned char hash[MD5_DIGEST_SIZE]; + char a1buf[MD5_DIGEST_SIZE * 2 + 1], a2buf[MD5_DIGEST_SIZE * 2 + 1]; + char response_digest[MD5_DIGEST_SIZE * 2 + 1]; + + /* A1BUF = H(user ":" realm ":" password) */ + md5_init_ctx (&ctx); + md5_process_bytes ((unsigned char *)user, strlen (user), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)realm, strlen (realm), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)passwd, strlen (passwd), &ctx); + md5_finish_ctx (&ctx, hash); + + dump_hash (a1buf, hash); + + if (algorithm && !strcmp (algorithm, "MD5-sess")) + { + /* A1BUF = H( H(user ":" realm ":" password) ":" nonce ":" cnonce ) */ + snprintf (cnonce, sizeof (cnonce), "%08x", + (unsigned) random_number (INT_MAX)); + + md5_init_ctx (&ctx); + /* md5_process_bytes (hash, MD5_DIGEST_SIZE, &ctx); */ + md5_process_bytes (a1buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)cnonce, strlen (cnonce), &ctx); + md5_finish_ctx (&ctx, hash); + + dump_hash (a1buf, hash); + } + + /* A2BUF = H(method ":" path) */ + md5_init_ctx (&ctx); + md5_process_bytes ((unsigned char *)method, strlen (method), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)path, strlen (path), &ctx); + md5_finish_ctx (&ctx, hash); + dump_hash (a2buf, hash); + + if (qop && !strcmp (qop, "auth")) + { + /* RFC 2617 Digest Access Authentication */ + /* generate random hex string */ + if (!*cnonce) + snprintf (cnonce, sizeof (cnonce), "%08x", + (unsigned) random_number (INT_MAX)); + + /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" noncecount ":" clientnonce ":" qop ": " A2BUF) */ + md5_init_ctx (&ctx); + md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)"00000001", 8, &ctx); /* TODO: keep track of server nonce values */ + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)cnonce, strlen (cnonce), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)qop, strlen (qop), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_finish_ctx (&ctx, hash); + } + else + { + /* RFC 2069 Digest Access Authentication */ + /* RESPONSE_DIGEST = H(A1BUF ":" nonce ":" A2BUF) */ + md5_init_ctx (&ctx); + md5_process_bytes ((unsigned char *)a1buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)nonce, strlen (nonce), &ctx); + md5_process_bytes ((unsigned char *)":", 1, &ctx); + md5_process_bytes ((unsigned char *)a2buf, MD5_DIGEST_SIZE * 2, &ctx); + md5_finish_ctx (&ctx, hash); + } + + dump_hash (response_digest, hash); + + res_size = strlen (user) + + strlen (realm) + + strlen (nonce) + + strlen (path) + + 2 * MD5_DIGEST_SIZE /*strlen (response_digest)*/ + + (opaque ? strlen (opaque) : 0) + + (algorithm ? strlen (algorithm) : 0) + + (qop ? 128: 0) + + strlen (cnonce) + + 128; + + res = xmalloc (res_size); + + if (qop && !strcmp (qop, "auth")) + { + res_len = snprintf (res, res_size, "Digest "\ + "username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\""\ + ", qop=auth, nc=00000001, cnonce=\"%s\"", + user, realm, nonce, path, response_digest, cnonce); + + } + else + { + res_len = snprintf (res, res_size, "Digest "\ + "username=\"%s\", realm=\"%s\", nonce=\"%s\", uri=\"%s\", response=\"%s\"", + user, realm, nonce, path, response_digest); + } + + if (opaque) + { + res_len += snprintf (res + res_len, res_size - res_len, ", opaque=\"%s\"", opaque); + } + + if (algorithm) + { + snprintf (res + res_len, res_size - res_len, ", algorithm=\"%s\"", algorithm); + } + } + +cleanup: + xfree (realm); + xfree (opaque); + xfree (nonce); + xfree (qop); + xfree (algorithm); + + return res; +} +#endif /* ENABLE_DIGEST */ + +/* Computing the size of a string literal must take into account that + value returned by sizeof includes the terminating \0. */ +#define STRSIZE(literal) (sizeof (literal) - 1) + +/* Whether chars in [b, e) begin with the literal string provided as + first argument and are followed by whitespace or terminating \0. + The comparison is case-insensitive. */ +#define STARTS(literal, b, e) \ + ((e > b) \ + && ((size_t) ((e) - (b))) >= STRSIZE (literal) \ + && 0 == c_strncasecmp (b, literal, STRSIZE (literal)) \ + && ((size_t) ((e) - (b)) == STRSIZE (literal) \ + || c_isspace (b[STRSIZE (literal)]))) + +static bool +known_authentication_scheme_p (const char *hdrbeg, const char *hdrend) +{ + return STARTS ("Basic", hdrbeg, hdrend) +#ifdef ENABLE_DIGEST + || STARTS ("Digest", hdrbeg, hdrend) +#endif +#ifdef ENABLE_NTLM + || STARTS ("NTLM", hdrbeg, hdrend) +#endif + ; +} + +#undef STARTS + +/* Create the HTTP authorization request header. When the + `WWW-Authenticate' response header is seen, according to the + authorization scheme specified in that header (`Basic' and `Digest' + are supported by the current implementation), produce an + appropriate HTTP authorization request header. */ +static char * +create_authorization_line (const char *au, const char *user, + const char *passwd, const char *method, + const char *path, bool *finished, uerr_t *auth_err) +{ + /* We are called only with known schemes, so we can dispatch on the + first letter. */ + switch (c_toupper (*au)) + { + case 'B': /* Basic */ + *finished = true; + return basic_authentication_encode (user, passwd); +#ifdef ENABLE_DIGEST + case 'D': /* Digest */ + *finished = true; + return digest_authentication_encode (au, user, passwd, method, path, auth_err); +#endif +#ifdef ENABLE_NTLM + case 'N': /* NTLM */ + if (!ntlm_input (&pconn.ntlm, au)) + { + *finished = true; + return NULL; + } + return ntlm_output (&pconn.ntlm, user, passwd, finished); +#endif + default: + /* We shouldn't get here -- this function should be only called + with values approved by known_authentication_scheme_p. */ + abort (); + } +} + +static void +load_cookies (void) +{ + if (!wget_cookie_jar) + wget_cookie_jar = cookie_jar_new (); + if (opt.cookies_input && !cookies_loaded_p) + { + cookie_jar_load (wget_cookie_jar, opt.cookies_input); + cookies_loaded_p = true; + } +} + +void +save_cookies (void) +{ + if (wget_cookie_jar) + cookie_jar_save (wget_cookie_jar, opt.cookies_output); +} + +#if defined DEBUG_MALLOC || defined TESTING +void +http_cleanup (void) +{ + if (pconn_active) + invalidate_persistent (); + + if (wget_cookie_jar) + { + cookie_jar_delete (wget_cookie_jar); + wget_cookie_jar = NULL; + } + + if (basic_authed_hosts) + { + hash_table_iterator iter; + for (hash_table_iterate (basic_authed_hosts, &iter); hash_table_iter_next (&iter); ) + { + xfree (iter.key); + } + hash_table_destroy (basic_authed_hosts); + basic_authed_hosts = NULL; + } +} +#endif + +void +ensure_extension (struct http_stat *hs, const char *ext, int *dt) +{ + char *last_period_in_local_filename = strrchr (hs->local_file, '.'); + char shortext[8]; + int len; + shortext[0] = '\0'; + len = strlen (ext); + if (len == 5) + { + memcpy (shortext, ext, len - 1); + shortext[len - 1] = '\0'; + } + + if (last_period_in_local_filename == NULL + || !(0 == strcasecmp (last_period_in_local_filename, shortext) + || 0 == strcasecmp (last_period_in_local_filename, ext))) + { + int local_filename_len = strlen (hs->local_file); + /* Resize the local file, allowing for ".html" preceded by + optional ".NUMBER". */ + hs->local_file = xrealloc (hs->local_file, + local_filename_len + 24 + len); + strcpy (hs->local_file + local_filename_len, ext); + /* If clobbering is not allowed and the file, as named, + exists, tack on ".NUMBER.html" instead. */ + if (!ALLOW_CLOBBER && file_exists_p (hs->local_file, NULL)) + { + int ext_num = 1; + do + sprintf (hs->local_file + local_filename_len, + ".%d%s", ext_num++, ext); + while (file_exists_p (hs->local_file, NULL)); + } + *dt |= ADDED_HTML_EXTENSION; + } +} + +#ifdef TESTING + +const char * +test_parse_range_header (void) +{ + unsigned i; + static const struct { + const char * rangehdr; + const wgint firstbyte; + const wgint lastbyte; + const wgint length; + const bool shouldPass; + } test_array[] = { + { "bytes 0-1000/1000", 0, 1000, 1000, false }, + { "bytes 0-999/1000", 0, 999, 1000, true }, + { "bytes 100-99/1000", 100, 99, 1000, false }, + { "bytes 100-100/1000", 100, 100, 1000, true }, + { "bytes 0-1000/100000000", 0, 1000, 100000000, true }, + { "bytes 1-999/1000", 1, 999, 1000, true }, + { "bytes 42-1233/1234", 42, 1233, 1234, true }, + { "bytes 42-1233/*", 42, 1233, -1, true }, + { "bytes 0-2147483648/2147483649", 0, 2147483648U, 2147483649U, true }, + { "bytes 2147483648-4294967296/4294967297", 2147483648U, 4294967296ULL, 4294967297ULL, true }, + }; + + wgint firstbyteptr[sizeof(wgint)]; + wgint lastbyteptr[sizeof(wgint)]; + wgint lengthptr[sizeof(wgint)]; + bool result; + for (i = 0; i < countof (test_array); i++) + { + result = parse_content_range (test_array[i].rangehdr, firstbyteptr, lastbyteptr, lengthptr); +#if 0 + printf ("%ld %ld\n", test_array[i].firstbyte, *firstbyteptr); + printf ("%ld %ld\n", test_array[i].lastbyte, *lastbyteptr); + printf ("%ld %ld\n", test_array[i].length, *lengthptr); + printf ("\n"); +#endif + mu_assert ("test_parse_range_header: False Negative", result == test_array[i].shouldPass); + mu_assert ("test_parse_range_header: Bad parse", test_array[i].firstbyte == *firstbyteptr && + test_array[i].lastbyte == *lastbyteptr && + test_array[i].length == *lengthptr); + } + + return NULL; +} + +const char * +test_parse_content_disposition (void) +{ + unsigned i; + static const struct { + const char *hdrval; + const char *filename; + bool result; + } test_array[] = { + { "filename=\"file.ext\"", "file.ext", true }, + { "attachment; filename=\"file.ext\"", "file.ext", true }, + { "attachment; filename=\"file.ext\"; dummy", "file.ext", true }, + { "attachment", NULL, false }, + { "attachment; filename*=UTF-8'en-US'hello.txt", "hello.txt", true }, + { "attachment; filename*0=\"hello\"; filename*1=\"world.txt\"", + "helloworld.txt", true }, + { "attachment; filename=\"A.ext\"; filename*=\"B.ext\"", "B.ext", true }, + { "attachment; filename*=\"A.ext\"; filename*0=\"B\"; filename*1=\"B.ext\"", + "A.ext", true }, + { "filename**0=\"A\"; filename**1=\"A.ext\"; filename*0=\"B\";\ +filename*1=\"B\"", "AA.ext", true }, + }; + + for (i = 0; i < countof (test_array); ++i) + { + char *filename; + bool res; + + res = parse_content_disposition (test_array[i].hdrval, &filename); + + mu_assert ("test_parse_content_disposition: wrong result", + res == test_array[i].result + && (res == false + || 0 == strcmp (test_array[i].filename, filename))); + xfree (filename); + } + + return NULL; +} + +#endif /* TESTING */ + +/* + * vim: et sts=2 sw=2 cino+={s + */ diff --git a/src/http.h b/src/http.h new file mode 100644 index 0000000..5eac0da --- /dev/null +++ b/src/http.h @@ -0,0 +1,51 @@ +/* Declarations for HTTP. + Copyright (C) 2005-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef HTTP_H +#define HTTP_H + +#include "hsts.h" + +struct url; + +uerr_t http_loop (const struct url *, struct url *, char **, char **, const char *, + int *, struct url *, struct iri *); +void save_cookies (void); +void http_cleanup (void); +time_t http_atotm (const char *); + +typedef struct { + /* A token consists of characters in the [b, e) range. */ + const char *b, *e; +} param_token; +bool extract_param (const char **, param_token *, param_token *, char, bool *); + + +#endif /* HTTP_H */ diff --git a/src/init.c b/src/init.c new file mode 100644 index 0000000..aa526de --- /dev/null +++ b/src/init.c @@ -0,0 +1,2135 @@ +/* Reading/parsing the initialization file. + Copyright (C) 1996-2012, 2014-2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" +#include "exits.h" + +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <unistd.h> +#include <string.h> +#include <errno.h> +#include <limits.h> +/* not all systems provide PATH_MAX in limits.h */ +#ifndef PATH_MAX +# include <sys/param.h> +# ifndef PATH_MAX +# define PATH_MAX MAXPATHLEN +# endif +#endif + +#include <regex.h> + +#ifdef HAVE_PWD_H +# include <pwd.h> +#endif +#include <assert.h> + +#include "utils.h" +#include "init.h" +#include "host.h" +#include "netrc.h" +#include "progress.h" +#include "connect.h" /* for connect_cleanup */ +#include "ssl.h" /* for ssl_cleanup */ +#include "recur.h" /* for INFINITE_RECURSION */ +#include "convert.h" /* for convert_cleanup */ +#include "res.h" /* for res_cleanup */ +#include "http.h" /* for http_cleanup */ +#include "retr.h" /* for output_stream */ +#include "warc.h" /* for warc_close */ +#include "spider.h" /* for spider_cleanup */ +#include "html-url.h" /* for cleanup_html_url */ +#include "ptimer.h" /* for ptimer_destroy */ +#include "c-strcase.h" + +#ifdef TESTING +#include "../tests/unit-tests.h" +#endif + + + +#define CMD_DECLARE(func) static bool func (const char *, const char *, void *) + +CMD_DECLARE (cmd_boolean); +CMD_DECLARE (cmd_bytes); +CMD_DECLARE (cmd_bytes_sum); +#ifdef HAVE_SSL +CMD_DECLARE (cmd_cert_type); +#endif +CMD_DECLARE (cmd_directory_vector); +CMD_DECLARE (cmd_number); +CMD_DECLARE (cmd_number_inf); +CMD_DECLARE (cmd_string); +CMD_DECLARE (cmd_string_uppercase); +CMD_DECLARE (cmd_file); +CMD_DECLARE (cmd_file_once); +CMD_DECLARE (cmd_directory); +CMD_DECLARE (cmd_time); +CMD_DECLARE (cmd_vector); + +CMD_DECLARE (cmd_use_askpass); + +#ifdef HAVE_LIBZ +CMD_DECLARE (cmd_spec_compression); +#endif +CMD_DECLARE (cmd_spec_dirstruct); +CMD_DECLARE (cmd_spec_header); +CMD_DECLARE (cmd_spec_warc_header); +CMD_DECLARE (cmd_spec_htmlify); +CMD_DECLARE (cmd_spec_mirror); +CMD_DECLARE (cmd_spec_prefer_family); +CMD_DECLARE (cmd_spec_progress); +CMD_DECLARE (cmd_spec_progressdisp); +CMD_DECLARE (cmd_spec_recursive); +CMD_DECLARE (cmd_spec_regex_type); +CMD_DECLARE (cmd_spec_restrict_file_names); +CMD_DECLARE (cmd_spec_report_speed); +#ifdef HAVE_SSL +CMD_DECLARE (cmd_spec_secure_protocol); +#endif +CMD_DECLARE (cmd_spec_timeout); +CMD_DECLARE (cmd_spec_useragent); +CMD_DECLARE (cmd_spec_verbose); +CMD_DECLARE (cmd_check_cert); + +/* List of recognized commands, each consisting of name, place and + function. When adding a new command, simply add it to the list, + but be sure to keep the list sorted alphabetically, as + command_by_name's binary search depends on it. Also, be sure to + add any entries that allocate memory (e.g. cmd_string and + cmd_vector) to the cleanup() function below. */ + +static const struct { + const char *name; + void *place; + bool (*action) (const char *, const char *, void *); +} commands[] = { + /* KEEP THIS LIST ALPHABETICALLY SORTED */ + { "accept", &opt.accepts, cmd_vector }, + { "acceptregex", &opt.acceptregex_s, cmd_string }, + { "addhostdir", &opt.add_hostdir, cmd_boolean }, + { "adjustextension", &opt.adjust_extension, cmd_boolean }, + { "alwaysrest", &opt.always_rest, cmd_boolean }, /* deprecated */ + { "askpassword", &opt.ask_passwd, cmd_boolean }, + { "authnochallenge", &opt.auth_without_challenge, + cmd_boolean }, + { "background", &opt.background, cmd_boolean }, + { "backupconverted", &opt.backup_converted, cmd_boolean }, + { "backups", &opt.backups, cmd_number }, + { "base", &opt.base_href, cmd_string }, + { "bindaddress", &opt.bind_address, cmd_string }, +#ifdef HAVE_LIBCARES + { "binddnsaddress", &opt.bind_dns_address, cmd_string }, +#endif + { "bodydata", &opt.body_data, cmd_string }, + { "bodyfile", &opt.body_file, cmd_string }, +#ifdef HAVE_SSL + { "cacertificate", &opt.ca_cert, cmd_file }, +#endif + { "cache", &opt.allow_cache, cmd_boolean }, +#ifdef HAVE_SSL + { "cadirectory", &opt.ca_directory, cmd_directory }, + { "certificate", &opt.cert_file, cmd_file }, + { "certificatetype", &opt.cert_type, cmd_cert_type }, + { "checkcertificate", &opt.check_cert, cmd_check_cert }, +#endif + { "chooseconfig", &opt.choose_config, cmd_file }, +#ifdef HAVE_SSL + { "ciphers", &opt.tls_ciphers_string, cmd_string }, +#endif +#ifdef HAVE_LIBZ + { "compression", &opt.compression, cmd_spec_compression }, +#endif + { "connecttimeout", &opt.connect_timeout, cmd_time }, + { "contentdisposition", &opt.content_disposition, cmd_boolean }, + { "contentonerror", &opt.content_on_error, cmd_boolean }, + { "continue", &opt.always_rest, cmd_boolean }, + { "convertfileonly", &opt.convert_file_only, cmd_boolean }, + { "convertlinks", &opt.convert_links, cmd_boolean }, + { "cookies", &opt.cookies, cmd_boolean }, +#ifdef HAVE_SSL + { "crlfile", &opt.crl_file, cmd_file_once }, +#endif + { "cutdirs", &opt.cut_dirs, cmd_number }, + { "debug", &opt.debug, cmd_boolean }, + { "defaultpage", &opt.default_page, cmd_string }, + { "deleteafter", &opt.delete_after, cmd_boolean }, + { "dirprefix", &opt.dir_prefix, cmd_directory }, + { "dirstruct", NULL, cmd_spec_dirstruct }, + { "dnscache", &opt.dns_cache, cmd_boolean }, +#ifdef HAVE_LIBCARES + { "dnsservers", &opt.dns_servers, cmd_string }, +#endif + { "dnstimeout", &opt.dns_timeout, cmd_time }, + { "domains", &opt.domains, cmd_vector }, + { "dotbytes", &opt.dot_bytes, cmd_bytes }, + { "dotsinline", &opt.dots_in_line, cmd_number }, + { "dotspacing", &opt.dot_spacing, cmd_number }, + { "dotstyle", &opt.dot_style, cmd_string }, /* deprecated */ +#ifdef HAVE_SSL + { "egdfile", &opt.egd_file, cmd_file }, +#endif + { "excludedirectories", &opt.excludes, cmd_directory_vector }, + { "excludedomains", &opt.exclude_domains, cmd_vector }, + { "followftp", &opt.follow_ftp, cmd_boolean }, + { "followtags", &opt.follow_tags, cmd_vector }, + { "forcehtml", &opt.force_html, cmd_boolean }, + { "ftppasswd", &opt.ftp_passwd, cmd_string }, /* deprecated */ + { "ftppassword", &opt.ftp_passwd, cmd_string }, + { "ftpproxy", &opt.ftp_proxy, cmd_string }, +#ifdef HAVE_SSL + { "ftpscleardataconnection", &opt.ftps_clear_data_connection, cmd_boolean }, + { "ftpsfallbacktoftp", &opt.ftps_fallback_to_ftp, cmd_boolean }, + { "ftpsimplicit", &opt.ftps_implicit, cmd_boolean }, + { "ftpsresumessl", &opt.ftps_resume_ssl, cmd_boolean }, +#endif +#ifdef __VMS + { "ftpstmlf", &opt.ftp_stmlf, cmd_boolean }, +#endif /* def __VMS */ + { "ftpuser", &opt.ftp_user, cmd_string }, + { "glob", &opt.ftp_glob, cmd_boolean }, + { "header", NULL, cmd_spec_header }, +#ifdef HAVE_HSTS + { "hsts", &opt.hsts, cmd_boolean }, + { "hstsfile", &opt.hsts_file, cmd_file }, +#endif + { "htmlextension", &opt.adjust_extension, cmd_boolean }, /* deprecated */ + { "htmlify", NULL, cmd_spec_htmlify }, + { "httpkeepalive", &opt.http_keep_alive, cmd_boolean }, + { "httppasswd", &opt.http_passwd, cmd_string }, /* deprecated */ + { "httppassword", &opt.http_passwd, cmd_string }, + { "httpproxy", &opt.http_proxy, cmd_string }, +#ifdef HAVE_SSL + { "httpsonly", &opt.https_only, cmd_boolean }, +#endif + { "httpsproxy", &opt.https_proxy, cmd_string }, + { "httpuser", &opt.http_user, cmd_string }, + { "ifmodifiedsince", &opt.if_modified_since, cmd_boolean }, + { "ignorecase", &opt.ignore_case, cmd_boolean }, + { "ignorelength", &opt.ignore_length, cmd_boolean }, + { "ignoretags", &opt.ignore_tags, cmd_vector }, + { "includedirectories", &opt.includes, cmd_directory_vector }, +#ifdef ENABLE_IPV6 + { "inet4only", &opt.ipv4_only, cmd_boolean }, + { "inet6only", &opt.ipv6_only, cmd_boolean }, +#endif + { "input", &opt.input_filename, cmd_file }, +#ifdef HAVE_METALINK + { "inputmetalink", &opt.input_metalink, cmd_file }, +#endif + { "iri", &opt.enable_iri, cmd_boolean }, + { "keepbadhash", &opt.keep_badhash, cmd_boolean }, + { "keepsessioncookies", &opt.keep_session_cookies, cmd_boolean }, + { "limitrate", &opt.limit_rate, cmd_bytes }, + { "loadcookies", &opt.cookies_input, cmd_file }, + { "localencoding", &opt.locale, cmd_string }, + { "logfile", &opt.lfilename, cmd_file }, + { "login", &opt.ftp_user, cmd_string },/* deprecated*/ + { "maxredirect", &opt.max_redirect, cmd_number }, +#ifdef HAVE_METALINK + { "metalinkindex", &opt.metalink_index, cmd_number_inf }, + { "metalinkoverhttp", &opt.metalink_over_http, cmd_boolean }, +#endif + { "method", &opt.method, cmd_string_uppercase }, + { "mirror", NULL, cmd_spec_mirror }, + { "netrc", &opt.netrc, cmd_boolean }, + { "noclobber", &opt.noclobber, cmd_boolean }, + { "noconfig", &opt.noconfig, cmd_boolean }, + { "noparent", &opt.no_parent, cmd_boolean }, + { "noproxy", &opt.no_proxy, cmd_vector }, + { "numtries", &opt.ntry, cmd_number_inf },/* deprecated*/ + { "outputdocument", &opt.output_document, cmd_file }, + { "pagerequisites", &opt.page_requisites, cmd_boolean }, + { "passiveftp", &opt.ftp_pasv, cmd_boolean }, + { "passwd", &opt.ftp_passwd, cmd_string },/* deprecated*/ + { "password", &opt.passwd, cmd_string }, +#ifdef HAVE_SSL + { "pinnedpubkey", &opt.pinnedpubkey, cmd_string }, +#endif + { "postdata", &opt.post_data, cmd_string }, + { "postfile", &opt.post_file_name, cmd_file }, + { "preferfamily", NULL, cmd_spec_prefer_family }, +#ifdef HAVE_METALINK + { "preferredlocation", &opt.preferred_location, cmd_string }, +#endif + { "preservepermissions", &opt.preserve_perm, cmd_boolean }, +#ifdef HAVE_SSL + { "privatekey", &opt.private_key, cmd_file }, + { "privatekeytype", &opt.private_key_type, cmd_cert_type }, +#endif + { "progress", &opt.progress_type, cmd_spec_progress }, + { "protocoldirectories", &opt.protocol_directories, cmd_boolean }, + { "proxypasswd", &opt.proxy_passwd, cmd_string }, /* deprecated */ + { "proxypassword", &opt.proxy_passwd, cmd_string }, + { "proxyuser", &opt.proxy_user, cmd_string }, + { "quiet", &opt.quiet, cmd_boolean }, + { "quota", &opt.quota, cmd_bytes_sum }, +#ifdef HAVE_SSL + { "randomfile", &opt.random_file, cmd_file }, +#endif + { "randomwait", &opt.random_wait, cmd_boolean }, + { "readtimeout", &opt.read_timeout, cmd_time }, + { "reclevel", &opt.reclevel, cmd_number_inf }, + { "recursive", NULL, cmd_spec_recursive }, + { "referer", &opt.referer, cmd_string }, + { "regextype", &opt.regex_type, cmd_spec_regex_type }, + { "reject", &opt.rejects, cmd_vector }, + { "rejectedlog", &opt.rejected_log, cmd_file }, + { "rejectregex", &opt.rejectregex_s, cmd_string }, + { "relativeonly", &opt.relative_only, cmd_boolean }, + { "remoteencoding", &opt.encoding_remote, cmd_string }, + { "removelisting", &opt.remove_listing, cmd_boolean }, + { "reportspeed", &opt.report_bps, cmd_spec_report_speed}, + { "restrictfilenames", NULL, cmd_spec_restrict_file_names }, + { "retrsymlinks", &opt.retr_symlinks, cmd_boolean }, + { "retryconnrefused", &opt.retry_connrefused, cmd_boolean }, + { "retryonhosterror", &opt.retry_on_host_error, cmd_boolean }, + { "retryonhttperror", &opt.retry_on_http_error, cmd_string }, + { "robots", &opt.use_robots, cmd_boolean }, + { "savecookies", &opt.cookies_output, cmd_file }, + { "saveheaders", &opt.save_headers, cmd_boolean }, +#ifdef HAVE_SSL + { "secureprotocol", &opt.secure_protocol, cmd_spec_secure_protocol }, +#endif + { "serverresponse", &opt.server_response, cmd_boolean }, + { "showalldnsentries", &opt.show_all_dns_entries, cmd_boolean }, + { "showprogress", &opt.show_progress, cmd_spec_progressdisp }, + { "spanhosts", &opt.spanhost, cmd_boolean }, + { "spider", &opt.spider, cmd_boolean }, + { "startpos", &opt.start_pos, cmd_bytes }, + { "strictcomments", &opt.strict_comments, cmd_boolean }, + { "timeout", NULL, cmd_spec_timeout }, + { "timestamping", &opt.timestamping, cmd_boolean }, + { "tries", &opt.ntry, cmd_number_inf }, + { "trustservernames", &opt.trustservernames, cmd_boolean }, + { "unlink", &opt.unlink_requested, cmd_boolean }, +#ifndef __VMS + { "useaskpass" , &opt.use_askpass, cmd_use_askpass }, +#endif + { "useproxy", &opt.use_proxy, cmd_boolean }, + { "user", &opt.user, cmd_string }, + { "useragent", NULL, cmd_spec_useragent }, + { "useservertimestamps", &opt.useservertimestamps, cmd_boolean }, + { "verbose", NULL, cmd_spec_verbose }, + { "wait", &opt.wait, cmd_time }, + { "waitretry", &opt.waitretry, cmd_time }, + { "warccdx", &opt.warc_cdx_enabled, cmd_boolean }, + { "warccdxdedup", &opt.warc_cdx_dedup_filename, cmd_file }, +#ifdef HAVE_LIBZ + { "warccompression", &opt.warc_compression_enabled, cmd_boolean }, +#endif + { "warcdigests", &opt.warc_digests_enabled, cmd_boolean }, + { "warcfile", &opt.warc_filename, cmd_file }, + { "warcheader", NULL, cmd_spec_warc_header }, + { "warckeeplog", &opt.warc_keep_log, cmd_boolean }, + { "warcmaxsize", &opt.warc_maxsize, cmd_bytes }, + { "warctempdir", &opt.warc_tempdir, cmd_directory }, +#ifdef USE_WATT32 + { "wdebug", &opt.wdebug, cmd_boolean }, +#endif +#ifdef ENABLE_XATTR + { "xattr", &opt.enable_xattr, cmd_boolean }, +#endif +}; + +/* Look up CMDNAME in the commands[] and return its position in the + array. If CMDNAME is not found, return -1. */ + +static int +command_by_name (const char *cmdname) +{ + /* Use binary search for speed. Wget has ~100 commands, which + guarantees a worst case performance of 7 string comparisons. */ + int lo = 0, hi = countof (commands) - 1; + + while (lo <= hi) + { + int mid = (lo + hi) >> 1; + int cmp = c_strcasecmp (cmdname, commands[mid].name); + if (cmp < 0) + hi = mid - 1; + else if (cmp > 0) + lo = mid + 1; + else + return mid; + } + return -1; +} + +/* Reset the variables to default values. */ +void +defaults (void) +{ + char *tmp; + + /* Most of the default values are 0 (and 0.0, NULL, and false). + Just reset everything, and fill in the non-zero values. Note + that initializing pointers to NULL this way is technically + illegal, but porting Wget to a machine where NULL is not all-zero + bit pattern will be the least of the implementors' worries. */ + xzero (opt); + +#ifdef HAVE_METALINK + opt.metalink_index = -1; +#endif + + opt.cookies = true; + opt.verbose = -1; + opt.ntry = 20; + opt.reclevel = 5; + opt.add_hostdir = true; + opt.netrc = true; + opt.ftp_glob = true; + opt.htmlify = true; + opt.http_keep_alive = true; + opt.use_proxy = true; + opt.convert_file_only = false; + tmp = getenv ("no_proxy"); + if (tmp) + opt.no_proxy = sepstring (tmp); + opt.prefer_family = prefer_none; + opt.allow_cache = true; + opt.if_modified_since = true; + + opt.read_timeout = 900; + opt.use_robots = true; + + opt.remove_listing = true; + + opt.dot_bytes = 1024; + opt.dot_spacing = 10; + opt.dots_in_line = 50; + + opt.dns_cache = true; + opt.ftp_pasv = true; + /* 2014-09-07 Darshit Shah <darnir@gmail.com> + * opt.retr_symlinks is set to true by default. Creating symbolic links on the + * local filesystem pose a security threat by malicious FTP Servers that + * server a specially crafted .listing file akin to this: + * + * lrwxrwxrwx 1 root root 33 Dec 25 2012 JoCxl6d8rFU -> / + * drwxrwxr-x 15 1024 106 4096 Aug 28 02:02 JoCxl6d8rFU + * + * A .listing file in this fashion makes Wget susceptiple to a symlink attack + * wherein the attacker is able to create arbitrary files, directories and + * symbolic links on the target system and even set permissions. + * + * Hence, by default Wget attempts to retrieve the pointed-to files and does + * not create the symbolic links locally. + */ + opt.retr_symlinks = true; + +#ifdef HAVE_SSL + opt.check_cert = CHECK_CERT_ON; + opt.ftps_resume_ssl = true; + opt.ftps_fallback_to_ftp = false; + opt.ftps_implicit = false; + opt.ftps_clear_data_connection = false; +#endif + +#ifdef HAVE_LIBZ + opt.compression = compression_none; +#endif + + /* The default for file name restriction defaults to the OS type. */ +#if defined(WINDOWS) || defined(MSDOS) || defined(__CYGWIN__) + opt.restrict_files_os = restrict_windows; +#elif defined(__VMS) + opt.restrict_files_os = restrict_vms; +#else + opt.restrict_files_os = restrict_unix; +#endif + opt.restrict_files_ctrl = true; + opt.restrict_files_nonascii = false; + opt.restrict_files_case = restrict_no_case_restriction; + + opt.regex_type = regex_type_posix; + + opt.max_redirect = 20; + + opt.waitretry = 10; + +#ifdef ENABLE_IRI + opt.enable_iri = true; +#else + opt.enable_iri = false; +#endif + opt.locale = NULL; + opt.encoding_remote = NULL; + + opt.useservertimestamps = true; + opt.show_all_dns_entries = false; + + opt.warc_maxsize = 0; /* 1024 * 1024 * 1024; */ +#ifdef HAVE_LIBZ + opt.warc_compression_enabled = true; +#else + opt.warc_compression_enabled = false; +#endif + opt.warc_digests_enabled = true; + opt.warc_cdx_enabled = false; + opt.warc_cdx_dedup_filename = NULL; + opt.warc_tempdir = NULL; + opt.warc_keep_log = true; + + /* Use a negative value to mark the absence of --start-pos option */ + opt.start_pos = -1; + opt.show_progress = -1; + opt.noscroll = false; + +#ifdef HAVE_HSTS + /* HSTS is enabled by default */ + opt.hsts = true; +#endif + + opt.enable_xattr = false; +} + +/* Return the user's home directory (strdup-ed), or NULL if none is + found. */ +char * +home_dir (void) +{ + static char *buf = NULL; + static char *home, *ret; + + if (!home) + { + home = getenv ("HOME"); + if (!home) + { +#if defined(MSDOS) + int len; + + /* Under MSDOS, if $HOME isn't defined, use the directory where + `wget.exe' resides. */ + const char *_w32_get_argv0 (void); /* in libwatt.a/pcconfig.c */ + char *p; + + buff = _w32_get_argv0 (); + + p = strrchr (buf, '/'); /* djgpp */ + if (!p) + p = strrchr (buf, '\\'); /* others */ + assert (p); + + len = p - buff + 1; + buff = strdup (_w32_get_argv0 ()); + + home = buf; +#elif !defined(WINDOWS) + /* If HOME is not defined, try getting it from the password + file. */ + struct passwd *pwd = getpwuid (getuid ()); + if (!pwd || !pwd->pw_dir) + return NULL; + home = pwd->pw_dir; +#else /* !WINDOWS */ + /* Under Windows, if $HOME isn't defined, use the directory where + `wget.exe' resides. */ + home = ws_mypath (); +#endif /* WINDOWS */ + } + } + + ret = home ? xstrdup (home) : NULL; + xfree (buf); + + return ret; +} + +/* Check the 'WGETRC' environment variable and return the file name + if 'WGETRC' is set and is a valid file. + If the `WGETRC' variable exists but the file does not exist, the + function will exit(). */ +char * +wgetrc_env_file_name (void) +{ + char *env = getenv ("WGETRC"); + if (env && *env) + { + file_stats_t flstat; + if (!file_exists_p (env, &flstat)) + { + fprintf (stderr, _("%s: WGETRC points to %s, which couldn't be accessed because of error: %s.\n"), + exec_name, env, strerror(flstat.access_err)); + exit (WGET_EXIT_GENERIC_ERROR); + } + return xstrdup (env); + } + return NULL; +} + +/* Append file name to (locally appropriate) directory spec. + Return pointer to allocated storage. */ +char * +ajoin_dir_file (const char *dir, const char *file) +{ + char *dir_file; +#ifdef __VMS + /* No separator: "dev:[dir]" + "name.type" */ + dir_file = aprintf ("%s%s", dir, file); +#else /* def __VMS */ + /* Slash separator: "/a/b" + "/" + "name.type" */ + dir_file = aprintf ("%s/%s", dir, file); +#endif /* def __VMS [else] */ + return dir_file; +} + +/* Check for the existence of '$HOME/.wgetrc' and return its path + if it exists and is set. */ +char * +wgetrc_user_file_name (void) +{ + char *file = NULL; + + /* Join opt.homedir ($HOME) and ".wgetrc" */ + if (opt.homedir) { + file = ajoin_dir_file(opt.homedir, ".wgetrc"); + } + + if (!file) + return NULL; +#ifndef FUZZING + if (!file_exists_p (file, NULL)) + { + xfree (file); + return NULL; + } +#endif + return file; +} + +/* Return the path to the user's .wgetrc. This is either the value of + `WGETRC' environment variable, or `$HOME/.wgetrc'. + + Additionally, for windows, look in the directory where wget.exe + resides. */ +char * +wgetrc_file_name (void) +{ + char *file = wgetrc_env_file_name (); + if (file && *file) + return file; + + file = wgetrc_user_file_name (); + +#ifdef WINDOWS + /* Under Windows, if we still haven't found .wgetrc, look for the file + `wget.ini' in the directory where `wget.exe' resides; we do this for + backward compatibility with previous versions of Wget. + SYSTEM_WGETRC should not be defined under WINDOWS. */ + if (!file) + { + const char *home = ws_mypath (); + if (home) + { + file = aprintf ("%s/wget.ini", home); + if (!file_exists_p (file, NULL)) + { + xfree (file); + } + } + } +#endif /* WINDOWS */ + + return file; +} + +/* Return values of parse_line. */ +enum parse_line { + line_ok, + line_empty, + line_syntax_error, + line_unknown_command +}; + +static enum parse_line parse_line (const char *, char **, char **, int *); +static bool setval_internal (int, const char *, const char *); +static bool setval_internal_tilde (int, const char *, const char *); + +/* Initialize variables from a wgetrc file. Returns zero (failure) if + there were errors in the file. */ + +bool +run_wgetrc (const char *file, file_stats_t *flstats) +{ + FILE *fp; + char *line = NULL; + size_t bufsize = 0; + int ln; + int errcnt = 0; + + fp = fopen_stat (file, "r", flstats); + if (!fp) + { + fprintf (stderr, _("%s: Cannot read %s (%s).\n"), exec_name, + file, strerror (errno)); + return true; /* not a fatal error */ + } + ln = 1; + while (getline (&line, &bufsize, fp) > 0) + { + char *com = NULL, *val = NULL; + int comind; + + /* Parse the line. */ + switch (parse_line (line, &com, &val, &comind)) + { + case line_ok: + /* If everything is OK, set the value. */ + if (!setval_internal_tilde (comind, com, val)) + { + fprintf (stderr, _("%s: Error in %s at line %d.\n"), + exec_name, file, ln); + ++errcnt; + } + break; + case line_syntax_error: + fprintf (stderr, _("%s: Syntax error in %s at line %d.\n"), + exec_name, file, ln); + ++errcnt; + break; + case line_unknown_command: + fprintf (stderr, _("%s: Unknown command %s in %s at line %d.\n"), + exec_name, quote (com), file, ln); + ++errcnt; + break; + case line_empty: + break; + default: + abort (); + } + xfree (com); + xfree (val); + ++ln; + } + xfree (line); + fclose (fp); + + return errcnt == 0; +} + +/* Initialize the defaults and run the system wgetrc and user's own + wgetrc. */ +int +initialize (void) +{ + char *env_sysrc; + file_stats_t flstats; + bool ok = true; + + memset(&flstats, 0, sizeof(flstats)); + /* Run a non-standard system rc file when the according environment + variable has been set. For internal testing purposes only! */ + env_sysrc = getenv ("SYSTEM_WGETRC"); + if (env_sysrc && file_exists_p (env_sysrc, &flstats)) + { + ok &= run_wgetrc (env_sysrc, &flstats); + /* If there are any problems parsing the system wgetrc file, tell + the user and exit */ + if (! ok) + { + fprintf (stderr, _("\ +Parsing system wgetrc file (env SYSTEM_WGETRC) failed. Please check\n\ +'%s',\n\ +or specify a different file using --config.\n"), env_sysrc); + return WGET_EXIT_PARSE_ERROR; + } + } + /* Otherwise, if SYSTEM_WGETRC is defined, use it. */ +#ifdef SYSTEM_WGETRC + else if (file_exists_p (SYSTEM_WGETRC, &flstats)) + ok &= run_wgetrc (SYSTEM_WGETRC, &flstats); + /* If there are any problems parsing the system wgetrc file, tell + the user and exit */ + if (! ok) + { + fprintf (stderr, _("\ +Parsing system wgetrc file failed. Please check\n\ +'%s',\n\ +or specify a different file using --config.\n"), SYSTEM_WGETRC); + return WGET_EXIT_PARSE_ERROR; + } +#endif + /* Override it with your own, if one exists. */ + opt.wgetrcfile = wgetrc_file_name (); + if (!opt.wgetrcfile) + return 0; + /* #### We should canonicalize `file' and SYSTEM_WGETRC with + something like realpath() before comparing them with `strcmp' */ +#ifdef SYSTEM_WGETRC + if (!strcmp (opt.wgetrcfile, SYSTEM_WGETRC)) + { + fprintf (stderr, _("\ +%s: Warning: Both system and user wgetrc point to %s.\n"), + exec_name, quote (opt.wgetrcfile)); + } + else +#endif +#ifndef FUZZING + if (file_exists_p (opt.wgetrcfile, &flstats)) +#endif + ok &= run_wgetrc (opt.wgetrcfile, &flstats); + + xfree (opt.wgetrcfile); + + /* If there were errors processing either `.wgetrc', abort. */ + if (!ok) + return WGET_EXIT_PARSE_ERROR; + + return 0; +} + +/* Remove dashes and underscores from S, modifying S in the + process. */ + +static void +dehyphen (char *s) +{ + char *t = s; /* t - tortoise */ + char *h = s; /* h - hare */ + while (*h) + if (*h == '_' || *h == '-') + ++h; + else + *t++ = *h++; + *t = '\0'; +} + +/* Parse the line pointed by line, with the syntax: + <sp>* command <sp>* = <sp>* value <sp>* + Uses malloc to allocate space for command and value. + + Returns one of line_ok, line_empty, line_syntax_error, or + line_unknown_command. + + In case of line_ok, *COM and *VAL point to freshly allocated + strings, and *COMIND points to com's index. In case of error or + empty line, their values are unmodified. */ + +static enum parse_line +parse_line (const char *line, char **com, char **val, int *comind) +{ + const char *p; + const char *end = line + strlen (line); + const char *cmdstart, *cmdend; + const char *valstart, *valend; + char buf[1024]; + size_t len; + + char *cmdcopy; + int ind; + + /* Skip leading and trailing whitespace. */ + while (*line && c_isspace (*line)) + ++line; + while (end > line && c_isspace (end[-1])) + --end; + + /* Skip empty lines and comments. */ + if (!*line || *line == '#') + return line_empty; + + p = line; + + cmdstart = p; + while (p < end && (c_isalnum (*p) || *p == '_' || *p == '-')) + ++p; + cmdend = p; + + /* Skip '=', as well as any space before or after it. */ + while (p < end && c_isspace (*p)) + ++p; + if (p == end || *p != '=') + return line_syntax_error; + ++p; + while (p < end && c_isspace (*p)) + ++p; + + valstart = p; + valend = end; + + /* The syntax is valid (even though the command might not be). Fill + in the command name and value. */ + *com = strdupdelim (cmdstart, cmdend); + *val = strdupdelim (valstart, valend); + + /* The line now known to be syntactically correct. Check whether + the command is valid. */ + len = cmdend - cmdstart; + if (len < sizeof (buf)) + cmdcopy = buf; + else + cmdcopy = xmalloc (len + 1); + memcpy (cmdcopy, cmdstart, len); + cmdcopy[len] = 0; + + dehyphen (cmdcopy); + ind = command_by_name (cmdcopy); + if (cmdcopy != buf) + xfree (cmdcopy); + if (ind == -1) + return line_unknown_command; + + /* Report success to the caller. */ + *comind = ind; + return line_ok; +} + +#if defined(WINDOWS) || defined(MSDOS) +# define ISSEP(c) ((c) == '/' || (c) == '\\') +# define SEPSTRING "/\\" +#else +# define ISSEP(c) ((c) == '/') +# define SEPSTRING "/" +#endif + +/* Run commands[comind].action. */ + +static bool +setval_internal (int comind, const char *com, const char *val) +{ + assert (0 <= comind && ((size_t) comind) < countof (commands)); + + if ((unsigned) comind >= countof (commands)) + return NULL; + + DEBUGP (("Setting %s (%s) to %s\n", com, commands[comind].name, val)); + return commands[comind].action (com, val, commands[comind].place); +} + +static bool +setval_internal_tilde (int comind, const char *com, const char *val) +{ + bool ret; + int homelen; + char **pstring; + ret = setval_internal (comind, com, val); + + /* We make tilde expansion for cmd_file and cmd_directory */ + if (((commands[comind].action == cmd_file) || + (commands[comind].action == cmd_directory)) + && ret && (*val == '~' && ISSEP (val[1]))) + { + pstring = commands[comind].place; + if (opt.homedir) + { + char *home = xstrdup(opt.homedir); + homelen = strlen (home); + while (homelen && ISSEP (home[homelen - 1])) + home[--homelen] = '\0'; + + xfree (*pstring); + + /* Skip the leading "~/". */ + val += strspn(val + 1, SEPSTRING) + 1; + *pstring = concat_strings (home, "/", val, (char *)0); + xfree (home); + } + } + return ret; +} + +/* Run command COM with value VAL. If running the command produces an + error, report the error and exit. + + This is intended to be called from main to modify Wget's behavior + through command-line switches. Since COM is hard-coded in main, + it is not canonicalized, and this aborts when COM is not found. + + If COMIND's are exported to init.h, this function will be changed + to accept COMIND directly. */ + +void +setoptval (const char *com, const char *val, const char *optname) +{ + /* Prepend "--" to OPTNAME. */ + char dd_optname[2 + MAX_LONGOPTION + 1]; + + if ((unsigned) snprintf(dd_optname, sizeof (dd_optname), "--%s", optname) > sizeof (dd_optname)) + exit (WGET_EXIT_PARSE_ERROR); + + assert (val != NULL); + + if (!setval_internal (command_by_name (com), dd_optname, val)) + exit (WGET_EXIT_PARSE_ERROR); +} + +/* Parse OPT into command and value and run it. For example, + run_command("foo=bar") is equivalent to setoptval("foo", "bar"). + This is used by the `--execute' flag in main.c. */ + +void +run_command (const char *cmdopt) +{ + char *com, *val; + int comind; + switch (parse_line (cmdopt, &com, &val, &comind)) + { + case line_ok: + if (!setval_internal (comind, com, val)) + exit (WGET_EXIT_PARSE_ERROR); + xfree (com); + xfree (val); + break; + default: + fprintf (stderr, _("%s: Invalid --execute command %s\n"), + exec_name, quote (cmdopt)); + exit (WGET_EXIT_PARSE_ERROR); + } +} + +/* Generic helper functions, for use with `commands'. */ + +/* Forward declarations: */ +struct decode_item { + const char *name; + int code; +}; +static bool decode_string (const char *, const struct decode_item *, int, int *); +static bool simple_atof (const char *, const char *, double *); + +#define CMP1(p, c0) (c_tolower((p)[0]) == (c0) && (p)[1] == '\0') + +#define CMP2(p, c0, c1) (c_tolower((p)[0]) == (c0) \ + && c_tolower((p)[1]) == (c1) \ + && (p)[2] == '\0') + +#define CMP3(p, c0, c1, c2) (c_tolower((p)[0]) == (c0) \ + && c_tolower((p)[1]) == (c1) \ + && c_tolower((p)[2]) == (c2) \ + && (p)[3] == '\0') + + +static int +cmd_boolean_internal (const char *com _GL_UNUSED, const char *val, void *place _GL_UNUSED) +{ + if (CMP2 (val, 'o', 'n') || CMP3 (val, 'y', 'e', 's') || CMP1 (val, '1')) + /* "on", "yes" and "1" mean true. */ + return 1; + else if (CMP3 (val, 'o', 'f', 'f') || CMP2 (val, 'n', 'o') || CMP1 (val, '0')) + /* "off", "no" and "0" mean false. */ + return 0; + return -1; +} + +/* Store the boolean value from VAL to PLACE. COM is ignored, + except for error messages. */ +static bool +cmd_boolean (const char *com, const char *val, void *place) +{ + bool value; + + switch (cmd_boolean_internal (com, val, place)) + { + case 0: + value = false; + break; + + case 1: + value = true; + break; + + default: + { + fprintf (stderr, + _("%s: %s: Invalid boolean %s; use `on' or `off'.\n"), + exec_name, com, quote (val)); + return false; + } + } + *(bool *) place = value; + return true; +} + +/* Store the check_cert value from VAL to PLACE. COM is ignored, + except for error messages. */ +static bool +cmd_check_cert (const char *com, const char *val, void *place) +{ + int value; + + switch (cmd_boolean_internal (com, val, place)) + { + case 0: + value = CHECK_CERT_OFF; + break; + + case 1: + value = CHECK_CERT_ON; + break; + + default: + { + if (!c_strcasecmp (val, "quiet")) + value = CHECK_CERT_QUIET; + else + { + fprintf (stderr, + _("%s: %s: Invalid %s; use `on', `off' or `quiet'.\n"), + exec_name, com, quote (val)); + return false; + } + } + } + *(int *) place = value; + return true; +} + +/* Set the non-negative integer value from VAL to PLACE. With + incorrect specification, the number remains unchanged. */ +static bool +cmd_number (const char *com, const char *val, void *place) +{ + long l = strtol(val, NULL, 10); + + if (((l == LONG_MIN || l == LONG_MAX) && errno == ERANGE) + || l < 0 || l > INT_MAX) + { + fprintf (stderr, _("%s: %s: Invalid number %s.\n"), + exec_name, com, quote (val)); + return false; + } + *(int *) place = (int) l; + return true; +} + +/* Similar to cmd_number(), only accepts `inf' as a synonym for 0. */ +static bool +cmd_number_inf (const char *com, const char *val, void *place) +{ + if (!c_strcasecmp (val, "inf")) + { + *(int *) place = 0; + return true; + } + return cmd_number (com, val, place); +} + +/* Copy (strdup) the string at COM to a new location and place a + pointer to *PLACE. */ +static bool +cmd_string (const char *com _GL_UNUSED, const char *val, void *place) +{ + char **pstring = (char **)place; + + xfree (*pstring); + *pstring = xstrdup (val); + return true; +} + +/* Like cmd_string but ensure the string is upper case. */ +static bool +cmd_string_uppercase (const char *com _GL_UNUSED, const char *val, void *place) +{ + char *q, **pstring; + pstring = (char **)place; + xfree (*pstring); + + *pstring = xmalloc (strlen (val) + 1); + + for (q = *pstring; *val; val++, q++) + *q = c_toupper (*val); + + *q = '\0'; + return true; +} + + +/* Like cmd_string, but handles tilde-expansion when reading a user's + `.wgetrc'. In that case, and if VAL begins with `~', the tilde + gets expanded to the user's home directory. */ +static bool +cmd_file (const char *com _GL_UNUSED, const char *val, void *place) +{ + char **pstring = (char **)place; + + xfree (*pstring); + + /* #### If VAL is empty, perhaps should set *PLACE to NULL. */ + + *pstring = xstrdup (val); + +#if defined(WINDOWS) || defined(MSDOS) + /* Convert "\" to "/". */ + { + char *s; + for (s = *pstring; *s; s++) + if (*s == '\\') + *s = '/'; + } +#endif + return true; +} + +/* like cmd_file, but insist on just a single option usage */ +static bool +cmd_file_once (const char *com _GL_UNUSED, const char *val, void *place) +{ + if (*(char **)place) + { + fprintf (stderr, _("%s: %s must only be used once\n"), + exec_name, com); + return false; + } + + return cmd_file(com, val, place); +} + +/* Like cmd_file, but strips trailing '/' characters. */ +static bool +cmd_directory (const char *com, const char *val, void *place) +{ + char *s, *t; + + /* Call cmd_file() for tilde expansion and separator + canonicalization (backslash -> slash under Windows). These + things should perhaps be in a separate function. */ + if (!cmd_file (com, val, place)) + return false; + + s = *(char **)place; + t = s + strlen (s); + while (t > s && *--t == '/') + *t = '\0'; + + return true; +} + +/* Split VAL by space to a vector of values, and append those values + to vector pointed to by the PLACE argument. If VAL is empty, the + PLACE vector is cleared instead. */ + +static bool +cmd_vector (const char *com _GL_UNUSED, const char *val, void *place) +{ + char ***pvec = (char ***)place; + + if (*val) + *pvec = merge_vecs (*pvec, sepstring (val)); + else + { + free_vec (*pvec); + *pvec = NULL; + } + return true; +} + +static bool +cmd_directory_vector (const char *com _GL_UNUSED, const char *val, void *place) +{ + char ***pvec = (char ***)place; + + if (*val) + { + /* Strip the trailing slashes from directories. */ + char **t, **seps; + + seps = sepstring (val); + for (t = seps; t && *t; t++) + { + int len = strlen (*t); + /* Skip degenerate case of root directory. */ + if (len > 1) + { + if ((*t)[len - 1] == '/') + (*t)[len - 1] = '\0'; + } + } + *pvec = merge_vecs (*pvec, seps); + } + else + { + free_vec (*pvec); + *pvec = NULL; + } + return true; +} + +/* Engine for cmd_bytes and cmd_bytes_sum: converts a string such as + "100k" or "2.5G" to a floating point number. */ + +static bool +parse_bytes_helper (const char *val, double *result) +{ + double number, mult; + const char *end = val + strlen (val); + + /* Check for "inf". */ + if (0 == strcmp (val, "inf")) + { + *result = 0; + return true; + } + + /* Strip trailing whitespace. */ + while (val < end && c_isspace (end[-1])) + --end; + if (val == end) + return false; + + switch (c_tolower (end[-1])) + { + case 'k': + --end, mult = 1024.0; + break; + case 'm': + --end, mult = 1048576.0; + break; + case 'g': + --end, mult = 1073741824.0; + break; + case 't': + --end, mult = 1099511627776.0; + break; + default: + /* Not a recognized suffix: assume it's a digit. (If not, + simple_atof will raise an error.) */ + mult = 1; + } + + /* Skip leading and trailing whitespace. */ + while (val < end && c_isspace (*val)) + ++val; + while (val < end && c_isspace (end[-1])) + --end; + if (val == end) + return false; + + if (!simple_atof (val, end, &number) || number < 0) + return false; + + *result = number * mult; + return true; +} + +/* Parse VAL as a number and set its value to PLACE (which should + point to a wgint). + + By default, the value is assumed to be in bytes. If "K", "M", or + "G" are appended, the value is multiplied with 1<<10, 1<<20, or + 1<<30, respectively. Floating point values are allowed and are + cast to integer before use. The idea is to be able to use things + like 1.5k instead of "1536". + + The string "inf" is returned as 0. + + In case of error, false is returned and memory pointed to by PLACE + remains unmodified. */ + +static bool +cmd_bytes (const char *com, const char *val, void *place) +{ + double byte_value; + if (!parse_bytes_helper (val, &byte_value)) + { + fprintf (stderr, _("%s: %s: Invalid byte value %s\n"), + exec_name, com, quote (val)); + return false; + } + *(wgint *)place = (wgint)byte_value; + return true; +} + +/* Like cmd_bytes, but PLACE is interpreted as a pointer to + SIZE_SUM. It works by converting the string to double, therefore + working with values up to 2^53-1 without loss of precision. This + value (8192 TB) is large enough to serve for a while. */ + +static bool +cmd_bytes_sum (const char *com, const char *val, void *place) +{ + double byte_value; + + if (!parse_bytes_helper (val, &byte_value) + || byte_value < LONG_MIN || byte_value > LONG_MAX) + { + fprintf (stderr, _("%s: %s: Invalid byte value %s\n"), + exec_name, com, quote (val)); + return false; + } + *(SUM_SIZE_INT *) place = (SUM_SIZE_INT) byte_value; + return true; +} + +/* Store the value of VAL to *OUT. The value is a time period, by + default expressed in seconds, but also accepting suffixes "m", "h", + "d", and "w" for minutes, hours, days, and weeks respectively. */ + +static bool +cmd_time (const char *com, const char *val, void *place) +{ + double number, mult; + const char *end = val + strlen (val); + + /* Strip trailing whitespace. */ + while (val < end && c_isspace (end[-1])) + --end; + + if (val == end) + { + err: + fprintf (stderr, _("%s: %s: Invalid time period %s\n"), + exec_name, com, quote (val)); + return false; + } + + switch (c_tolower (end[-1])) + { + case 's': + --end, mult = 1; /* seconds */ + break; + case 'm': + --end, mult = 60; /* minutes */ + break; + case 'h': + --end, mult = 3600; /* hours */ + break; + case 'd': + --end, mult = 86400.0; /* days */ + break; + case 'w': + --end, mult = 604800.0; /* weeks */ + break; + default: + /* Not a recognized suffix: assume it belongs to the number. + (If not, simple_atof will raise an error.) */ + mult = 1; + } + + /* Skip leading and trailing whitespace. */ + while (val < end && c_isspace (*val)) + ++val; + while (val < end && c_isspace (end[-1])) + --end; + if (val == end) + goto err; + + if (!simple_atof (val, end, &number)) + goto err; + + if (number < 0) + { + fprintf (stderr, _("%s: %s: Negative time period %s\n"), + exec_name, com, quote (val)); + return false; + } + + *(double *)place = number * mult; + return true; +} + + +static bool +cmd_use_askpass (const char *com _GL_UNUSED, const char *val, void *place) +{ + const char *env_name = "WGET_ASKPASS"; + const char *env; + + if (val && *val) + return cmd_string (com, val, place); + + env = getenv (env_name); + if (!(env && *env)) + { + env_name = "SSH_ASKPASS"; + env = getenv (env_name); + } + + if (!(env && *env)) + { + fprintf (stderr, _("use-askpass requires a string or either environment variable WGET_ASKPASS or SSH_ASKPASS to be set.\n")); + return false; + } + + return cmd_string (com, env, place); +} + +#ifdef HAVE_SSL +static bool +cmd_cert_type (const char *com, const char *val, void *place) +{ + static const struct decode_item choices[] = { + { "pem", keyfile_pem }, + { "der", keyfile_asn1 }, + { "asn1", keyfile_asn1 }, + }; + int ok = decode_string (val, choices, countof (choices), place); + if (!ok) + fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val)); + return ok; +} +#endif + +/* Specialized helper functions, used by `commands' to handle some + options specially. */ + +static bool check_user_specified_header (const char *); + +#ifdef HAVE_LIBZ +static bool +cmd_spec_compression (const char *com, const char *val, void *place) +{ + static const struct decode_item choices[] = { + { "auto", compression_auto }, + { "gzip", compression_gzip }, + { "none", compression_none }, + }; + int ok = decode_string (val, choices, countof (choices), place); + if (!ok) + { + fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, + quote (val)); + } + return ok; +} +#endif + +static bool +cmd_spec_dirstruct (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + if (!cmd_boolean (com, val, &opt.dirstruct)) + return false; + /* Since dirstruct behaviour is explicitly changed, no_dirstruct + must be affected inversely. */ + if (opt.dirstruct) + opt.no_dirstruct = false; + else + opt.no_dirstruct = true; + return true; +} + +static bool +cmd_spec_header (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + /* Empty value means reset the list of headers. */ + if (*val == '\0') + { + free_vec (opt.user_headers); + opt.user_headers = NULL; + return true; + } + + if (!check_user_specified_header (val)) + { + fprintf (stderr, _("%s: %s: Invalid header %s.\n"), + exec_name, com, quote (val)); + return false; + } + opt.user_headers = vec_append (opt.user_headers, val); + return true; +} + +static bool +cmd_spec_warc_header (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + /* Empty value means reset the list of headers. */ + if (*val == '\0') + { + free_vec (opt.warc_user_headers); + opt.warc_user_headers = NULL; + return true; + } + + if (!check_user_specified_header (val)) + { + fprintf (stderr, _("%s: %s: Invalid WARC header %s.\n"), + exec_name, com, quote (val)); + return false; + } + opt.warc_user_headers = vec_append (opt.warc_user_headers, val); + return true; +} + +static bool +cmd_spec_htmlify (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + int flag = cmd_boolean (com, val, &opt.htmlify); + if (flag && !opt.htmlify) + opt.remove_listing = false; + return flag; +} + +/* Set the "mirror" mode. It means: recursive download, timestamping, + no limit on max. recursion depth, and don't remove listings. */ + +static bool +cmd_spec_mirror (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + bool mirror; + + if (!cmd_boolean (com, val, &mirror)) + return false; + if (mirror) + { + opt.recursive = true; + if (!opt.no_dirstruct) + opt.dirstruct = true; + opt.timestamping = true; + opt.reclevel = INFINITE_RECURSION; + opt.remove_listing = false; + } + return true; +} + +/* Validate --prefer-family and set the choice. Allowed values are + "IPv4", "IPv6", and "none". */ + +static bool +cmd_spec_prefer_family (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + static const struct decode_item choices[] = { + { "IPv4", prefer_ipv4 }, + { "IPv6", prefer_ipv6 }, + { "none", prefer_none }, + }; + int prefer_family = prefer_none; + int ok = decode_string (val, choices, countof (choices), &prefer_family); + if (!ok) + fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val)); + opt.prefer_family = prefer_family; + return ok; +} + +/* Set progress.type to VAL, but verify that it's a valid progress + implementation before that. */ + +static bool +cmd_spec_progress (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + if (!valid_progress_implementation_p (val)) + { + fprintf (stderr, _("%s: %s: Invalid progress type %s.\n"), + exec_name, com, quote (val)); + return false; + } + xfree (opt.progress_type); + + /* Don't call set_progress_implementation here. It will be called + in main when it becomes clear what the log output is. */ + opt.progress_type = xstrdup (val); + return true; +} + +/* Set opt.recursive to VAL as with cmd_boolean. If opt.recursive is + set to true, also set opt.dirstruct to true, unless opt.no_dirstruct + is specified. */ + +static bool +cmd_spec_recursive (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + if (!cmd_boolean (com, val, &opt.recursive)) + return false; + else + { + if (opt.recursive && !opt.no_dirstruct) + opt.dirstruct = true; + } + return true; +} + +/* Validate --regex-type and set the choice. */ + +static bool +cmd_spec_regex_type (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + static const struct decode_item choices[] = { + { "posix", regex_type_posix }, +#if defined HAVE_LIBPCRE || defined HAVE_LIBPCRE2 + { "pcre", regex_type_pcre }, +#endif + }; + int regex_type = regex_type_posix; + int ok = decode_string (val, choices, countof (choices), ®ex_type); + if (!ok) + fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val)); + opt.regex_type = regex_type; + return ok; +} + +static bool +cmd_spec_restrict_file_names (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + int restrict_os = opt.restrict_files_os; + int restrict_ctrl = opt.restrict_files_ctrl; + int restrict_case = opt.restrict_files_case; + int restrict_nonascii = opt.restrict_files_nonascii; + + const char *end; + +#define VAL_IS(string_literal) BOUNDED_EQUAL (val, end, string_literal) + + do + { + end = strchr (val, ','); + if (!end) + end = val + strlen (val); + + if (VAL_IS ("unix")) + restrict_os = restrict_unix; + else if (VAL_IS ("vms")) + restrict_os = restrict_vms; + else if (VAL_IS ("windows")) + restrict_os = restrict_windows; + else if (VAL_IS ("lowercase")) + restrict_case = restrict_lowercase; + else if (VAL_IS ("uppercase")) + restrict_case = restrict_uppercase; + else if (VAL_IS ("nocontrol")) + restrict_ctrl = false; + else if (VAL_IS ("ascii")) + restrict_nonascii = true; + else + { + fprintf (stderr, _("\ +%s: %s: Invalid restriction %s,\n\ + use [unix|vms|windows],[lowercase|uppercase],[nocontrol],[ascii].\n"), + exec_name, com, quote (val)); + return false; + } + + if (*end) + val = end + 1; + } + while (*val && *end); + +#undef VAL_IS + + opt.restrict_files_os = restrict_os; + opt.restrict_files_ctrl = restrict_ctrl; + opt.restrict_files_case = restrict_case; + opt.restrict_files_nonascii = restrict_nonascii; + + return true; +} + +static bool +cmd_spec_report_speed (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + opt.report_bps = c_strcasecmp (val, "bits") == 0; + if (!opt.report_bps) + fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val)); + return opt.report_bps; +} + +#ifdef HAVE_SSL +static bool +cmd_spec_secure_protocol (const char *com, const char *val, void *place) +{ + static const struct decode_item choices[] = { + { "auto", secure_protocol_auto }, + { "sslv2", secure_protocol_sslv2 }, + { "sslv3", secure_protocol_sslv3 }, + { "tlsv1", secure_protocol_tlsv1 }, + { "tlsv1_1", secure_protocol_tlsv1_1 }, + { "tlsv1_2", secure_protocol_tlsv1_2 }, + { "tlsv1_3", secure_protocol_tlsv1_3 }, + { "pfs", secure_protocol_pfs }, + }; + int ok = decode_string (val, choices, countof (choices), place); + if (!ok) + fprintf (stderr, _("%s: %s: Invalid value %s.\n"), exec_name, com, quote (val)); + return ok; +} +#endif + +/* Set all three timeout values. */ + +static bool +cmd_spec_timeout (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + double value; + if (!cmd_time (com, val, &value)) + return false; + opt.read_timeout = value; + opt.connect_timeout = value; + opt.dns_timeout = value; + return true; +} + +static bool +cmd_spec_useragent (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + /* Disallow embedded newlines. */ + if (strchr (val, '\n')) + { + fprintf (stderr, _("%s: %s: Invalid value %s.\n"), + exec_name, com, quote (val)); + return false; + } + xfree (opt.useragent); + opt.useragent = xstrdup (val); + return true; +} + +/* The --show-progress option is not a cmd_boolean since we need to keep track + * of whether the user explicitly requested the option or not. -1 means + * uninitialized. */ +static bool +cmd_spec_progressdisp (const char *com, const char *val, void *place _GL_UNUSED) +{ + bool flag; + if (cmd_boolean (com, val, &flag)) + { + opt.show_progress = flag; + return true; + } + return false; +} + + +/* The "verbose" option cannot be cmd_boolean because the variable is + not bool -- it's of type int (-1 means uninitialized because of + some random hackery for disallowing -q -v). */ + +static bool +cmd_spec_verbose (const char *com, const char *val, void *place_ignored _GL_UNUSED) +{ + bool flag; + if (cmd_boolean (com, val, &flag)) + { + opt.verbose = flag; + opt.show_progress = -1; + return true; + } + return false; +} + +/* Miscellaneous useful routines. */ + +/* Trivial atof, with error reporting. Handles "<digits>[.<digits>]", + doesn't handle exponential notation. Returns true on success, + false on failure. In case of success, stores its result to + *DEST. */ + +static bool +simple_atof (const char *beg, const char *end, double *dest) +{ + double result = 0; + + bool negative = false; + bool seen_dot = false; + bool seen_digit = false; + double divider = 1; + + const char *p = beg; + + while (p < end && c_isspace (*p)) + ++p; + if (p < end && (*p == '-' || *p == '+')) + { + negative = (*p == '-'); + ++p; + } + + for (; p < end; p++) + { + char ch = *p; + if (c_isdigit (ch)) + { + if (!seen_dot) + result = (10 * result) + (ch - '0'); + else + result += (ch - '0') / (divider *= 10); + seen_digit = true; + } + else if (ch == '.') + { + if (!seen_dot) + seen_dot = true; + else + return false; + } + else + return false; + } + if (!seen_digit) + return false; + if (negative) + result = -result; + + *dest = result; + return true; +} + +/* Verify that the user-specified header in S is valid. It must + contain a colon preceded by non-white-space characters and must not + contain newlines. */ + +static bool +check_user_specified_header (const char *s) +{ + const char *p; + + for (p = s; *p && *p != ':' && !c_isspace (*p); p++) + ; + /* The header MUST contain `:' preceded by at least one + non-whitespace character. */ + if (*p != ':' || p == s) + return false; + /* The header MUST NOT contain newlines. */ + if (strchr (s, '\n')) + return false; + return true; +} + +/* Decode VAL into a number, according to ITEMS. */ + +static bool +decode_string (const char *val, const struct decode_item *items, int itemcount, + int *place) +{ + int i; + for (i = 0; i < itemcount; i++) + if (0 == c_strcasecmp (val, items[i].name)) + { + *place = items[i].code; + return true; + } + return false; +} + +extern struct ptimer *timer; +extern int cleaned_up; + +/* Free the memory allocated by global variables. */ +void +cleanup (void) +{ + /* Free external resources, close files, etc. */ + + if (cleaned_up++) + return; /* cleanup() must not be called twice */ + + /* Close WARC file. */ + if (opt.warc_filename != 0) + warc_close (); + + log_close (); + + if (output_stream && output_stream != stderr) + { + FILE *fp = output_stream; + output_stream = NULL; + if (fclose (fp) == EOF) + inform_exit_status (CLOSEFAILED); + } + + /* No need to check for error because Wget flushes its output (and + checks for errors) after any data arrives. */ + + /* We're exiting anyway so there's no real need to call free() + hundreds of times. Skipping the frees will make Wget exit + faster. + * + However, when detecting leaks, it's crucial to free() everything + because then you can find the real leaks, i.e. the allocated + memory which grows with the size of the program. */ + +#if defined DEBUG_MALLOC || defined TESTING + convert_cleanup (); + res_cleanup (); + http_cleanup (); + cleanup_html_url (); + spider_cleanup (); + host_cleanup (); + log_cleanup (); + netrc_cleanup (); +#ifdef HAVE_SSL + ssl_cleanup (); +#endif + connect_cleanup (); + + xfree (opt.choose_config); + xfree (opt.lfilename); + xfree (opt.dir_prefix); + xfree (opt.input_filename); +#ifdef HAVE_METALINK + xfree (opt.input_metalink); + xfree (opt.preferred_location); +#endif + xfree (opt.output_document); + xfree (opt.default_page); + if (opt.regex_type == regex_type_posix) + { + if (opt.acceptregex) + regfree (opt.acceptregex); + if (opt.rejectregex) + regfree (opt.rejectregex); + } + xfree (opt.acceptregex); + xfree (opt.rejectregex); + xfree (opt.acceptregex_s); + xfree (opt.rejectregex_s); + free_vec (opt.accepts); + free_vec (opt.rejects); + free_vec ((char **)opt.excludes); + free_vec ((char **)opt.includes); + free_vec (opt.domains); + free_vec (opt.exclude_domains); + free_vec (opt.follow_tags); + free_vec (opt.ignore_tags); + xfree (opt.progress_type); + xfree (opt.warc_filename); + xfree (opt.warc_tempdir); + xfree (opt.warc_cdx_dedup_filename); + xfree (opt.ftp_user); + xfree (opt.ftp_passwd); + xfree (opt.ftp_proxy); + xfree (opt.https_proxy); + xfree (opt.http_proxy); + free_vec (opt.no_proxy); + xfree (opt.proxy_user); + xfree (opt.proxy_passwd); + xfree (opt.useragent); + xfree (opt.referer); + xfree (opt.http_user); + xfree (opt.http_passwd); + xfree (opt.dot_style); + free_vec (opt.user_headers); + free_vec (opt.warc_user_headers); +# ifdef HAVE_SSL + xfree (opt.cert_file); + xfree (opt.private_key); + xfree (opt.ca_directory); + xfree (opt.ca_cert); + xfree (opt.crl_file); + xfree (opt.pinnedpubkey); + xfree (opt.random_file); + xfree (opt.egd_file); +# endif + xfree (opt.bind_address); + xfree (opt.cookies_input); + xfree (opt.cookies_output); + xfree (opt.user); + xfree (opt.passwd); + xfree (opt.base_href); + xfree (opt.method); + xfree (opt.post_file_name); + xfree (opt.post_data); + xfree (opt.body_data); + xfree (opt.body_file); + xfree (opt.rejected_log); + xfree (opt.use_askpass); + xfree (opt.retry_on_http_error); + + xfree (opt.encoding_remote); + xfree (opt.locale); +#ifdef HAVE_HSTS + xfree (opt.hsts_file); +#endif + + xfree (opt.wgetrcfile); + xfree (opt.homedir); + xfree (exec_name); + xfree (program_argstring); + ptimer_destroy (timer); timer = NULL; + +#ifdef HAVE_LIBCARES +#include <ares.h> + { + extern ares_channel ares; + + xfree (opt.bind_dns_address); + xfree (opt.dns_servers); + ares_destroy (ares); + ares_library_cleanup (); + } +#endif + + quotearg_free (); + +#endif /* DEBUG_MALLOC || TESTING */ +} + +/* Unit testing routines. */ + +#ifdef TESTING + +const char * +test_commands_sorted(void) +{ + unsigned i; + + for (i = 1; i < countof(commands); ++i) + { + if (c_strcasecmp (commands[i - 1].name, commands[i].name) > 0) + { + mu_assert ("FAILED", false); + break; + } + } + return NULL; +} + +const char * +test_cmd_spec_restrict_file_names(void) +{ + unsigned i; + static const struct { + const char *val; + int expected_restrict_files_os; + bool expected_restrict_files_ctrl; + int expected_restrict_files_case; + bool result; + } test_array[] = { + { "windows", restrict_windows, true, restrict_no_case_restriction, true }, + { "windows,", restrict_windows, true, restrict_no_case_restriction, true }, + { "windows,lowercase", restrict_windows, true, restrict_lowercase, true }, + { "unix,nocontrol,lowercase,", restrict_unix, false, restrict_lowercase, true }, + }; + + for (i = 0; i < countof(test_array); ++i) + { + bool res; + + defaults(); + res = cmd_spec_restrict_file_names ("dummy", test_array[i].val, NULL); + + /* + fprintf (stderr, "test_cmd_spec_restrict_file_names: TEST %d\n", i); fflush (stderr); + fprintf (stderr, "opt.restrict_files_os: %d\n", opt.restrict_files_os); fflush (stderr); + fprintf (stderr, "opt.restrict_files_ctrl: %d\n", opt.restrict_files_ctrl); fflush (stderr); + fprintf (stderr, "opt.restrict_files_case: %d\n", opt.restrict_files_case); fflush (stderr); + */ + mu_assert ("test_cmd_spec_restrict_file_names: wrong result", + res == test_array[i].result + && (int) opt.restrict_files_os == test_array[i].expected_restrict_files_os + && opt.restrict_files_ctrl == test_array[i].expected_restrict_files_ctrl + && (int) opt.restrict_files_case == test_array[i].expected_restrict_files_case); + } + + return NULL; +} + +#endif /* TESTING */ diff --git a/src/init.h b/src/init.h new file mode 100644 index 0000000..a7619ce --- /dev/null +++ b/src/init.h @@ -0,0 +1,48 @@ +/* Declarations for init.c. + Copyright (C) 1996-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef INIT_H +#define INIT_H + +char *ajoin_dir_file (const char *, const char *); +char *wgetrc_env_file_name (void); +char *wgetrc_user_file_name (void); +char *wgetrc_file_name (void); +int initialize (void); +void run_command (const char *); +void setoptval (const char *, const char *, const char *); +char *home_dir (void); +void cleanup (void); +void defaults (void); +bool run_wgetrc (const char *file, file_stats_t *); + +#define MAX_LONGOPTION 26 + +#endif /* INIT_H */ diff --git a/src/iri.c b/src/iri.c new file mode 100644 index 0000000..afdadf7 --- /dev/null +++ b/src/iri.c @@ -0,0 +1,449 @@ +/* IRI related functions. + Copyright (C) 2008-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <langinfo.h> +#include <errno.h> +#ifdef HAVE_ICONV +# include <iconv.h> +#endif +#include <idn2.h> +#if IDN2_VERSION_NUMBER < 0x00140000 +# include <unicase.h> +# include <unistr.h> +#endif + +#include "utils.h" +#include "url.h" +#include "c-strcase.h" +#include "c-strcasestr.h" +#include "xstrndup.h" + +/* Note: locale encoding is kept in options struct (opt.locale) */ + +/* Given a string containing "charset=XXX", return the encoding if found, + or NULL otherwise */ +char * +parse_charset (const char *str) +{ + const char *end; + char *charset; + + if (!str || !*str) + return NULL; + + str = c_strcasestr (str, "charset="); + if (!str) + return NULL; + + str += 8; + end = str; + + /* sXXXav: which chars should be banned ??? */ + while (*end && !c_isspace (*end)) + end++; + + /* sXXXav: could strdupdelim return NULL ? */ + charset = strdupdelim (str, end); + + /* Do a minimum check on the charset value */ + if (!check_encoding_name (charset)) + { + xfree (charset); + return NULL; + } + + /*logprintf (LOG_VERBOSE, "parse_charset: %s\n", quote (charset));*/ + + return charset; +} + +/* Find the locale used, or fall back on a default value */ +const char * +find_locale (void) +{ + const char *encoding = nl_langinfo(CODESET); + + if (!encoding || !*encoding) + return xstrdup("ASCII"); + + return xstrdup(encoding); +} + +/* Basic check of an encoding name. */ +bool +check_encoding_name (const char *encoding) +{ + const char *s = encoding; + + while (*s) + { + if (!c_isascii (*s) || c_isspace (*s)) + { + logprintf (LOG_VERBOSE, _("Encoding %s isn't valid\n"), quote (encoding)); + return false; + } + + s++; + } + + return true; +} + +#ifdef HAVE_ICONV +/* Do the conversion according to the passed conversion descriptor cd. *out + will contain the transcoded string on success. *out content is + unspecified otherwise. */ +static bool +do_conversion (const char *tocode, const char *fromcode, char const *in_org, size_t inlen, char **out) +{ + iconv_t cd; + /* sXXXav : hummm hard to guess... */ + size_t len, done, outlen; + int invalid = 0, tooshort = 0; + char *s, *in, *in_save; + + cd = iconv_open (tocode, fromcode); + if (cd == (iconv_t)(-1)) + { + logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"), + quote (fromcode), quote (tocode)); + *out = NULL; + return false; + } + + /* iconv() has to work on an unescaped string */ + in_save = in = xstrndup (in_org, inlen); + url_unescape_except_reserved (in); + inlen = strlen(in); + + len = outlen = inlen * 2; + *out = s = xmalloc (outlen + 1); + done = 0; + + for (;;) + { + if (iconv (cd, (ICONV_CONST char **) &in, &inlen, out, &outlen) != (size_t)(-1) && + iconv (cd, NULL, NULL, out, &outlen) != (size_t)(-1)) + { + *out = s; + *(s + len - outlen - done) = '\0'; + xfree(in_save); + iconv_close(cd); + IF_DEBUG + { + /* not not print out embedded passwords, in_org might be an URL */ + if (!strchr(in_org, '@') && !strchr(*out, '@')) + debug_logprintf ("converted '%s' (%s) -> '%s' (%s)\n", in_org, fromcode, *out, tocode); + else + debug_logprintf ("logging suppressed, strings may contain password\n"); + } + return true; + } + + /* Incomplete or invalid multibyte sequence */ + if (errno == EINVAL || errno == EILSEQ) + { + if (!invalid) + logprintf (LOG_VERBOSE, + _("Incomplete or invalid multibyte sequence encountered\n")); + + invalid++; + **out = *in; + in++; + inlen--; + (*out)++; + outlen--; + } + else if (errno == E2BIG) /* Output buffer full */ + { + tooshort++; + done = len; + len = done + inlen * 2; + s = xrealloc (s, len + 1); + *out = s + done - outlen; + outlen += inlen * 2; + } + else /* Weird, we got an unspecified error */ + { + logprintf (LOG_VERBOSE, _("Unhandled errno %d\n"), errno); + break; + } + } + + xfree(in_save); + iconv_close(cd); + IF_DEBUG + { + /* not not print out embedded passwords, in_org might be an URL */ + if (!strchr(in_org, '@') && !strchr(*out, '@')) + debug_logprintf ("converted '%s' (%s) -> '%s' (%s)\n", in_org, fromcode, *out, tocode); + else + debug_logprintf ("logging suppressed, strings may contain password\n"); + } + return false; +} +#else +static bool +do_conversion (const char *tocode _GL_UNUSED, const char *fromcode _GL_UNUSED, + char const *in_org _GL_UNUSED, size_t inlen _GL_UNUSED, char **out) +{ + *out = NULL; + return false; +} +#endif + +/* Try converting string str from locale to UTF-8. Return a new string + on success, or str on error or if conversion isn't needed. */ +const char * +locale_to_utf8 (const char *str) +{ + char *new; + + /* That shouldn't happen, just in case */ + if (!opt.locale) + { + logprintf (LOG_VERBOSE, _("locale_to_utf8: locale is unset\n")); + opt.locale = find_locale (); + } + + if (!opt.locale || !c_strcasecmp (opt.locale, "utf-8")) + return str; + + if (do_conversion ("UTF-8", opt.locale, (char *) str, strlen ((char *) str), &new)) + return (const char *) new; + + xfree (new); + return str; +} + +/* Try to "ASCII encode" UTF-8 host. Return the new domain on success or NULL + on error. */ +char * +idn_encode (const struct iri *i, const char *host) +{ + int ret; + char *ascii_encoded; + char *utf8_encoded = NULL; + const char *src; +#if IDN2_VERSION_NUMBER < 0x00140000 + uint8_t *lower; + size_t len = 0; +#endif + + /* Encode to UTF-8 if not done */ + if (!i->utf8_encode) + { + if (!remote_to_utf8 (i, host, &utf8_encoded)) + return NULL; /* Nothing to encode or an error occurred */ + src = utf8_encoded; + } + else + src = host; + +#if IDN2_VERSION_NUMBER >= 0x00140000 + /* IDN2_TRANSITIONAL implies input NFC encoding */ + ret = idn2_lookup_u8 ((uint8_t *) src, (uint8_t **) &ascii_encoded, IDN2_NONTRANSITIONAL); + if (ret != IDN2_OK) + /* fall back to TR46 Transitional mode, max IDNA2003 compatibility */ + ret = idn2_lookup_u8 ((uint8_t *) src, (uint8_t **) &ascii_encoded, IDN2_TRANSITIONAL); + + if (ret != IDN2_OK) + logprintf (LOG_VERBOSE, _("idn_encode failed (%d): %s\n"), ret, + quote (idn2_strerror (ret))); +#else + /* we need a conversion to lowercase */ + lower = u8_tolower ((uint8_t *) src, u8_strlen ((uint8_t *) src) + 1, 0, UNINORM_NFKC, NULL, &len); + if (!lower) + { + logprintf (LOG_VERBOSE, _("Failed to convert to lower: %d: %s\n"), + errno, quote (src)); + xfree (utf8_encoded); + return NULL; + } + + if ((ret = idn2_lookup_u8 (lower, (uint8_t **) &ascii_encoded, IDN2_NFC_INPUT)) != IDN2_OK) + { + logprintf (LOG_VERBOSE, _("idn_encode failed (%d): %s\n"), ret, + quote (idn2_strerror (ret))); + } + + xfree (lower); +#endif + + xfree (utf8_encoded); + + if (ret == IDN2_OK && ascii_encoded) + { + char *tmp = xstrdup (ascii_encoded); + idn2_free (ascii_encoded); + ascii_encoded = tmp; + } + + return ret == IDN2_OK ? ascii_encoded : NULL; +} + +/* Try to decode an "ASCII encoded" host. Return the new domain in the locale + on success or NULL on error. */ +char * +idn_decode (const char *host) +{ +/* + char *new; + int ret; + + ret = idn2_register_u8 (NULL, host, (uint8_t **) &new, 0); + if (ret != IDN2_OK) + { + logprintf (LOG_VERBOSE, _("idn2_register_u8 failed (%d): %s: %s\n"), ret, + quote (idn2_strerror (ret)), host); + return NULL; + } + + return new; +*/ + /* idn2_register_u8() just works label by label. + * That is pretty much overhead for just displaying the original ulabels. + * To keep at least the debug output format, return a cloned host. */ + return xstrdup(host); +} + +/* Try to transcode string str from remote encoding to UTF-8. On success, *new + contains the transcoded string. *new content is unspecified otherwise. */ +bool +remote_to_utf8 (const struct iri *iri, const char *str, char **new) +{ + bool ret = false; + + if (!iri->uri_encoding) + return false; + + /* When `i->uri_encoding' == "UTF-8" there is nothing to convert. But we must + test for non-ASCII symbols for correct hostname processing in `idn_encode' + function. */ + if (!c_strcasecmp (iri->uri_encoding, "UTF-8")) + { + const unsigned char *p; + for (p = (unsigned char *) str; *p; p++) + if (*p > 127) + { + *new = strdup (str); + return true; + } + return false; + } + + if (do_conversion ("UTF-8", iri->uri_encoding, str, strlen (str), new)) + ret = true; + + /* Test if something was converted */ + if (*new && !strcmp (str, *new)) + { + xfree (*new); + return false; + } + + return ret; +} + +/* Allocate a new iri structure and return a pointer to it. */ +struct iri * +iri_new (void) +{ + struct iri *i = xmalloc (sizeof *i); + i->uri_encoding = opt.encoding_remote ? xstrdup (opt.encoding_remote) : NULL; + i->content_encoding = NULL; + i->orig_url = NULL; + i->utf8_encode = opt.enable_iri; + return i; +} + +struct iri *iri_dup (const struct iri *src) +{ + struct iri *i = xmalloc (sizeof *i); + i->uri_encoding = src->uri_encoding ? xstrdup (src->uri_encoding) : NULL; + i->content_encoding = (src->content_encoding ? + xstrdup (src->content_encoding) : NULL); + i->orig_url = src->orig_url ? xstrdup (src->orig_url) : NULL; + i->utf8_encode = src->utf8_encode; + return i; +} + +/* Completely free an iri structure. */ +void +iri_free (struct iri *i) +{ + if (i) + { + xfree (i->uri_encoding); + xfree (i->content_encoding); + xfree (i->orig_url); + xfree (i); + } +} + +/* Set uri_encoding of struct iri i. If a remote encoding was specified, use + it unless force is true. */ +void +set_uri_encoding (struct iri *i, const char *charset, bool force) +{ + DEBUGP (("URI encoding = %s\n", charset ? quote (charset) : "None")); + if (!force && opt.encoding_remote) + return; + if (i->uri_encoding) + { + if (charset && !c_strcasecmp (i->uri_encoding, charset)) + return; + xfree (i->uri_encoding); + } + + i->uri_encoding = charset ? xstrdup (charset) : NULL; +} + +/* Set content_encoding of struct iri i. */ +void +set_content_encoding (struct iri *i, const char *charset) +{ + DEBUGP (("URI content encoding = %s\n", charset ? quote (charset) : "None")); + if (opt.encoding_remote) + return; + if (i->content_encoding) + { + if (charset && !c_strcasecmp (i->content_encoding, charset)) + return; + xfree (i->content_encoding); + } + + i->content_encoding = charset ? xstrdup (charset) : NULL; +} diff --git a/src/iri.h b/src/iri.h new file mode 100644 index 0000000..c6b75e8 --- /dev/null +++ b/src/iri.h @@ -0,0 +1,75 @@ +/* Internationalization related declarations. + Copyright (C) 2008-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef IRI_H +#define IRI_H + +struct iri { + char *uri_encoding; /* Encoding of the uri to fetch */ + char *content_encoding; /* Encoding of links inside the fetched file */ + char *orig_url; /* */ + bool utf8_encode; /* Will/Is the current url encoded in utf8 */ +}; + +#ifdef ENABLE_IRI + +char *parse_charset (const char *str); +const char *find_locale (void); +bool check_encoding_name (const char *encoding); +const char *locale_to_utf8 (const char *str); +char *idn_encode (const struct iri *i, const char *host); +char *idn_decode (const char *host); +bool remote_to_utf8 (const struct iri *i, const char *str, char **newstr); +struct iri *iri_new (void); +struct iri *iri_dup (const struct iri *); +void iri_free (struct iri *i); +void set_uri_encoding (struct iri *i, const char *charset, bool force); +void set_content_encoding (struct iri *i, const char *charset); + +#else /* ENABLE_IRI */ + +extern struct iri dummy_iri; + +#define parse_charset(str) NULL +#define find_locale() NULL +#define check_encoding_name(str) false +#define locale_to_utf8(str) (str) +#define idn_encode(a,b) NULL +#define idn_decode(str) NULL +#define idn2_free(str) ((void)0) +#define remote_to_utf8(a,b,c) false +#define iri_new() (&dummy_iri) +#define iri_dup(a) (&dummy_iri) +#define iri_free(a) +#define set_uri_encoding(a,b,c) +#define set_content_encoding(a,b) + +#endif /* ENABLE_IRI */ +#endif /* IRI_H */ diff --git a/src/log.c b/src/log.c new file mode 100644 index 0000000..7de23ee --- /dev/null +++ b/src/log.c @@ -0,0 +1,995 @@ +/* Messages logging. + Copyright (C) 1998-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <stdarg.h> +#include <unistd.h> +#include <assert.h> +#include <errno.h> + +#include "utils.h" +#include "exits.h" +#include "log.h" + +/* 2005-10-25 SMS. + VMS log files are often VFC record format, not stream, so fputs() can + produce multiple records, even when there's no newline terminator in + the buffer. The result is unsightly output with spurious newlines. + Using fprintf() instead of fputs(), along with inhibiting some + fflush() activity below, seems to solve the problem. +*/ +#ifdef __VMS +# define FPUTS( s, f) fprintf( (f), "%s", (s)) +#else /* def __VMS */ +# define FPUTS( s, f) fputs( (s), (f)) +#endif /* def __VMS [else] */ + +/* This file implements support for "logging". Logging means printing + output, plus several additional features: + + - Cataloguing output by importance. You can specify that a log + message is "verbose" or "debug", and it will not be printed unless + in verbose or debug mode, respectively. + + - Redirecting the log to the file. When Wget's output goes to the + terminal, and Wget receives SIGHUP, all further output is + redirected to a log file. When this is the case, Wget can also + print the last several lines of "context" to the log file so that + it does not begin in the middle of a line. For this to work, the + logging code stores the last several lines of context. Callers may + request for certain output not to be stored. + + - Inhibiting output. When Wget receives SIGHUP, but redirecting + the output fails, logging is inhibited. */ + + +/* The file descriptor used for logging. This is NULL before log_init + is called; logging functions log to stderr then. log_init sets it + either to stderr or to a file pointer obtained from fopen(). If + logging is inhibited, logfp is set back to NULL. */ +static FILE *logfp; + +/* Descriptor of the stdout|stderr */ +static FILE *stdlogfp; + +/* Descriptor of the wget.log* file (if created) */ +static FILE *filelogfp; + +/* Name of log file */ +static char *logfile; + +/* Is interactive shell ? */ +static int shell_is_interactive; + +/* A second file descriptor pointing to the temporary log file for the + WARC writer. If WARC writing is disabled, this is NULL. */ +static FILE *warclogfp; + +/* If true, it means logging is inhibited, i.e. nothing is printed or + stored. */ +static bool inhibit_logging; + +/* Whether the last output lines are stored for use as context. */ +static bool save_context_p; + +/* Whether the log is flushed after each command. */ +static bool flush_log_p = true; + +/* Whether any output has been received while flush_log_p was 0. */ +static bool needs_flushing; + +/* In the event of a hang-up, and if its output was on a TTY, Wget + redirects its output to `wget-log'. + + For the convenience of reading this newly-created log, we store the + last several lines ("screenful", hence the choice of 24) of Wget + output, and dump them as context when the time comes. */ +#define SAVED_LOG_LINES 24 + +/* log_lines is a circular buffer that stores SAVED_LOG_LINES lines of + output. log_line_current always points to the position in the + buffer that will be written to next. When log_line_current reaches + SAVED_LOG_LINES, it is reset to zero. + + The problem here is that we'd have to either (re)allocate and free + strings all the time, or limit the lines to an arbitrary number of + characters. Instead of settling for either of these, we do both: + if the line is smaller than a certain "usual" line length (128 + chars by default), a preallocated memory is used. The rare lines + that are longer than 128 characters are malloc'ed and freed + separately. This gives good performance with minimum memory + consumption and fragmentation. */ + +#define STATIC_LENGTH 128 + +static struct log_ln { + char static_line[STATIC_LENGTH + 1]; /* statically allocated + line. */ + char *malloced_line; /* malloc'ed line, for lines of output + larger than 80 characters. */ + char *content; /* this points either to malloced_line + or to the appropriate static_line. + If this is NULL, it means the line + has not yet been used. */ +} log_lines[SAVED_LOG_LINES]; + +/* The current position in the ring. */ +static int log_line_current = -1; + +/* Whether the most recently written line was "trailing", i.e. did not + finish with \n. This is an important piece of information because + the code is always careful to append data to trailing lines, rather + than create new ones. */ +static bool trailing_line; + +static void check_redirect_output (void); + +#define ROT_ADVANCE(num) do { \ + if (++num >= SAVED_LOG_LINES) \ + num = 0; \ +} while (0) + +/* Free the log line index with NUM. This calls free on + ln->malloced_line if it's non-NULL, and it also resets + ln->malloced_line and ln->content to NULL. */ + +static void +free_log_line (int num) +{ + struct log_ln *ln = log_lines + num; + xfree (ln->malloced_line); + ln->content = NULL; +} + +/* Append bytes in the range [start, end) to one line in the log. The + region is not supposed to contain newlines, except for the last + character (at end[-1]). */ + +static void +saved_append_1 (const char *start, const char *end) +{ + int len = end - start; + if (!len) + return; + + /* First, check whether we need to append to an existing line or to + create a new one. */ + if (!trailing_line) + { + /* Create a new line. */ + struct log_ln *ln; + + if (log_line_current == -1) + log_line_current = 0; + else + free_log_line (log_line_current); + ln = log_lines + log_line_current; + if (len > STATIC_LENGTH) + { + ln->malloced_line = strdupdelim (start, end); + ln->content = ln->malloced_line; + } + else + { + memcpy (ln->static_line, start, len); + ln->static_line[len] = '\0'; + ln->content = ln->static_line; + } + } + else + { + /* Append to the last line. If the line is malloc'ed, we just + call realloc and append the new string. If the line is + static, we have to check whether appending the new string + would make it exceed STATIC_LENGTH characters, and if so, + convert it to malloc(). */ + struct log_ln *ln = log_lines + log_line_current; + if (ln->malloced_line) + { + /* Resize malloc'ed line and append. */ + int old_len = strlen (ln->malloced_line); + ln->malloced_line = xrealloc (ln->malloced_line, old_len + len + 1); + memcpy (ln->malloced_line + old_len, start, len); + ln->malloced_line[old_len + len] = '\0'; + /* might have changed due to realloc */ + ln->content = ln->malloced_line; + } + else + { + int old_len = strlen (ln->static_line); + if (old_len + len > STATIC_LENGTH) + { + /* Allocate memory and concatenate the old and the new + contents. */ + ln->malloced_line = xmalloc (old_len + len + 1); + memcpy (ln->malloced_line, ln->static_line, + old_len); + memcpy (ln->malloced_line + old_len, start, len); + ln->malloced_line[old_len + len] = '\0'; + ln->content = ln->malloced_line; + } + else + { + /* Just append to the old, statically allocated + contents. */ + memcpy (ln->static_line + old_len, start, len); + ln->static_line[old_len + len] = '\0'; + ln->content = ln->static_line; + } + } + } + trailing_line = !(end[-1] == '\n'); + if (!trailing_line) + ROT_ADVANCE (log_line_current); +} + +/* Log the contents of S, as explained above. If S consists of + multiple lines, they are logged separately. If S does not end with + a newline, it will form a "trailing" line, to which things will get + appended the next time this function is called. */ + +static void +saved_append (const char *s) +{ + while (*s) + { + const char *end = strchr (s, '\n'); + if (!end) + end = s + strlen (s); + else + ++end; + saved_append_1 (s, end); + s = end; + } +} + +/* Check X against opt.verbose and opt.quiet. The semantics is as + follows: + + * LOG_ALWAYS - print the message unconditionally; + + * LOG_NOTQUIET - print the message if opt.quiet is non-zero; + + * LOG_NONVERBOSE - print the message if opt.verbose is zero; + + * LOG_VERBOSE - print the message if opt.verbose is non-zero. */ +#define CHECK_VERBOSE(x) \ + switch (x) \ + { \ + case LOG_PROGRESS: \ + if (!opt.show_progress) \ + return; \ + break; \ + case LOG_ALWAYS: \ + break; \ + case LOG_NOTQUIET: \ + if (opt.quiet) \ + return; \ + break; \ + case LOG_NONVERBOSE: \ + if (opt.verbose || opt.quiet) \ + return; \ + break; \ + case LOG_VERBOSE: \ + if (!opt.verbose) \ + return; \ + } + +/* Returns the file descriptor for logging. This is LOGFP, except if + called before log_init, in which case it returns stderr. This is + useful in case someone calls a logging function before log_init. + + If logging is inhibited, return NULL. */ + +static FILE * +get_log_fp (void) +{ + if (inhibit_logging) + return NULL; + if (logfp) + return logfp; + return stderr; +} + +static FILE * +get_progress_fp (void) +{ + if (opt.show_progress == true) + return stderr; + return get_log_fp(); +} + +/* Returns the file descriptor for the secondary log file. This is + WARCLOGFP, except if called before log_init, in which case it + returns stderr. This is useful in case someone calls a logging + function before log_init. + + If logging is inhibited, return NULL. */ + +static FILE * +get_warc_log_fp (void) +{ + if (inhibit_logging) + return NULL; + if (warclogfp) + return warclogfp; + if (logfp) + return NULL; + return stderr; +} + +/* Sets the file descriptor for the secondary log file. */ + +void +log_set_warc_log_fp (FILE * fp) +{ + warclogfp = fp; +} + +/* Log a literal string S. The string is logged as-is, without a + newline appended. */ + +void +logputs (enum log_options o, const char *s) +{ + FILE *fp; + FILE *warcfp; + int errno_save = errno; + + check_redirect_output (); + if (o == LOG_PROGRESS) + fp = get_progress_fp (); + else + fp = get_log_fp (); + + errno = errno_save; + + if (fp == NULL) + return; + + warcfp = get_warc_log_fp (); + errno = errno_save; + + CHECK_VERBOSE (o); + + FPUTS (s, fp); + if (warcfp != NULL) + FPUTS (s, warcfp); + if (save_context_p) + saved_append (s); + if (flush_log_p) + logflush (); + else + needs_flushing = true; + + errno = errno_save; +} + +struct logvprintf_state { + char *bigmsg; + int expected_size; + int allocated; +}; + +/* Print a message to the log. A copy of message will be saved to + saved_log, for later reusal by log_dump_context(). + + Normally we'd want this function to loop around vsnprintf until + sufficient room is allocated, as the Linux man page recommends. + However each call to vsnprintf() must be preceded by va_start and + followed by va_end. Since calling va_start/va_end is possible only + in the function that contains the `...' declaration, we cannot call + vsnprintf more than once. Therefore this function saves its state + to logvprintf_state and signals the parent to call it again. + + (An alternative approach would be to use va_copy, but that's not + portable.) */ + +static bool GCC_FORMAT_ATTR (2, 0) +log_vprintf_internal (struct logvprintf_state *state, const char *fmt, + va_list args) +{ + char smallmsg[128]; + char *write_ptr = smallmsg; + int available_size = sizeof (smallmsg); + int numwritten; + FILE *fp = get_log_fp (); + FILE *warcfp = get_warc_log_fp (); + + if (fp == NULL) + return false; + + if (!save_context_p && warcfp == NULL) + { + /* In the simple case just call vfprintf(), to avoid needless + allocation and games with vsnprintf(). */ + vfprintf (fp, fmt, args); + goto flush; + } + + if (state->allocated != 0) + { + write_ptr = state->bigmsg; + available_size = state->allocated; + } + + /* The GNU coding standards advise not to rely on the return value + of sprintf(). However, vsnprintf() is a relatively new function + missing from legacy systems. Therefore I consider it safe to + assume that its return value is meaningful. On the systems where + vsnprintf() is not available, we use the implementation from + snprintf.c which does return the correct value. */ + numwritten = vsnprintf (write_ptr, available_size, fmt, args); + + /* vsnprintf() will not step over the limit given by available_size. + If it fails, it returns either -1 (older implementations) or the + number of characters (not counting the terminating \0) that + *would have* been written if there had been enough room (C99). + In the former case, we double available_size and malloc to get a + larger buffer, and try again. In the latter case, we use the + returned information to build a buffer of the correct size. */ + + if (numwritten == -1) + { + /* Writing failed, and we don't know the needed size. Try + again with doubled size. */ + int newsize = available_size << 1; + state->bigmsg = xrealloc (state->bigmsg, newsize); + state->allocated = newsize; + return false; + } + else if (numwritten >= available_size) + { + /* Writing failed, but we know exactly how much space we + need. */ + int newsize = numwritten + 1; + state->bigmsg = xrealloc (state->bigmsg, newsize); + state->allocated = newsize; + return false; + } + + /* Writing succeeded. */ + if (save_context_p) + saved_append (write_ptr); + FPUTS (write_ptr, fp); + if (warcfp != NULL && warcfp != fp) + FPUTS (write_ptr, warcfp); + xfree (state->bigmsg); + + flush: + if (flush_log_p) + logflush (); + else + needs_flushing = true; + + return true; +} + +/* Flush LOGFP. Useful while flushing is disabled. */ +void +logflush (void) +{ + FILE *fp = get_log_fp (); + FILE *warcfp = get_warc_log_fp (); + if (fp) + { +/* 2005-10-25 SMS. + On VMS, flush only for a terminal. See note at FPUTS macro, above. +*/ +#ifdef __VMS + if (isatty( fileno( fp))) + { + fflush (fp); + } +#else /* def __VMS */ + fflush (fp); +#endif /* def __VMS [else] */ + } + + if (warcfp != NULL) + fflush (warcfp); + + needs_flushing = false; +} + +/* Enable or disable log flushing. */ +void +log_set_flush (bool flush) +{ + if (flush == flush_log_p) + return; + + if (flush == false) + { + /* Disable flushing by setting flush_log_p to 0. */ + flush_log_p = false; + } + else + { + /* Re-enable flushing. If anything was printed in no-flush mode, + flush the log now. */ + if (needs_flushing) + logflush (); + flush_log_p = true; + } +} + +/* (Temporarily) disable storing log to memory. Returns the old + status of storing, with which this function can be called again to + reestablish storing. */ + +bool +log_set_save_context (bool savep) +{ + bool old = save_context_p; + save_context_p = savep; + return old; +} + +/* Print a message to the screen or to the log. The first argument + defines the verbosity of the message, and the rest are as in + printf(3). */ + +void +logprintf (enum log_options o, const char *fmt, ...) +{ + va_list args; + struct logvprintf_state lpstate; + bool done; + int errno_saved = errno; + + check_redirect_output (); + errno = errno_saved; + if (inhibit_logging) + return; + CHECK_VERBOSE (o); + + xzero (lpstate); + errno = 0; + do + { + va_start (args, fmt); + done = log_vprintf_internal (&lpstate, fmt, args); + va_end (args); + + if (done && errno == EPIPE) + exit (WGET_EXIT_GENERIC_ERROR); + } + while (!done); + + errno = errno_saved; +} + +#ifdef ENABLE_DEBUG +/* The same as logprintf(), but does anything only if opt.debug is + true. */ +void +debug_logprintf (const char *fmt, ...) +{ + if (opt.debug) + { + va_list args; + struct logvprintf_state lpstate; + bool done; + +#ifndef TESTING + check_redirect_output (); +#endif + if (inhibit_logging) + return; + + xzero (lpstate); + do + { + va_start (args, fmt); + done = log_vprintf_internal (&lpstate, fmt, args); + va_end (args); + } + while (!done); + } +} +#endif /* ENABLE_DEBUG */ + +/* Open FILE and set up a logging stream. If FILE cannot be opened, + exit with status of 1. */ +void +log_init (const char *file, bool appendp) +{ + if (file) + { + if (HYPHENP (file)) + { + stdlogfp = stdout; + logfp = stdlogfp; + } + else + { + filelogfp = fopen (file, appendp ? "a" : "w"); + if (!filelogfp) + { + fprintf (stderr, "%s: %s: %s\n", exec_name, file, strerror (errno)); + exit (WGET_EXIT_GENERIC_ERROR); + } + logfp = filelogfp; + } + } + else + { + /* The log goes to stderr to avoid collisions with the output if + the user specifies `-O -'. #### Francois Pinard suggests + that it's a better idea to print to stdout by default, and to + stderr only if the user actually specifies `-O -'. He says + this inconsistency is harder to document, but is overall + easier on the user. */ + stdlogfp = stderr; + logfp = stdlogfp; + + if (1 +#ifdef HAVE_ISATTY + && isatty (fileno (logfp)) +#endif + ) + { + /* If the output is a TTY, enable save context, i.e. store + the most recent several messages ("context") and dump + them to a log file in case SIGHUP or SIGUSR1 is received + (or Ctrl+Break is pressed under Windows). */ + save_context_p = true; + } + } + +#ifndef WINDOWS + /* Initialize this values so we don't have to ask every time we print line */ + shell_is_interactive = isatty (STDIN_FILENO); +#endif +} + +/* Close LOGFP (only if we opened it, not if it's stderr), inhibit + further logging and free the memory associated with it. */ +void +log_close (void) +{ + int i; + + if (logfp && logfp != stderr && logfp != stdout) + { + if (logfp == stdlogfp) + stdlogfp = NULL; + if (logfp == filelogfp) + filelogfp = NULL; + fclose (logfp); + } + logfp = NULL; + + inhibit_logging = true; + save_context_p = false; + + for (i = 0; i < SAVED_LOG_LINES; i++) + free_log_line (i); + log_line_current = -1; + trailing_line = false; +} + +/* Dump saved lines to logfp. */ +static void +log_dump_context (void) +{ + int num = log_line_current; + FILE *fp = get_log_fp (); + FILE *warcfp = get_warc_log_fp (); + if (!fp) + return; + + if (num == -1) + return; + if (trailing_line) + ROT_ADVANCE (num); + do + { + struct log_ln *ln = log_lines + num; + if (ln->content) + { + FPUTS (ln->content, fp); + if (warcfp != NULL) + FPUTS (ln->content, warcfp); + } + ROT_ADVANCE (num); + } + while (num != log_line_current); + if (trailing_line) + if (log_lines[log_line_current].content) + { + FPUTS (log_lines[log_line_current].content, fp); + if (warcfp != NULL) + FPUTS (log_lines[log_line_current].content, warcfp); + } + fflush (fp); + fflush (warcfp); +} + +/* String escape functions. */ + +/* Return the number of non-printable characters in SOURCE. + Non-printable characters are determined as per c-ctype.c. */ + +static int +count_nonprint (const char *source) +{ + const char *p; + int cnt; + for (p = source, cnt = 0; *p; p++) + if (!c_isprint (*p)) + ++cnt; + return cnt; +} + +/* Copy SOURCE to DEST, escaping non-printable characters. + + Non-printable refers to anything outside the non-control ASCII + range (32-126) which means that, for example, CR, LF, and TAB are + considered non-printable along with ESC, BS, and other control + chars. This is by design: it makes sure that messages from remote + servers cannot be easily used to deceive the users by mimicking + Wget's output. Disallowing non-ASCII characters is another + necessary security measure, which makes sure that remote servers + cannot garble the screen or guess the local charset and perform + homographic attacks. + + Of course, the above mandates that escnonprint only be used in + contexts expected to be ASCII, such as when printing host names, + URL components, HTTP headers, FTP server messages, and the like. + + ESCAPE is the leading character of the escape sequence. BASE + should be the base of the escape sequence, and must be either 8 for + octal or 16 for hex. + + DEST must point to a location with sufficient room to store an + encoded version of SOURCE. */ + +static void +copy_and_escape (const char *source, char *dest, char escape, int base) +{ + const char *from = source; + char *to = dest; + unsigned char c; + + /* Copy chars from SOURCE to DEST, escaping non-printable ones. */ + switch (base) + { + case 8: + while ((c = *from++) != '\0') + if (c_isprint (c)) + *to++ = c; + else + { + *to++ = escape; + *to++ = '0' + (c >> 6); + *to++ = '0' + ((c >> 3) & 7); + *to++ = '0' + (c & 7); + } + break; + case 16: + while ((c = *from++) != '\0') + if (c_isprint (c)) + *to++ = c; + else + { + *to++ = escape; + *to++ = XNUM_TO_DIGIT (c >> 4); + *to++ = XNUM_TO_DIGIT (c & 0xf); + } + break; + default: + abort (); + } + *to = '\0'; +} + +#define RING_SIZE 3 +struct ringel { + char *buffer; + int size; +}; +static struct ringel ring[RING_SIZE]; /* ring data */ + +static const char * +escnonprint_internal (const char *str, char escape, int base) +{ + static int ringpos; /* current ring position */ + int nprcnt; + + assert (base == 8 || base == 16); + + nprcnt = count_nonprint (str); + if (nprcnt == 0) + /* If there are no non-printable chars in STR, don't bother + copying anything, just return STR. */ + return str; + + { + /* Set up a pointer to the current ring position, so we can write + simply r->X instead of ring[ringpos].X. */ + struct ringel *r = ring + ringpos; + + /* Every non-printable character is replaced with the escape char + and three (or two, depending on BASE) *additional* chars. Size + must also include the length of the original string and one + additional char for the terminating \0. */ + int needed_size = strlen (str) + 1 + (base == 8 ? 3 * nprcnt : 2 * nprcnt); + + /* If the current buffer is uninitialized or too small, + (re)allocate it. */ + if (r->buffer == NULL || r->size < needed_size) + { + r->buffer = xrealloc (r->buffer, needed_size); + r->size = needed_size; + } + + copy_and_escape (str, r->buffer, escape, base); + ringpos = (ringpos + 1) % RING_SIZE; + return r->buffer; + } +} + +/* Return a pointer to a static copy of STR with the non-printable + characters escaped as \ooo. If there are no non-printable + characters in STR, STR is returned. See copy_and_escape for more + information on which characters are considered non-printable. + + DON'T call this function on translated strings because escaping + will break them. Don't call it on literal strings from the source, + which are by definition trusted. If newlines are allowed in the + string, escape and print it line by line because escaping the whole + string will convert newlines to \012. (This is so that expectedly + single-line messages cannot use embedded newlines to mimic Wget's + output and deceive the user.) + + escnonprint doesn't quote its escape character because it is notf + meant as a general and reversible quoting mechanism, but as a quick + way to defang binary junk sent by malicious or buggy servers. + + NOTE: since this function can return a pointer to static data, be + careful to copy its result before calling it again. However, to be + more useful with printf, it maintains an internal ring of static + buffers to return. Currently the ring size is 3, which means you + can print up to three values in the same printf; if more is needed, + bump RING_SIZE. */ + +const char * +escnonprint (const char *str) +{ + return escnonprint_internal (str, '\\', 8); +} + +/* Return a pointer to a static copy of STR with the non-printable + characters escaped as %XX. If there are no non-printable + characters in STR, STR is returned. + + See escnonprint for usage details. */ + +const char * +escnonprint_uri (const char *str) +{ + return escnonprint_internal (str, '%', 16); +} + +#if defined DEBUG_MALLOC || defined TESTING +void +log_cleanup (void) +{ + size_t i; + for (i = 0; i < countof (ring); i++) + xfree (ring[i].buffer); +} +#endif + +/* When SIGHUP or SIGUSR1 are received, the output is redirected + elsewhere. Such redirection is only allowed once. */ +static const char *redirect_request_signal_name; + +/* Redirect output to `wget-log' or back to stdout/stderr. */ + +void +redirect_output (bool to_file, const char *signal_name) +{ + if (to_file && logfp != filelogfp) + { + if (signal_name) + { + fprintf (stderr, "\n%s received.", signal_name); + } + if (!filelogfp) + { + filelogfp = unique_create (DEFAULT_LOGFILE, false, &logfile); + if (filelogfp) + { + fprintf (stderr, _("\nRedirecting output to %s.\n"), + quote (logfile)); + /* Store signal name to tell wget it's permanent redirect to log file */ + redirect_request_signal_name = signal_name; + logfp = filelogfp; + /* Dump the context output to the newly opened log. */ + log_dump_context (); + } + else + { + /* Eek! Opening the alternate log file has failed. Nothing we + can do but disable printing completely. */ + fprintf (stderr, _("%s: %s; disabling logging.\n"), + (logfile) ? logfile : DEFAULT_LOGFILE, strerror (errno)); + inhibit_logging = true; + } + } + else + { + fprintf (stderr, _("\nRedirecting output to %s.\n"), + quote (logfile)); + logfp = filelogfp; + log_dump_context (); + } + } + else if (!to_file && logfp != stdlogfp) + { + logfp = stdlogfp; + log_dump_context (); + } +} + +/* Check whether there's a need to redirect output. */ + +static void +check_redirect_output (void) +{ +#if !defined(WINDOWS) && !defined(__VMS) + /* If it was redirected already to log file by SIGHUP, SIGUSR1 or -o parameter, + * it was permanent. + * If there was no SIGHUP or SIGUSR1 and shell is interactive + * we check if process is fg or bg before every line is printed.*/ + if (!redirect_request_signal_name && shell_is_interactive && !opt.lfilename) + { + pid_t foreground_pgrp = tcgetpgrp (STDIN_FILENO); + + if (foreground_pgrp != -1 && foreground_pgrp != getpgrp () && !opt.quiet) + { + /* Process backgrounded */ + redirect_output (true,NULL); + } + else + { + /* Process foregrounded */ + redirect_output (false,NULL); + } + } +#endif /* !defined(WINDOWS) && !defined(__VMS) */ +} diff --git a/src/log.h b/src/log.h new file mode 100644 index 0000000..40d4d88 --- /dev/null +++ b/src/log.h @@ -0,0 +1,60 @@ +/* Declarations for log.c. + Copyright (C) 1998-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef LOG_H +#define LOG_H + +/* The log file to which Wget writes to after HUP. */ +#define DEFAULT_LOGFILE "wget-log" + +#include <stdio.h> + +enum log_options { LOG_VERBOSE, LOG_NOTQUIET, LOG_NONVERBOSE, LOG_ALWAYS, LOG_PROGRESS }; + +void log_set_warc_log_fp (FILE *); + +void logprintf (enum log_options, const char *, ...) + GCC_FORMAT_ATTR (2, 3); +void debug_logprintf (const char *, ...) GCC_FORMAT_ATTR (1, 2); +void logputs (enum log_options, const char *); +void logflush (void); +void log_set_flush (bool); +bool log_set_save_context (bool); + +void log_init (const char *, bool); +void log_close (void); +void log_cleanup (void); +void log_request_redirect_output (const char *); +void redirect_output (bool, const char *); + +const char *escnonprint (const char *); +const char *escnonprint_uri (const char *); + +#endif /* LOG_H */ diff --git a/src/main.c b/src/main.c new file mode 100644 index 0000000..c84f6e4 --- /dev/null +++ b/src/main.c @@ -0,0 +1,2306 @@ +/* Command line parsing. + Copyright (C) 1996-2015, 2018-2020 Free Software Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <signal.h> +#include <spawn.h> +#if defined(ENABLE_NLS) || defined(WINDOWS) +# include <locale.h> +#endif +#include <assert.h> +#include <errno.h> +#include <time.h> + +#include "exits.h" +#include "utils.h" +#include "init.h" +#include "retr.h" +#include "recur.h" +#include "host.h" +#include "url.h" +#include "progress.h" /* for progress_handle_sigwinch */ +#include "convert.h" +#include "spider.h" +#include "http.h" /* for save_cookies */ +#include "hsts.h" /* for initializing hsts_store to NULL */ +#include "ptimer.h" +#include "warc.h" +#include "version.h" +#include "c-strcase.h" +#include "dirname.h" +#include "xmemdup0.h" +#include <getopt.h> +#include <getpass.h> +#include <quote.h> + +#ifdef TESTING +/* Rename the main function so we can have a main() in fuzzing code + and call the original main. */ +# define main main_wget +#endif + +#ifdef HAVE_METALINK +# include <metalink/metalink_parser.h> +# include "metalink.h" +#endif + +#ifdef WINDOWS +# include <io.h> +# include <fcntl.h> +#ifndef ENABLE_NLS +# include <mbctype.h> +#endif +#endif + +#ifdef __VMS +# include "vms.h" +#endif /* __VMS */ + +#ifndef PATH_SEPARATOR +# define PATH_SEPARATOR '/' +#endif + +#ifndef ENABLE_IRI +struct iri dummy_iri; +#endif + +#ifdef HAVE_LIBCARES +#include <ares.h> +ares_channel ares; +#else +void *ares; +#endif + +struct options opt; + +/* defined in version.c */ +extern char *system_getrc; +/* Used for --version output in print_version */ +#define MAX_CHARS_PER_LINE 72 +#define TABULATION 4 + +const char *exec_name; + +/* Number of successfully downloaded URLs */ +int numurls = 0; + +/* Initialize I18N/L10N. That amounts to invoking setlocale, and + setting up gettext's message catalog using bindtextdomain and + textdomain. Does nothing if NLS is disabled or missing. */ + +#if defined(SIGHUP) || defined(SIGUSR1) +/* Hangup signal handler. When wget receives SIGHUP or SIGUSR1, it + will proceed operation as usual, trying to write into a log file. + If that is impossible, the output will be turned off. */ + +static void +redirect_output_signal (int sig) +{ + const char *signal_name = "WTF?!"; + +#ifdef SIGHUP + if (sig == SIGHUP) + signal_name = "SIGHUP"; +#endif +#ifdef SIGUSR1 + if (sig == SIGUSR1) + signal_name = "SIGUSR1"; +#endif + + redirect_output (true,signal_name); + progress_schedule_redirect (); + signal (sig, redirect_output_signal); +} +#endif /* defined(SIGHUP) || defined(SIGUSR1) */ + +static void +i18n_initialize (void) +{ + /* ENABLE_NLS implies existence of functions invoked here. */ +#ifdef ENABLE_NLS + /* Set the current locale. */ + setlocale (LC_ALL, ""); + /* Set the text message domain. */ + bindtextdomain ("wget", LOCALEDIR); + bindtextdomain ("wget-gnulib", LOCALEDIR); + textdomain ("wget"); +#elif defined WINDOWS + char MBCP[16] = ""; + int CP; + + CP = _getmbcp(); /* Consider it's different from default. */ + if (CP > 0) + snprintf(MBCP, sizeof(MBCP), ".%d", CP); + setlocale(LC_ALL, MBCP); +#endif /* ENABLE_NLS */ +} + +#ifdef HAVE_HSTS +/* make the HSTS store global */ +hsts_store_t hsts_store; + +static char* +get_hsts_database (void) +{ + if (opt.hsts_file) + return xstrdup (opt.hsts_file); + + if (opt.homedir) + { + char *dir = ajoin_dir_file(opt.homedir, ".wget-hsts"); + return dir; + } + + return NULL; +} + +static void +load_hsts (void) +{ + if (!hsts_store) + { + char *filename = get_hsts_database (); + + if (filename) + { + DEBUGP (("Reading HSTS entries from %s\n", filename)); + + hsts_store = hsts_store_open (filename); + + if (!hsts_store) + logprintf (LOG_NOTQUIET, "ERROR: could not open HSTS store at '%s'. " + "HSTS will be disabled.\n", + filename); + } + else + logprintf (LOG_NOTQUIET, "ERROR: could not open HSTS store. HSTS will be disabled.\n"); + + xfree (filename); + } +} + +static void +save_hsts (void) +{ + if (hsts_store) + { + char *filename = get_hsts_database (); + + if (filename && hsts_store_has_changed (hsts_store)) + { + DEBUGP (("Saving HSTS entries to %s\n", filename)); + hsts_store_save (hsts_store, filename); + } + + hsts_store_close (hsts_store); + xfree (hsts_store); + + xfree (filename); + } +} +#endif + +/* Definition of command-line options. */ + +_Noreturn static void print_help (void); +_Noreturn static void print_version (void); + +#ifdef HAVE_SSL +# define IF_SSL(a,b,c,d,e) { a, b, c, d , e }, +#else +# define IF_SSL(a,b,c,d,e) +#endif + +struct cmdline_option { + char long_name[MAX_LONGOPTION]; + char short_name; + enum { + OPT_VALUE, + OPT_BOOLEAN, + OPT_FUNCALL, + /* Non-standard options that have to be handled specially in + main(). */ + OPT__APPEND_OUTPUT, + OPT__CLOBBER, + OPT__DONT_REMOVE_LISTING, + OPT__EXECUTE, + OPT__NO, + OPT__PARENT + } type; + const void *data; /* for standard options */ + int argtype; /* for non-standard options */ +}; + +static struct cmdline_option option_data[] = + { + { "accept", 'A', OPT_VALUE, "accept", -1 }, + { "accept-regex", 0, OPT_VALUE, "acceptregex", -1 }, + { "adjust-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 }, + { "append-output", 'a', OPT__APPEND_OUTPUT, NULL, required_argument }, + { "ask-password", 0, OPT_BOOLEAN, "askpassword", -1 }, + { "auth-no-challenge", 0, OPT_BOOLEAN, "authnochallenge", -1 }, + { "background", 'b', OPT_BOOLEAN, "background", -1 }, + { "backup-converted", 'K', OPT_BOOLEAN, "backupconverted", -1 }, + { "backups", 0, OPT_BOOLEAN, "backups", -1 }, + { "base", 'B', OPT_VALUE, "base", -1 }, + { "bind-address", 0, OPT_VALUE, "bindaddress", -1 }, +#ifdef HAVE_LIBCARES + { "bind-dns-address", 0, OPT_VALUE, "binddnsaddress", -1 }, +#endif + { "body-data", 0, OPT_VALUE, "bodydata", -1 }, + { "body-file", 0, OPT_VALUE, "bodyfile", -1 }, + IF_SSL ( "ca-certificate", 0, OPT_VALUE, "cacertificate", -1 ) + IF_SSL ( "ca-directory", 0, OPT_VALUE, "cadirectory", -1 ) + { "cache", 0, OPT_BOOLEAN, "cache", -1 }, + IF_SSL ( "certificate", 0, OPT_VALUE, "certificate", -1 ) + IF_SSL ( "certificate-type", 0, OPT_VALUE, "certificatetype", -1 ) + IF_SSL ( "check-certificate", 0, OPT_BOOLEAN, "checkcertificate", -1 ) + { "clobber", 0, OPT__CLOBBER, NULL, optional_argument }, +#ifdef HAVE_LIBZ + { "compression", 0, OPT_VALUE, "compression", -1 }, +#endif + { "config", 0, OPT_VALUE, "chooseconfig", -1 }, + { "connect-timeout", 0, OPT_VALUE, "connecttimeout", -1 }, + { "continue", 'c', OPT_BOOLEAN, "continue", -1 }, + { "convert-file-only", 0, OPT_BOOLEAN, "convertfileonly", -1 }, + { "convert-links", 'k', OPT_BOOLEAN, "convertlinks", -1 }, + { "content-disposition", 0, OPT_BOOLEAN, "contentdisposition", -1 }, + { "content-on-error", 0, OPT_BOOLEAN, "contentonerror", -1 }, + { "cookies", 0, OPT_BOOLEAN, "cookies", -1 }, + IF_SSL ( "crl-file", 0, OPT_VALUE, "crlfile", -1 ) + { "cut-dirs", 0, OPT_VALUE, "cutdirs", -1 }, + { "debug", 'd', OPT_BOOLEAN, "debug", -1 }, + { "default-page", 0, OPT_VALUE, "defaultpage", -1 }, + { "delete-after", 0, OPT_BOOLEAN, "deleteafter", -1 }, + { "directories", 0, OPT_BOOLEAN, "dirstruct", -1 }, + { "directory-prefix", 'P', OPT_VALUE, "dirprefix", -1 }, + { "dns-cache", 0, OPT_BOOLEAN, "dnscache", -1 }, +#ifdef HAVE_LIBCARES + { "dns-servers", 0, OPT_VALUE, "dnsservers", -1 }, +#endif + { "dns-timeout", 0, OPT_VALUE, "dnstimeout", -1 }, + { "domains", 'D', OPT_VALUE, "domains", -1 }, + { "dont-remove-listing", 0, OPT__DONT_REMOVE_LISTING, NULL, no_argument }, + { "dot-style", 0, OPT_VALUE, "dotstyle", -1 }, /* deprecated */ + { "egd-file", 0, OPT_VALUE, "egdfile", -1 }, + { "exclude-directories", 'X', OPT_VALUE, "excludedirectories", -1 }, + { "exclude-domains", 0, OPT_VALUE, "excludedomains", -1 }, + { "execute", 'e', OPT__EXECUTE, NULL, required_argument }, + { "follow-ftp", 0, OPT_BOOLEAN, "followftp", -1 }, + { "follow-tags", 0, OPT_VALUE, "followtags", -1 }, + { "force-directories", 'x', OPT_BOOLEAN, "dirstruct", -1 }, + { "force-html", 'F', OPT_BOOLEAN, "forcehtml", -1 }, + { "ftp-password", 0, OPT_VALUE, "ftppassword", -1 }, +#ifdef __VMS + { "ftp-stmlf", 0, OPT_BOOLEAN, "ftpstmlf", -1 }, +#endif /* def __VMS */ + { "ftp-user", 0, OPT_VALUE, "ftpuser", -1 }, + IF_SSL ( "ftps-clear-data-connection", 0, OPT_BOOLEAN, "ftpscleardataconnection", -1 ) + IF_SSL ( "ftps-fallback-to-ftp", 0, OPT_BOOLEAN, "ftpsfallbacktoftp", -1 ) + IF_SSL ( "ftps-implicit", 0, OPT_BOOLEAN, "ftpsimplicit", -1 ) + IF_SSL ( "ftps-resume-ssl", 0, OPT_BOOLEAN, "ftpsresumessl", -1 ) + { "glob", 0, OPT_BOOLEAN, "glob", -1 }, + { "header", 0, OPT_VALUE, "header", -1 }, + { "help", 'h', OPT_FUNCALL, (void *)print_help, no_argument }, + { "host-directories", 0, OPT_BOOLEAN, "addhostdir", -1 }, +#ifdef HAVE_HSTS + { "hsts", 0, OPT_BOOLEAN, "hsts", -1}, + { "hsts-file", 0, OPT_VALUE, "hstsfile", -1 }, +#endif + { "html-extension", 'E', OPT_BOOLEAN, "adjustextension", -1 }, /* deprecated */ + { "htmlify", 0, OPT_BOOLEAN, "htmlify", -1 }, + { "http-keep-alive", 0, OPT_BOOLEAN, "httpkeepalive", -1 }, + { "http-passwd", 0, OPT_VALUE, "httppassword", -1 }, /* deprecated */ + { "http-password", 0, OPT_VALUE, "httppassword", -1 }, + { "http-user", 0, OPT_VALUE, "httpuser", -1 }, + IF_SSL ( "https-only", 0, OPT_BOOLEAN, "httpsonly", -1 ) + { "ignore-case", 0, OPT_BOOLEAN, "ignorecase", -1 }, + { "ignore-length", 0, OPT_BOOLEAN, "ignorelength", -1 }, + { "ignore-tags", 0, OPT_VALUE, "ignoretags", -1 }, + { "include-directories", 'I', OPT_VALUE, "includedirectories", -1 }, +#ifdef ENABLE_IPV6 + { "inet4-only", '4', OPT_BOOLEAN, "inet4only", -1 }, + { "inet6-only", '6', OPT_BOOLEAN, "inet6only", -1 }, +#endif + { "input-file", 'i', OPT_VALUE, "input", -1 }, +#ifdef HAVE_METALINK + { "input-metalink", 0, OPT_VALUE, "inputmetalink", -1 }, +#endif + { "iri", 0, OPT_BOOLEAN, "iri", -1 }, + { "keep-badhash", 0, OPT_BOOLEAN, "keepbadhash", -1 }, + { "keep-session-cookies", 0, OPT_BOOLEAN, "keepsessioncookies", -1 }, + { "level", 'l', OPT_VALUE, "reclevel", -1 }, + { "limit-rate", 0, OPT_VALUE, "limitrate", -1 }, + { "load-cookies", 0, OPT_VALUE, "loadcookies", -1 }, + { "local-encoding", 0, OPT_VALUE, "localencoding", -1 }, + { "rejected-log", 0, OPT_VALUE, "rejectedlog", -1 }, + { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 }, +#ifdef HAVE_METALINK + { "metalink-index", 0, OPT_VALUE, "metalinkindex", -1 }, + { "metalink-over-http", 0, OPT_BOOLEAN, "metalinkoverhttp", -1 }, +#endif + { "method", 0, OPT_VALUE, "method", -1 }, + { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 }, + { "netrc", 0, OPT_BOOLEAN, "netrc", -1 }, + { "no", 'n', OPT__NO, NULL, required_argument }, + { "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 }, + { "no-config", 0, OPT_BOOLEAN, "noconfig", -1}, + { "no-parent", 0, OPT_BOOLEAN, "noparent", -1 }, + { "output-document", 'O', OPT_VALUE, "outputdocument", -1 }, + { "output-file", 'o', OPT_VALUE, "logfile", -1 }, + { "page-requisites", 'p', OPT_BOOLEAN, "pagerequisites", -1 }, + { "parent", 0, OPT__PARENT, NULL, optional_argument }, + { "passive-ftp", 0, OPT_BOOLEAN, "passiveftp", -1 }, + { "password", 0, OPT_VALUE, "password", -1 }, + IF_SSL ( "pinnedpubkey", 0, OPT_VALUE, "pinnedpubkey", -1 ) + { "post-data", 0, OPT_VALUE, "postdata", -1 }, + { "post-file", 0, OPT_VALUE, "postfile", -1 }, + { "prefer-family", 0, OPT_VALUE, "preferfamily", -1 }, +#ifdef HAVE_METALINK + { "preferred-location", 0, OPT_VALUE, "preferredlocation", -1 }, +#endif + { "preserve-permissions", 0, OPT_BOOLEAN, "preservepermissions", -1 }, + IF_SSL ( "ciphers", 0, OPT_VALUE, "ciphers", -1 ) + IF_SSL ( "private-key", 0, OPT_VALUE, "privatekey", -1 ) + IF_SSL ( "private-key-type", 0, OPT_VALUE, "privatekeytype", -1 ) + { "progress", 0, OPT_VALUE, "progress", -1 }, + { "show-progress", 0, OPT_BOOLEAN, "showprogress", -1 }, + { "protocol-directories", 0, OPT_BOOLEAN, "protocoldirectories", -1 }, + { "proxy", 0, OPT_BOOLEAN, "useproxy", -1 }, + { "proxy__compat", 'Y', OPT_VALUE, "useproxy", -1 }, /* back-compatible */ + { "proxy-passwd", 0, OPT_VALUE, "proxypassword", -1 }, /* deprecated */ + { "proxy-password", 0, OPT_VALUE, "proxypassword", -1 }, + { "proxy-user", 0, OPT_VALUE, "proxyuser", -1 }, + { "quiet", 'q', OPT_BOOLEAN, "quiet", -1 }, + { "quota", 'Q', OPT_VALUE, "quota", -1 }, + { "random-file", 0, OPT_VALUE, "randomfile", -1 }, + { "random-wait", 0, OPT_BOOLEAN, "randomwait", -1 }, + { "read-timeout", 0, OPT_VALUE, "readtimeout", -1 }, + { "recursive", 'r', OPT_BOOLEAN, "recursive", -1 }, + { "referer", 0, OPT_VALUE, "referer", -1 }, + { "regex-type", 0, OPT_VALUE, "regextype", -1 }, + { "reject", 'R', OPT_VALUE, "reject", -1 }, + { "reject-regex", 0, OPT_VALUE, "rejectregex", -1 }, + { "relative", 'L', OPT_BOOLEAN, "relativeonly", -1 }, + { "remote-encoding", 0, OPT_VALUE, "remoteencoding", -1 }, + { "remove-listing", 0, OPT_BOOLEAN, "removelisting", -1 }, + { "report-speed", 0, OPT_BOOLEAN, "reportspeed", -1 }, + { "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 }, + { "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 }, + { "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 }, + { "retry-on-host-error", 0, OPT_BOOLEAN, "retryonhosterror", -1 }, + { "retry-on-http-error", 0, OPT_VALUE, "retryonhttperror", -1 }, + { "save-cookies", 0, OPT_VALUE, "savecookies", -1 }, + { "save-headers", 0, OPT_BOOLEAN, "saveheaders", -1 }, + IF_SSL ( "secure-protocol", 0, OPT_VALUE, "secureprotocol", -1 ) + { "server-response", 'S', OPT_BOOLEAN, "serverresponse", -1 }, + { "span-hosts", 'H', OPT_BOOLEAN, "spanhosts", -1 }, + { "spider", 0, OPT_BOOLEAN, "spider", -1 }, + { "start-pos", 0, OPT_VALUE, "startpos", -1 }, + { "strict-comments", 0, OPT_BOOLEAN, "strictcomments", -1 }, + { "timeout", 'T', OPT_VALUE, "timeout", -1 }, + { "timestamping", 'N', OPT_BOOLEAN, "timestamping", -1 }, + { "if-modified-since", 0, OPT_BOOLEAN, "ifmodifiedsince", -1 }, + { "tries", 't', OPT_VALUE, "tries", -1 }, + { "unlink", 0, OPT_BOOLEAN, "unlink", -1 }, + { "trust-server-names", 0, OPT_BOOLEAN, "trustservernames", -1 }, +#ifndef __VMS + { "use-askpass", 0, OPT_VALUE, "useaskpass", -1}, +#endif + { "use-server-timestamps", 0, OPT_BOOLEAN, "useservertimestamps", -1 }, + { "user", 0, OPT_VALUE, "user", -1 }, + { "user-agent", 'U', OPT_VALUE, "useragent", -1 }, + { "verbose", 'v', OPT_BOOLEAN, "verbose", -1 }, + { "version", 'V', OPT_FUNCALL, (void *) print_version, no_argument }, + { "wait", 'w', OPT_VALUE, "wait", -1 }, + { "waitretry", 0, OPT_VALUE, "waitretry", -1 }, + { "warc-cdx", 0, OPT_BOOLEAN, "warccdx", -1 }, +#ifdef HAVE_LIBZ + { "warc-compression", 0, OPT_BOOLEAN, "warccompression", -1 }, +#endif + { "warc-dedup", 0, OPT_VALUE, "warccdxdedup", -1 }, + { "warc-digests", 0, OPT_BOOLEAN, "warcdigests", -1 }, + { "warc-file", 0, OPT_VALUE, "warcfile", -1 }, + { "warc-header", 0, OPT_VALUE, "warcheader", -1 }, + { "warc-keep-log", 0, OPT_BOOLEAN, "warckeeplog", -1 }, + { "warc-max-size", 0, OPT_VALUE, "warcmaxsize", -1 }, + { "warc-tempdir", 0, OPT_VALUE, "warctempdir", -1 }, +#ifdef USE_WATT32 + { "wdebug", 0, OPT_BOOLEAN, "wdebug", -1 }, +#endif +#ifdef ENABLE_XATTR + { "xattr", 0, OPT_BOOLEAN, "xattr", -1 }, +#endif + }; + +#undef IF_SSL + +/* Return a string that contains S with "no-" prepended. The string + is NUL-terminated and allocated off static storage at Wget + startup. */ + +static char * +no_prefix (const char *s) +{ + static char buffer[2048]; + static char *p = buffer; + + char *cp = p; + int size = 3 + strlen (s) + 1; /* "no-STRING\0" */ + assert(p + size <= buffer + sizeof (buffer)); + + cp[0] = 'n', cp[1] = 'o', cp[2] = '-'; + strcpy (cp + 3, s); + p += size; + return cp; +} + +/* The arguments that that main passes to getopt_long. */ +static struct option long_options[2 * countof (option_data) + 1]; +static char short_options[128]; + +/* Mapping between short option chars and option_data indices. */ +static unsigned char optmap[96]; + +/* Marker for `--no-FOO' values in long_options. */ +#define BOOLEAN_NEG_MARKER 1024 + +/* Initialize the long_options array used by getopt_long from the data + in option_data. */ + +static void +init_switches (void) +{ + static bool initialized; + char *p = short_options; + size_t i, o = 0; + + if (initialized) + return; + initialized = 1; + + for (i = 0; i < countof (option_data); i++) + { + struct cmdline_option *cmdopt = &option_data[i]; + struct option *longopt; + + longopt = &long_options[o++]; + longopt->name = cmdopt->long_name; + longopt->val = i; + if (cmdopt->short_name) + { + *p++ = cmdopt->short_name; + optmap[cmdopt->short_name - 32] = longopt - long_options; + } + switch (cmdopt->type) + { + case OPT_VALUE: + longopt->has_arg = required_argument; + if (cmdopt->short_name) + *p++ = ':'; + break; + case OPT_BOOLEAN: + /* Specify an optional argument for long options, so that + --option=off works the same as --no-option, for + compatibility with pre-1.10 Wget. However, don't specify + optional arguments short-option booleans because they + prevent combining of short options. */ + longopt->has_arg = optional_argument; + /* For Boolean options, add the "--no-FOO" variant, which is + identical to "--foo", except it has opposite meaning and + it doesn't allow an argument. */ + longopt = &long_options[o++]; + longopt->name = no_prefix (cmdopt->long_name); + longopt->has_arg = no_argument; + /* Mask the value so we'll be able to recognize that we're + dealing with the false value. */ + longopt->val = i | BOOLEAN_NEG_MARKER; + break; + default: + assert (cmdopt->argtype != -1); + longopt->has_arg = cmdopt->argtype; + if (cmdopt->short_name) + { + if (longopt->has_arg == required_argument) + *p++ = ':'; + /* Don't handle optional_argument */ + } + } + } + /* Terminate short_options. */ + *p = '\0'; + /* No need for xzero(long_options[o]) because its storage is static + and it will be zeroed by default. */ + assert (o <= countof (long_options)); +} + +/* Print the usage message. */ +static int +print_usage (_GL_UNUSED int error) +{ +#ifndef TESTING + return fprintf (error ? stderr : stdout, + _("Usage: %s [OPTION]... [URL]...\n"), exec_name); +#else + return 0; +#endif +} + +/* Print the help message, describing all the available options. If + you add an option, be sure to update this list. */ +_Noreturn static void +print_help (void) +{ +#ifndef TESTING + /* We split the help text this way to ease translation of individual + entries. */ + static const char *help[] = { + "\n", + N_("\ +Mandatory arguments to long options are mandatory for short options too.\n\n"), + N_("\ +Startup:\n"), + N_("\ + -V, --version display the version of Wget and exit\n"), + N_("\ + -h, --help print this help\n"), + N_("\ + -b, --background go to background after startup\n"), + N_("\ + -e, --execute=COMMAND execute a `.wgetrc'-style command\n"), + "\n", + + N_("\ +Logging and input file:\n"), + N_("\ + -o, --output-file=FILE log messages to FILE\n"), + N_("\ + -a, --append-output=FILE append messages to FILE\n"), +#ifdef ENABLE_DEBUG + N_("\ + -d, --debug print lots of debugging information\n"), +#endif +#ifdef USE_WATT32 + N_("\ + --wdebug print Watt-32 debug output\n"), +#endif + N_("\ + -q, --quiet quiet (no output)\n"), + N_("\ + -v, --verbose be verbose (this is the default)\n"), + N_("\ + -nv, --no-verbose turn off verboseness, without being quiet\n"), + N_("\ + --report-speed=TYPE output bandwidth as TYPE. TYPE can be bits\n"), + N_("\ + -i, --input-file=FILE download URLs found in local or external FILE\n"), +#ifdef HAVE_METALINK + N_("\ + --input-metalink=FILE download files covered in local Metalink FILE\n"), +#endif + N_("\ + -F, --force-html treat input file as HTML\n"), + N_("\ + -B, --base=URL resolves HTML input-file links (-i -F)\n\ + relative to URL\n"), + N_("\ + --config=FILE specify config file to use\n"), + N_("\ + --no-config do not read any config file\n"), + N_("\ + --rejected-log=FILE log reasons for URL rejection to FILE\n"), + "\n", + + N_("\ +Download:\n"), + N_("\ + -t, --tries=NUMBER set number of retries to NUMBER (0 unlimits)\n"), + N_("\ + --retry-connrefused retry even if connection is refused\n"), + N_("\ + --retry-on-http-error=ERRORS comma-separated list of HTTP errors to retry\n"), + N_("\ + -O, --output-document=FILE write documents to FILE\n"), + N_("\ + -nc, --no-clobber skip downloads that would download to\n\ + existing files (overwriting them)\n"), + N_("\ + --no-netrc don't try to obtain credentials from .netrc\n"), + N_("\ + -c, --continue resume getting a partially-downloaded file\n"), + N_("\ + --start-pos=OFFSET start downloading from zero-based position OFFSET\n"), + N_("\ + --progress=TYPE select progress gauge type\n"), + N_("\ + --show-progress display the progress bar in any verbosity mode\n"), + N_("\ + -N, --timestamping don't re-retrieve files unless newer than\n\ + local\n"), + N_("\ + --no-if-modified-since don't use conditional if-modified-since get\n\ + requests in timestamping mode\n"), + N_("\ + --no-use-server-timestamps don't set the local file's timestamp by\n\ + the one on the server\n"), + N_("\ + -S, --server-response print server response\n"), + N_("\ + --spider don't download anything\n"), + N_("\ + -T, --timeout=SECONDS set all timeout values to SECONDS\n"), +#ifdef HAVE_LIBCARES + N_("\ + --dns-servers=ADDRESSES list of DNS servers to query (comma separated)\n"), + N_("\ + --bind-dns-address=ADDRESS bind DNS resolver to ADDRESS (hostname or IP) on local host\n"), +#endif + N_("\ + --dns-timeout=SECS set the DNS lookup timeout to SECS\n"), + N_("\ + --connect-timeout=SECS set the connect timeout to SECS\n"), + N_("\ + --read-timeout=SECS set the read timeout to SECS\n"), + N_("\ + -w, --wait=SECONDS wait SECONDS between retrievals\n\ + (applies if more then 1 URL is to be retrieved)\n"), + N_("\ + --waitretry=SECONDS wait 1..SECONDS between retries of a retrieval\n\ + (applies if more then 1 URL is to be retrieved)\n"), + N_("\ + --random-wait wait from 0.5*WAIT...1.5*WAIT secs between retrievals\n\ + (applies if more then 1 URL is to be retrieved)\n"), + N_("\ + --no-proxy explicitly turn off proxy\n"), + N_("\ + -Q, --quota=NUMBER set retrieval quota to NUMBER\n"), + N_("\ + --bind-address=ADDRESS bind to ADDRESS (hostname or IP) on local host\n"), + N_("\ + --limit-rate=RATE limit download rate to RATE\n"), + N_("\ + --no-dns-cache disable caching DNS lookups\n"), + N_("\ + --restrict-file-names=OS restrict chars in file names to ones OS allows\n"), + N_("\ + --ignore-case ignore case when matching files/directories\n"), +#ifdef ENABLE_IPV6 + N_("\ + -4, --inet4-only connect only to IPv4 addresses\n"), + N_("\ + -6, --inet6-only connect only to IPv6 addresses\n"), + N_("\ + --prefer-family=FAMILY connect first to addresses of specified family,\n\ + one of IPv6, IPv4, or none\n"), +#endif + N_("\ + --user=USER set both ftp and http user to USER\n"), + N_("\ + --password=PASS set both ftp and http password to PASS\n"), + N_("\ + --ask-password prompt for passwords\n"), +#ifndef __VMS + N_("\ + --use-askpass=COMMAND specify credential handler for requesting \n\ + username and password. If no COMMAND is \n\ + specified the WGET_ASKPASS or the SSH_ASKPASS \n\ + environment variable is used.\n"), +#endif + N_("\ + --no-iri turn off IRI support\n"), + N_("\ + --local-encoding=ENC use ENC as the local encoding for IRIs\n"), + N_("\ + --remote-encoding=ENC use ENC as the default remote encoding\n"), + N_("\ + --unlink remove file before clobber\n"), +#ifdef HAVE_METALINK + N_("\ + --keep-badhash keep files with checksum mismatch (append .badhash)\n"), + N_("\ + --metalink-index=NUMBER Metalink application/metalink4+xml metaurl ordinal NUMBER\n"), + N_("\ + --metalink-over-http use Metalink metadata from HTTP response headers\n"), + N_("\ + --preferred-location preferred location for Metalink resources\n"), +#endif +#ifdef ENABLE_XATTR + N_("\ + --xattr turn on storage of metadata in extended file attributes\n"), +#endif + "\n", + + N_("\ +Directories:\n"), + N_("\ + -nd, --no-directories don't create directories\n"), + N_("\ + -x, --force-directories force creation of directories\n"), + N_("\ + -nH, --no-host-directories don't create host directories\n"), + N_("\ + --protocol-directories use protocol name in directories\n"), + N_("\ + -P, --directory-prefix=PREFIX save files to PREFIX/..\n"), + N_("\ + --cut-dirs=NUMBER ignore NUMBER remote directory components\n"), + "\n", + + N_("\ +HTTP options:\n"), + N_("\ + --http-user=USER set http user to USER\n"), + N_("\ + --http-password=PASS set http password to PASS\n"), + N_("\ + --no-cache disallow server-cached data\n"), + N_ ("\ + --default-page=NAME change the default page name (normally\n\ + this is 'index.html'.)\n"), + N_("\ + -E, --adjust-extension save HTML/CSS documents with proper extensions\n"), + N_("\ + --ignore-length ignore 'Content-Length' header field\n"), + N_("\ + --header=STRING insert STRING among the headers\n"), +#ifdef HAVE_LIBZ + N_("\ + --compression=TYPE choose compression, one of auto, gzip and none. (default: none)\n"), +#endif + N_("\ + --max-redirect maximum redirections allowed per page\n"), + N_("\ + --proxy-user=USER set USER as proxy username\n"), + N_("\ + --proxy-password=PASS set PASS as proxy password\n"), + N_("\ + --referer=URL include 'Referer: URL' header in HTTP request\n"), + N_("\ + --save-headers save the HTTP headers to file\n"), + N_("\ + -U, --user-agent=AGENT identify as AGENT instead of Wget/VERSION\n"), + N_("\ + --no-http-keep-alive disable HTTP keep-alive (persistent connections)\n"), + N_("\ + --no-cookies don't use cookies\n"), + N_("\ + --load-cookies=FILE load cookies from FILE before session\n"), + N_("\ + --save-cookies=FILE save cookies to FILE after session\n"), + N_("\ + --keep-session-cookies load and save session (non-permanent) cookies\n"), + N_("\ + --post-data=STRING use the POST method; send STRING as the data\n"), + N_("\ + --post-file=FILE use the POST method; send contents of FILE\n"), + N_("\ + --method=HTTPMethod use method \"HTTPMethod\" in the request\n"), + N_("\ + --body-data=STRING send STRING as data. --method MUST be set\n"), + N_("\ + --body-file=FILE send contents of FILE. --method MUST be set\n"), + N_("\ + --content-disposition honor the Content-Disposition header when\n\ + choosing local file names (EXPERIMENTAL)\n"), + N_("\ + --content-on-error output the received content on server errors\n"), + N_("\ + --auth-no-challenge send Basic HTTP authentication information\n\ + without first waiting for the server's\n\ + challenge\n"), + "\n", + +#ifdef HAVE_SSL + N_("\ +HTTPS (SSL/TLS) options:\n"), + N_("\ + --secure-protocol=PR choose secure protocol, one of auto, SSLv2,\n\ + SSLv3, TLSv1, TLSv1_1, TLSv1_2 and PFS\n"), + N_("\ + --https-only only follow secure HTTPS links\n"), + N_("\ + --no-check-certificate don't validate the server's certificate\n"), + N_("\ + --certificate=FILE client certificate file\n"), + N_("\ + --certificate-type=TYPE client certificate type, PEM or DER\n"), + N_("\ + --private-key=FILE private key file\n"), + N_("\ + --private-key-type=TYPE private key type, PEM or DER\n"), + N_("\ + --ca-certificate=FILE file with the bundle of CAs\n"), + N_("\ + --ca-directory=DIR directory where hash list of CAs is stored\n"), + N_("\ + --crl-file=FILE file with bundle of CRLs\n"), + N_("\ + --pinnedpubkey=FILE/HASHES Public key (PEM/DER) file, or any number\n\ + of base64 encoded sha256 hashes preceded by\n\ + \'sha256//\' and separated by \';\', to verify\n\ + peer against\n"), +#if defined(HAVE_LIBSSL) || defined(HAVE_LIBSSL32) + N_("\ + --random-file=FILE file with random data for seeding the SSL PRNG\n"), +#endif +#if (defined(HAVE_LIBSSL) || defined(HAVE_LIBSSL32)) && defined(HAVE_RAND_EGD) + N_("\ + --egd-file=FILE file naming the EGD socket with random data\n"), +#endif + "\n", + N_("\ + --ciphers=STR Set the priority string (GnuTLS) or cipher list string (OpenSSL) directly.\n\ + Use with care. This option overrides --secure-protocol.\n\ + The format and syntax of this string depend on the specific SSL/TLS engine.\n"), +#endif /* HAVE_SSL */ + +#ifdef HAVE_HSTS + N_("\ +HSTS options:\n"), + N_("\ + --no-hsts disable HSTS\n"), + N_("\ + --hsts-file path of HSTS database (will override default)\n"), + "\n", +#endif + + N_("\ +FTP options:\n"), +#ifdef __VMS + N_("\ + --ftp-stmlf use Stream_LF format for all binary FTP files\n"), +#endif /* def __VMS */ + N_("\ + --ftp-user=USER set ftp user to USER\n"), + N_("\ + --ftp-password=PASS set ftp password to PASS\n"), + N_("\ + --no-remove-listing don't remove '.listing' files\n"), + N_("\ + --no-glob turn off FTP file name globbing\n"), + N_("\ + --no-passive-ftp disable the \"passive\" transfer mode\n"), + N_("\ + --preserve-permissions preserve remote file permissions\n"), + N_("\ + --retr-symlinks when recursing, get linked-to files (not dir)\n"), + "\n", + +#ifdef HAVE_SSL + N_("\ +FTPS options:\n"), + N_("\ + --ftps-implicit use implicit FTPS (default port is 990)\n"), + N_("\ + --ftps-resume-ssl resume the SSL/TLS session started in the control connection when\n" + " opening a data connection\n"), + N_("\ + --ftps-clear-data-connection cipher the control channel only; all the data will be in plaintext\n"), + N_("\ + --ftps-fallback-to-ftp fall back to FTP if FTPS is not supported in the target server\n"), +#endif + + N_("\ +WARC options:\n"), + N_("\ + --warc-file=FILENAME save request/response data to a .warc.gz file\n"), + N_("\ + --warc-header=STRING insert STRING into the warcinfo record\n"), + N_("\ + --warc-max-size=NUMBER set maximum size of WARC files to NUMBER\n"), + N_("\ + --warc-cdx write CDX index files\n"), + N_("\ + --warc-dedup=FILENAME do not store records listed in this CDX file\n"), +#ifdef HAVE_LIBZ + N_("\ + --no-warc-compression do not compress WARC files with GZIP\n"), +#endif + N_("\ + --no-warc-digests do not calculate SHA1 digests\n"), + N_("\ + --no-warc-keep-log do not store the log file in a WARC record\n"), + N_("\ + --warc-tempdir=DIRECTORY location for temporary files created by the\n\ + WARC writer\n"), + "\n", + + N_("\ +Recursive download:\n"), + N_("\ + -r, --recursive specify recursive download\n"), + N_("\ + -l, --level=NUMBER maximum recursion depth (inf or 0 for infinite)\n"), + N_("\ + --delete-after delete files locally after downloading them\n"), + N_("\ + -k, --convert-links make links in downloaded HTML or CSS point to\n\ + local files\n"), + N_("\ + --convert-file-only convert the file part of the URLs only (usually known as the basename)\n"), + N_("\ + --backups=N before writing file X, rotate up to N backup files\n"), + +#ifdef __VMS + N_("\ + -K, --backup-converted before converting file X, back up as X_orig\n"), +#else /* def __VMS */ + N_("\ + -K, --backup-converted before converting file X, back up as X.orig\n"), +#endif /* def __VMS [else] */ + N_("\ + -m, --mirror shortcut for -N -r -l inf --no-remove-listing\n"), + N_("\ + -p, --page-requisites get all images, etc. needed to display HTML page\n"), + N_("\ + --strict-comments turn on strict (SGML) handling of HTML comments\n"), + "\n", + + N_("\ +Recursive accept/reject:\n"), + N_("\ + -A, --accept=LIST comma-separated list of accepted extensions\n"), + N_("\ + -R, --reject=LIST comma-separated list of rejected extensions\n"), + N_("\ + --accept-regex=REGEX regex matching accepted URLs\n"), + N_("\ + --reject-regex=REGEX regex matching rejected URLs\n"), +#if defined HAVE_LIBPCRE || defined HAVE_LIBPCRE2 + N_("\ + --regex-type=TYPE regex type (posix|pcre)\n"), +#else + N_("\ + --regex-type=TYPE regex type (posix)\n"), +#endif + N_("\ + -D, --domains=LIST comma-separated list of accepted domains\n"), + N_("\ + --exclude-domains=LIST comma-separated list of rejected domains\n"), + N_("\ + --follow-ftp follow FTP links from HTML documents\n"), + N_("\ + --follow-tags=LIST comma-separated list of followed HTML tags\n"), + N_("\ + --ignore-tags=LIST comma-separated list of ignored HTML tags\n"), + N_("\ + -H, --span-hosts go to foreign hosts when recursive\n"), + N_("\ + -L, --relative follow relative links only\n"), + N_("\ + -I, --include-directories=LIST list of allowed directories\n"), + N_("\ + --trust-server-names use the name specified by the redirection\n\ + URL's last component\n"), + N_("\ + -X, --exclude-directories=LIST list of excluded directories\n"), + N_("\ + -np, --no-parent don't ascend to the parent directory\n"), + "\n", + N_("Email bug reports, questions, discussions to <bug-wget@gnu.org>\n" + "and/or open issues at https://savannah.gnu.org/bugs/?func=additem&group=wget.\n") + }; + + size_t i; + + if (printf (_("GNU Wget %s, a non-interactive network retriever.\n"), + version_string) < 0) + exit (WGET_EXIT_IO_FAIL); + if (print_usage (0) < 0) + exit (WGET_EXIT_IO_FAIL); + + for (i = 0; i < countof (help); i++) + if (fputs (_(help[i]), stdout) < 0) + exit (WGET_EXIT_IO_FAIL); +#endif /* TESTING */ + exit (WGET_EXIT_SUCCESS); +} + +/* Return a human-readable printed representation of INTERVAL, + measured in seconds. */ + +static char * +secs_to_human_time (double interval) +{ + static char buf[32]; + int secs = (int) (interval + 0.5); + int hours, mins, days; + + days = secs / 86400, secs %= 86400; + hours = secs / 3600, secs %= 3600; + mins = secs / 60, secs %= 60; + + if (days) + sprintf (buf, "%dd %dh %dm %ds", days, hours, mins, secs); + else if (hours) + sprintf (buf, "%dh %dm %ds", hours, mins, secs); + else if (mins) + sprintf (buf, "%dm %ds", mins, secs); + else + sprintf (buf, "%ss", print_decimal (interval)); + + return buf; +} + +static char * +prompt_for_password (void) +{ + if (opt.user) + fprintf (stderr, _("Password for user %s: "), quote (opt.user)); + else + fprintf (stderr, _("Password: ")); +#ifndef TESTING + /* gnulib's getpass() uses static variables internally, bad for fuzing */ + return getpass(""); +#else + return xstrdup(""); +#endif +} + + +/* Execute external application opt.use_askpass */ +static void +run_use_askpass (char *question, char **answer) +{ + char tmp[1024]; + pid_t pid; + int status; + int com[2]; + ssize_t bytes = 0; + char *argv[3], *p; + posix_spawn_file_actions_t fa; + + if (pipe (com) == -1) + { + fprintf (stderr, _("Cannot create pipe\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + + status = posix_spawn_file_actions_init (&fa); + if (status) + { + fprintf (stderr, + _("Error initializing spawn file actions for use-askpass: %d\n"), + status); + exit (WGET_EXIT_GENERIC_ERROR); + } + + status = posix_spawn_file_actions_adddup2 (&fa, com[1], STDOUT_FILENO); + if (status) + { + fprintf (stderr, + _("Error setting spawn file actions for use-askpass: %d\n"), + status); + exit (WGET_EXIT_GENERIC_ERROR); + } + + /* C89 initializer lists must be computable at load time, + * thus this explicit initialization. */ + argv[0] = opt.use_askpass; + argv[1] = question; + argv[2] = NULL; + + status = posix_spawnp (&pid, opt.use_askpass, &fa, NULL, argv, environ); + if (status) + { + fprintf (stderr, "Error spawning %s: %d\n", opt.use_askpass, status); + exit (WGET_EXIT_GENERIC_ERROR); + } + + /* Parent process reads from child. */ + close (com[1]); + bytes = read (com[0], tmp, sizeof (tmp) - 1); + if (bytes <= 0) + { + fprintf (stderr, + _("Error reading response from command \"%s %s\": %s\n"), + opt.use_askpass, question, strerror (errno)); + exit (WGET_EXIT_GENERIC_ERROR); + } + + /* Make sure there is a trailing 0 */ + tmp[bytes] = '\0'; + + /* Remove a possible new line */ + if ((p = strpbrk (tmp, "\r\n"))) + bytes = p - tmp; + + *answer = xmemdup0 (tmp, bytes); +} + +/* set the user name and password*/ +static void +use_askpass (struct url *u) +{ + static char question[1024]; + + if (u->user == NULL || u->user[0] == '\0') + { + snprintf (question, sizeof (question), _("Username for '%s%s': "), + scheme_leading_string(u->scheme), u->host); + /* Prompt for username */ + run_use_askpass (question, &u->user); + if (opt.recursive) + opt.user = xstrdup (u->user); + } + + if (u->passwd == NULL || u->passwd[0] == '\0') + { + snprintf(question, sizeof (question), _("Password for '%s%s@%s': "), + scheme_leading_string (u->scheme), u->user, u->host); + /* Prompt for password */ + run_use_askpass (question, &u->passwd); + if (opt.recursive) + opt.passwd = xstrdup (u->passwd); + } +} +/* Function that prints the line argument while limiting it + to at most line_length. prefix is printed on the first line + and an appropriate number of spaces are added on subsequent + lines.*/ +static int +format_and_print_line (const char *prefix, const char *line, + int line_length) +{ + int remaining_chars; + char *line_dup, *token; + + assert (prefix != NULL); + assert (line != NULL); + assert (line_length > TABULATION); + + line_dup = xstrdup (line); + + if (printf ("%s", prefix) < 0) + { + xfree (line_dup); + return -1; + } + + /* Wrap to new line after prefix. */ + remaining_chars = 0; + + /* We break on spaces. */ + token = strtok (line_dup, " "); + while (token != NULL) + { + /* If however a token is much larger than the maximum + line length, all bets are off and we simply print the + token on the next line. */ + if (remaining_chars <= (int) strlen (token)) + { + if (printf ("\n%*c", TABULATION, ' ') < 0) + { + xfree (line_dup); + return -1; + } + remaining_chars = line_length - TABULATION; + } + if (printf ("%s ", token) < 0) + { + xfree (line_dup); + return -1; + } + remaining_chars -= strlen (token) + 1; /* account for " " */ + token = strtok (NULL, " "); + } + + if (printf ("\n") < 0) + { + xfree (line_dup); + return -1; + } + + xfree (line_dup); + return 0; +} + +_Noreturn static void +print_version (void) +{ + const char *wgetrc_title = _("Wgetrc: "); + const char *locale_title = _("Locale: "); + const char *compile_title = _("Compile: "); + const char *link_title = _("Link: "); + char *env_wgetrc, *user_wgetrc; + int i; + + if (printf (_("GNU Wget %s built on %s.\n\n"), version_string, OS_TYPE) < 0) + exit (WGET_EXIT_IO_FAIL); + + for (i = 0; compiled_features[i] != NULL; ) + { + int line_length = MAX_CHARS_PER_LINE; + while ((line_length > 0) && (compiled_features[i] != NULL)) + { + if (printf ("%s ", compiled_features[i]) < 0) + exit (WGET_EXIT_IO_FAIL); + line_length -= (int) strlen (compiled_features[i]) + 2; + i++; + } + if (printf ("\n") < 0) + exit (WGET_EXIT_IO_FAIL); + } + if (printf ("\n") < 0) + exit (WGET_EXIT_IO_FAIL); + + /* Print VMS-specific version info. */ +#ifdef __VMS + if (vms_version_supplement() < 0) + exit (WGET_EXIT_IO_FAIL); +#endif /* def __VMS */ + + /* Handle the case when $WGETRC is unset and $HOME/.wgetrc is + absent. */ + if (printf ("%s\n", wgetrc_title) < 0) + exit (WGET_EXIT_IO_FAIL); + + env_wgetrc = wgetrc_env_file_name (); + if (env_wgetrc && *env_wgetrc) + { + if (printf (_(" %s (env)\n"), env_wgetrc) < 0) + exit (WGET_EXIT_IO_FAIL); + xfree (env_wgetrc); + } + user_wgetrc = wgetrc_user_file_name (); + if (user_wgetrc) + { + if (printf (_(" %s (user)\n"), user_wgetrc) < 0) + exit (WGET_EXIT_IO_FAIL); + xfree (user_wgetrc); + } +#ifdef SYSTEM_WGETRC + if (printf (_(" %s (system)\n"), SYSTEM_WGETRC) < 0) + exit (WGET_EXIT_IO_FAIL); +#endif + +#ifdef ENABLE_NLS + if (format_and_print_line (locale_title, + LOCALEDIR, + MAX_CHARS_PER_LINE) < 0) + exit (WGET_EXIT_IO_FAIL); +#endif /* def ENABLE_NLS */ + + if (compilation_string != NULL) + if (format_and_print_line (compile_title, + compilation_string, + MAX_CHARS_PER_LINE) < 0) + exit (WGET_EXIT_IO_FAIL); + + if (link_string != NULL) + if (format_and_print_line (link_title, + link_string, + MAX_CHARS_PER_LINE) < 0) + exit (WGET_EXIT_IO_FAIL); + + if (printf ("\n") < 0) + exit (WGET_EXIT_IO_FAIL); + + /* TRANSLATORS: When available, an actual copyright character + (circle-c) should be used in preference to "(C)". */ + if (printf (_("\ +Copyright (C) %s Free Software Foundation, Inc.\n"), "2015") < 0) + exit (WGET_EXIT_IO_FAIL); + if (fputs (_("\ +License GPLv3+: GNU GPL version 3 or later\n\ +<http://www.gnu.org/licenses/gpl.html>.\n\ +This is free software: you are free to change and redistribute it.\n\ +There is NO WARRANTY, to the extent permitted by law.\n"), stdout) < 0) + exit (WGET_EXIT_IO_FAIL); + /* TRANSLATORS: When available, please use the proper diacritics for + names such as this one. See en_US.po for reference. */ + if (fputs (_("\nOriginally written by Hrvoje Niksic <hniksic@xemacs.org>.\n"), + stdout) < 0) + exit (WGET_EXIT_IO_FAIL); + if (fputs (_("Please send bug reports and questions to <bug-wget@gnu.org>.\n"), + stdout) < 0) + exit (WGET_EXIT_IO_FAIL); + + exit (WGET_EXIT_SUCCESS); +} + +const char *program_name; /* Needed by lib/error.c. */ +const char *program_argstring; /* Needed by wget_warc.c. */ +struct ptimer *timer; +int cleaned_up; + +int +main (int argc, char **argv) +{ + char *p; + int i, ret, longindex; + int nurls; + int retconf; + int argstring_length; + bool use_userconfig = false; + bool noconfig = false; + bool append_to_log = false; + + cleaned_up = 0; /* do cleanup later */ + + timer = ptimer_new (); + double start_time = ptimer_measure (timer); + + total_downloaded_bytes = 0; + program_name = argv[0]; + + i18n_initialize (); + + /* Construct the name of the executable, without the directory part. */ +#ifdef __VMS + /* On VMS, lose the "dev:[dir]" prefix and the ".EXE;nnn" suffix. */ + exec_name = vms_basename (argv[0]); +#else /* def __VMS */ + exec_name = base_name (argv[0]); +#endif /* def __VMS [else] */ + +#ifdef WINDOWS + /* Drop extension (typically .EXE) from executable filename. */ + windows_main ((char **) &exec_name); +#endif + + /* Construct the arguments string. */ + for (argstring_length = 1, i = 1; i < argc; i++) + argstring_length += strlen (argv[i]) + 3 + 1; + program_argstring = p = malloc (argstring_length); + if (p == NULL) + { + fprintf (stderr, _("Memory allocation problem\n")); + exit (WGET_EXIT_PARSE_ERROR); + } + for (i = 1; i < argc; i++) + { + int arglen; + + *p++ = '"'; + arglen = strlen (argv[i]); + memcpy (p, argv[i], arglen); + p += arglen; + *p++ = '"'; + *p++ = ' '; + } + *p = '\0'; + + /* Load the hard-coded defaults. */ + defaults (); + opt.homedir = home_dir(); + + init_switches (); + + /* This separate getopt_long is needed to find the user config file + option ("--config") and parse it before the other user options. */ + longindex = -1; + + while ((retconf = getopt_long (argc, argv, + short_options, long_options, &longindex)) != -1) + { + int confval; + struct cmdline_option *config_opt; + + /* There is no short option for "--config". */ + if (longindex >= 0) + { + confval = long_options[longindex].val; + config_opt = &option_data[confval & ~BOOLEAN_NEG_MARKER]; + if (strcmp (config_opt->long_name, "no-config") == 0) + { + noconfig = true; + break; + } + else if (strcmp (config_opt->long_name, "config") == 0) + { + file_stats_t flstats; + use_userconfig = true; + memset(&flstats, 0, sizeof(flstats)); + if (file_exists_p(optarg, &flstats) && run_wgetrc (optarg, &flstats)) + break; + else + { + fprintf (stderr, _("Exiting due to error in %s\n"), optarg); + exit (WGET_EXIT_PARSE_ERROR); + } + } + } + } + + /* If the user did not specify a config, read the system wgetrc and ~/.wgetrc. */ + if (noconfig == false && use_userconfig == false) + if ((ret = initialize ())) + return ret; + + opterr = 0; + optind = 0; + + longindex = -1; + while ((ret = getopt_long (argc, argv, + short_options, long_options, &longindex)) != -1) + { + int val; + struct cmdline_option *cmdopt; + + /* If LONGINDEX is unchanged, it means RET is referring a short + option. */ + if (longindex == -1) + { + if (ret == '?') + { + print_usage (1); + fprintf (stderr, "\n"); + fprintf (stderr, _("Try `%s --help' for more options.\n"), + exec_name); + exit (WGET_EXIT_PARSE_ERROR); + } + /* Find the short option character in the mapping. */ + longindex = optmap[ret - 32]; + } + val = long_options[longindex].val; + + /* Use the retrieved value to locate the option in the + option_data array, and to see if we're dealing with the + negated "--no-FOO" variant of the boolean option "--foo". */ + cmdopt = &option_data[val & ~BOOLEAN_NEG_MARKER]; + switch (cmdopt->type) + { + case OPT_VALUE: + setoptval (cmdopt->data, optarg, cmdopt->long_name); + break; + case OPT_BOOLEAN: + if (optarg) + /* The user has specified a value -- use it. */ + setoptval (cmdopt->data, optarg, cmdopt->long_name); + else + { + /* NEG is true for `--no-FOO' style boolean options. */ + bool neg = !!(val & BOOLEAN_NEG_MARKER); + setoptval (cmdopt->data, neg ? "0" : "1", cmdopt->long_name); + } + break; + case OPT_FUNCALL: + { + void (*func) (void) = (void (*) (void)) cmdopt->data; + func (); + } + break; + case OPT__APPEND_OUTPUT: + setoptval ("logfile", optarg, cmdopt->long_name); + append_to_log = true; + break; + case OPT__EXECUTE: + if (optarg) /* check silences static analyzer */ + run_command (optarg); + break; + case OPT__NO: + { + /* We support real --no-FOO flags now, but keep these + short options for convenience and backward + compatibility. */ + for (p = optarg; p && *p; p++) + switch (*p) + { + case 'v': + setoptval ("verbose", "0", cmdopt->long_name); + break; + case 'H': + setoptval ("addhostdir", "0", cmdopt->long_name); + break; + case 'd': + setoptval ("dirstruct", "0", cmdopt->long_name); + break; + case 'c': + setoptval ("noclobber", "1", cmdopt->long_name); + break; + case 'p': + setoptval ("noparent", "1", cmdopt->long_name); + break; + default: + fprintf (stderr, _("%s: illegal option -- `-n%c'\n"), + exec_name, *p); + print_usage (1); + fprintf (stderr, "\n"); + fprintf (stderr, _("Try `%s --help' for more options.\n"), + exec_name); + exit (WGET_EXIT_GENERIC_ERROR); + } + break; + } + case OPT__PARENT: + case OPT__CLOBBER: + { + /* The wgetrc commands are named noparent and noclobber, + so we must revert the meaning of the cmdline options + before passing the value to setoptval. */ + bool flag = true; + if (optarg) + flag = (*optarg == '1' || c_tolower (*optarg) == 'y' + || (c_tolower (optarg[0]) == 'o' + && c_tolower (optarg[1]) == 'n')); + setoptval (cmdopt->type == OPT__PARENT ? "noparent" : "noclobber", + flag ? "0" : "1", cmdopt->long_name); + break; + } + case OPT__DONT_REMOVE_LISTING: + setoptval ("removelisting", "0", cmdopt->long_name); + break; + } + + longindex = -1; + } + + nurls = argc - optind; + + /* Initialize logging ASAP. */ + log_init (opt.lfilename, append_to_log); + + /* If we do not have Debug support compiled in AND Wget is invoked with the + * --debug switch, instead of failing, we silently turn it into a no-op. For + * this no-op, we explicitly set opt.debug to false and hence none of the + * Debug output messages will be printed. + */ +#ifndef ENABLE_DEBUG + if (opt.debug) + { + fprintf (stderr, _("Debugging support not compiled in. " + "Ignoring --debug flag.\n")); + opt.debug = false; + } +#endif + + /* All user options have now been processed, so it's now safe to do + interoption dependency checks. */ + + if (opt.noclobber && (opt.convert_links || opt.convert_file_only)) + { + fprintf (stderr, + opt.convert_links ? + _("Both --no-clobber and --convert-links were specified," + " only --convert-links will be used.\n") : + _("Both --no-clobber and --convert-file-only were specified," + " only --convert-file-only will be used.\n")); + opt.noclobber = false; + } + + if (opt.reclevel == 0) + opt.reclevel = INFINITE_RECURSION; /* see recur.h for commentary */ + + if (opt.spider || opt.delete_after) + opt.no_dirstruct = true; + + if (opt.page_requisites && !opt.recursive) + { + /* Don't set opt.recursive here because it would confuse the FTP + code. Instead, call retrieve_tree below when either + page_requisites or recursive is requested. */ + opt.reclevel = 0; + if (!opt.no_dirstruct) + opt.dirstruct = 1; /* normally handled by cmd_spec_recursive() */ + } + + if (opt.verbose == -1) + opt.verbose = !opt.quiet; + + if (!opt.verbose && opt.show_progress == -1) + opt.show_progress = false; + + if (opt.quiet && opt.show_progress == -1) + opt.show_progress = false; + + /* Sanity checks. */ + if (opt.verbose && opt.quiet) + { + fprintf (stderr, _("Can't be verbose and quiet at the same time.\n")); + print_usage (1); + exit (WGET_EXIT_GENERIC_ERROR); + } + if (opt.timestamping && opt.noclobber) + { + fprintf (stderr, _("\ +Can't timestamp and not clobber old files at the same time.\n")); + print_usage (1); + exit (WGET_EXIT_GENERIC_ERROR); + } +#ifdef ENABLE_IPV6 + if (opt.ipv4_only && opt.ipv6_only) + { + fprintf (stderr, + _("Cannot specify both --inet4-only and --inet6-only.\n")); + print_usage (1); + exit (WGET_EXIT_GENERIC_ERROR); + } +#endif + if (opt.output_document) + { + if ((opt.convert_links || opt.convert_file_only) + && (nurls > 1 || opt.page_requisites || opt.recursive)) + { + fputs (_("\ +Cannot specify both -k or --convert-file-only and -O if multiple URLs are given, or in combination\n\ +with -p or -r. See the manual for details.\n\n"), stderr); + print_usage (1); + exit (WGET_EXIT_GENERIC_ERROR); + } + if (opt.page_requisites + || opt.recursive) + { + logprintf (LOG_NOTQUIET, "%s", _("\ +WARNING: combining -O with -r or -p will mean that all downloaded content\n\ +will be placed in the single file you specified.\n\n")); + } + if (opt.timestamping) + { + logprintf (LOG_NOTQUIET, "%s", _("\ +WARNING: timestamping does nothing in combination with -O. See the manual\n\ +for details.\n\n")); + opt.timestamping = false; + } + if (opt.noclobber && file_exists_p(opt.output_document, NULL)) + { + /* Check if output file exists; if it does, exit. */ + logprintf (LOG_VERBOSE, + _("File %s already there; not retrieving.\n"), + quote (opt.output_document)); + exit (WGET_EXIT_GENERIC_ERROR); + } + } + + if (opt.warc_filename != 0) + { + if (opt.noclobber) + { + fprintf (stderr, + _("WARC output does not work with --no-clobber, " + "--no-clobber will be disabled.\n")); + opt.noclobber = false; + } + if (opt.timestamping) + { + fprintf (stderr, + _("WARC output does not work with timestamping, " + "timestamping will be disabled.\n")); + opt.timestamping = false; + } + if (opt.spider) + { + fprintf (stderr, + _("WARC output does not work with --spider.\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + if (opt.always_rest || opt.start_pos >= 0) + { + fprintf (stderr, + _("WARC output does not work with --continue or" + " --start-pos, they will be disabled.\n")); + opt.always_rest = false; + opt.start_pos = -1; + } + if (opt.warc_cdx_dedup_filename != 0 && !opt.warc_digests_enabled) + { + fprintf (stderr, + _("Digests are disabled; WARC deduplication will " + "not find duplicate records.\n")); + } + if (opt.warc_keep_log) + { + opt.progress_type = xstrdup ("dot"); + } + } + +#ifdef HAVE_LIBZ + if (opt.always_rest || opt.start_pos >= 0) + { + if (opt.compression == compression_auto) + { + /* Compression does not work with --continue or --start-pos. + Since compression was not explicitly set, it will be disabled. */ + opt.compression = compression_none; + } + else if (opt.compression != compression_none) + { + fprintf (stderr, + _("Compression does not work with --continue or" + " --start-pos, they will be disabled.\n")); + opt.always_rest = false; + opt.start_pos = -1; + } + } +#endif + + if (opt.ask_passwd && opt.passwd) + { + fprintf (stderr, + _("Cannot specify both --ask-password and --password.\n")); + print_usage (1); + exit (WGET_EXIT_GENERIC_ERROR); + } + + if (opt.ask_passwd && !(opt.user || opt.http_user || opt.ftp_user)) + { + fprintf(stderr, + _("WARNING: No username set with --ask-password. This is usually not what you want.\n")); + } + + if (opt.start_pos >= 0 && opt.always_rest) + { + fprintf (stderr, + _("Specifying both --start-pos and --continue is not " + "recommended; --continue will be disabled.\n")); + opt.always_rest = false; + } + + if (!nurls && !opt.input_filename +#ifdef HAVE_METALINK + && !opt.input_metalink +#endif + ) + { + /* No URL specified. */ +#ifndef TESTING + fprintf (stderr, _("%s: missing URL\n"), exec_name); + print_usage (1); + fprintf (stderr, "\n"); + /* #### Something nicer should be printed here -- similar to the + pre-1.5 `--help' page. */ + fprintf (stderr, _("Try `%s --help' for more options.\n"), exec_name); +#endif + exit (WGET_EXIT_GENERIC_ERROR); + } + + /* Compile the regular expressions. */ + switch (opt.regex_type) + { +#ifdef HAVE_LIBPCRE2 + case regex_type_pcre: + opt.regex_compile_fun = compile_pcre2_regex; + opt.regex_match_fun = match_pcre2_regex; + break; +#endif +#ifdef HAVE_LIBPCRE + case regex_type_pcre: + opt.regex_compile_fun = compile_pcre_regex; + opt.regex_match_fun = match_pcre_regex; + break; +#endif + + case regex_type_posix: + default: + opt.regex_compile_fun = compile_posix_regex; + opt.regex_match_fun = match_posix_regex; + break; + } + if (opt.acceptregex_s) + { + opt.acceptregex = opt.regex_compile_fun (opt.acceptregex_s); + if (!opt.acceptregex) + exit (WGET_EXIT_GENERIC_ERROR); + } + if (opt.rejectregex_s) + { + opt.rejectregex = opt.regex_compile_fun (opt.rejectregex_s); + if (!opt.rejectregex) + exit (WGET_EXIT_GENERIC_ERROR); + } + if (opt.post_data || opt.post_file_name) + { + if (opt.post_data && opt.post_file_name) + { + fprintf (stderr, _("You cannot specify both --post-data and --post-file.\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + else if (opt.method) + { + fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. " + "--method expects data through --body-data and --body-file options\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + } + if (opt.body_data || opt.body_file) + { + if (!opt.method) + { + fprintf (stderr, _("You must specify a method through --method=HTTPMethod " + "to use with --body-data or --body-file.\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + else if (opt.body_data && opt.body_file) + { + fprintf (stderr, _("You cannot specify both --body-data and --body-file.\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + } + + /* Set various options as required for opt.method. */ + + /* When user specifies HEAD as the method, we do not wish to download any + files. Hence, set wget to run in spider mode. */ + if (opt.method && c_strcasecmp (opt.method, "HEAD") == 0) + setoptval ("spider", "1", "spider"); + + /* Convert post_data to body-data and post_file_name to body-file options. + This is required so as to remove redundant code later on in gethttp(). + The --post-data and --post-file options may also be removed in + the future hence it makes sense to convert them to aliases for + the more generic --method options. + This MUST occur only after the sanity checks so as to prevent the + user from setting both post and body options simultaneously. + */ + if (opt.post_data || opt.post_file_name) + { + setoptval ("method", "POST", "method"); + if (opt.post_data) + { + setoptval ("bodydata", opt.post_data, "body-data"); + xfree(opt.post_data); + } + else + { + setoptval ("bodyfile", opt.post_file_name, "body-file"); + xfree(opt.post_file_name); + } + } + +#ifdef ENABLE_IRI + if (opt.enable_iri) + { + if (opt.locale && !check_encoding_name (opt.locale)) + xfree (opt.locale); + + if (!opt.locale) + opt.locale = find_locale (); + + if (opt.encoding_remote && !check_encoding_name (opt.encoding_remote)) + xfree (opt.encoding_remote); + } +#else + memset (&dummy_iri, 0, sizeof (dummy_iri)); + if (opt.enable_iri || opt.locale || opt.encoding_remote) + { + /* sXXXav : be more specific... */ + fprintf (stderr, _("This version does not have support for IRIs\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } +#endif + + if (opt.ask_passwd) + { + opt.passwd = prompt_for_password (); + + if (opt.passwd == NULL || opt.passwd[0] == '\0') + exit (WGET_EXIT_GENERIC_ERROR); + } + + if (opt.use_askpass) + { + if (opt.use_askpass[0] == '\0') + { + fprintf (stderr, + _("use-askpass requires a string or either environment variable WGET_ASKPASS or SSH_ASKPASS to be set.\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + } + +#ifdef USE_WATT32 + if (opt.wdebug) + dbug_init(); + sock_init(); +#elif ! defined TESTING + if (opt.background) + { + bool logfile_changed = fork_to_background (); + + if (logfile_changed) + log_init (opt.lfilename, append_to_log); + } +#endif + + /* Initialize progress. Have to do this after the options are + processed so we know where the log file is. */ + if (opt.show_progress) + set_progress_implementation (opt.progress_type); + + /* Open WARC file. */ + if (opt.warc_filename != 0) + warc_init (); + + DEBUGP (("DEBUG output created by Wget %s on %s.\n\n", + version_string, OS_TYPE)); + + /* Open the output filename if necessary. */ + +/* 2005-04-17 SMS. + Note that having the output_stream ("-O") file opened here for an FTP + URL rather than in getftp() (ftp.c) (and the http equivalent) rather + limits the ability in VMS to open the file differently for ASCII + versus binary FTP there. (Of course, doing it here allows a open + failure to be detected immediately, without first connecting to the + server.) +*/ + if (opt.output_document) + { + if (HYPHENP (opt.output_document)) + { +#ifdef WINDOWS + _setmode (_fileno (stdout), _O_BINARY); +#endif + output_stream = stdout; + } + else + { + struct stat st; + +#ifdef __VMS +/* Common fopen() optional arguments: + sequential access only, access callback function. +*/ +# define FOPEN_OPT_ARGS , "fop=sqo", "acc", acc_cb, &open_id + int open_id = 7; +#else /* def __VMS */ +# define FOPEN_OPT_ARGS +#endif /* def __VMS [else] */ + + output_stream = fopen (opt.output_document, + opt.always_rest ? "ab" : "wb" + FOPEN_OPT_ARGS); + if (output_stream == NULL) + { + perror (opt.output_document); + exit (WGET_EXIT_GENERIC_ERROR); + } + if (fstat (fileno (output_stream), &st) == 0 && S_ISREG (st.st_mode)) + output_stream_regular = true; + } + if (!output_stream_regular && (opt.convert_links || opt.recursive)) + { + fprintf (stderr, _("-k or -r can be used together with -O only if \ +outputting to a regular file.\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + if (!output_stream_regular && (opt.convert_links || opt.convert_file_only)) + { + fprintf (stderr, _("--convert-links or --convert-file-only can be used together \ +only if outputting to a regular file.\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + } + +#ifdef HAVE_LIBCARES + if (opt.bind_dns_address || opt.dns_servers) + { + if (ares_library_init (ARES_LIB_INIT_ALL)) + { + fprintf (stderr, _("Failed to init libcares\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + + if (ares_init (&ares) != ARES_SUCCESS) + { + fprintf (stderr, _("Failed to init c-ares channel\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + + if (opt.bind_dns_address) + { + struct in_addr a4; +#ifdef ENABLE_IPV6 + struct in6_addr a6; +#endif + + if (inet_pton (AF_INET, opt.bind_dns_address, &a4) == 1) + { + ares_set_local_ip4 (ares, ntohl (a4.s_addr)); + } +#ifdef ENABLE_IPV6 + else if (inet_pton (AF_INET6, opt.bind_dns_address, &a6) == 1) + { + ares_set_local_ip6 (ares, (unsigned char *) &a6); + } +#endif + else + { + fprintf (stderr, _("Failed to parse IP address '%s'\n"), opt.bind_dns_address); + exit (WGET_EXIT_GENERIC_ERROR); + } + } + + if (opt.dns_servers) + { + int result; + + if ((result = ares_set_servers_csv (ares, opt.dns_servers)) != ARES_SUCCESS) + { + fprintf (stderr, _("Failed to set DNS server(s) '%s' (%d)\n"), opt.dns_servers, result); + exit (WGET_EXIT_GENERIC_ERROR); + } + } + } +#endif + +#ifdef __VMS + /* Set global ODS5 flag according to the specified destination (if + any), otherwise according to the current default device. + */ + if (output_stream == NULL) + set_ods5_dest( "SYS$DISK"); + else if (output_stream != stdout) + set_ods5_dest( opt.output_document); +#endif /* def __VMS */ + +#ifdef WINDOWS + ws_startup (); +#endif + +#ifdef SIGHUP + /* Setup the signal handler to redirect output when hangup is + received. */ + if (signal(SIGHUP, SIG_IGN) != SIG_IGN) + signal(SIGHUP, redirect_output_signal); +#endif + /* ...and do the same for SIGUSR1. */ +#ifdef SIGUSR1 + signal (SIGUSR1, redirect_output_signal); +#endif +#ifdef SIGPIPE + /* Writing to a closed socket normally signals SIGPIPE, and the + process exits. What we want is to ignore SIGPIPE and just check + for the return value of write(). */ + signal (SIGPIPE, SIG_IGN); +#endif +#ifdef SIGWINCH + signal (SIGWINCH, progress_handle_sigwinch); +#endif + +#ifdef HAVE_HSTS + /* Load the HSTS database. + Maybe all the URLs are FTP(S), in which case HSTS would not be needed, + but this is the best place to do it, and it shouldn't be a critical + performance hit. + */ + if (opt.hsts) + load_hsts (); +#endif + + /* Retrieve the URLs from argument list. */ + for (i = 0; i < nurls; i++, optind++) + { + char *t; + char *filename = NULL, *redirected_URL = NULL; + int dt = 0, url_err; + /* Need to do a new struct iri every time, because + * retrieve_url may modify it in some circumstances, + * currently. */ + struct iri *iri = iri_new (); + struct url *url_parsed; + + t = rewrite_shorthand_url (argv[optind]); + if (!t) + t = argv[optind]; + + set_uri_encoding (iri, opt.locale, true); + url_parsed = url_parse (t, &url_err, iri, true); + + if (!url_parsed) + { + char *error = url_error (t, url_err); + logprintf (LOG_NOTQUIET, "%s: %s.\n",t, error); + xfree (error); + inform_exit_status (URLERROR); + } + else + { + /* Request credentials if use_askpass is set. */ + if (opt.use_askpass) + use_askpass (url_parsed); + + if ((opt.recursive || opt.page_requisites) + && ((url_scheme (t) != SCHEME_FTP +#ifdef HAVE_SSL + && url_scheme (t) != SCHEME_FTPS +#endif + ) + || url_uses_proxy (url_parsed))) + { + int old_follow_ftp = opt.follow_ftp; + + /* Turn opt.follow_ftp on in case of recursive FTP retrieval */ + if (url_scheme (t) == SCHEME_FTP +#ifdef HAVE_SSL + || url_scheme (t) == SCHEME_FTPS +#endif + ) + opt.follow_ftp = 1; + + retrieve_tree (url_parsed, NULL); + + opt.follow_ftp = old_follow_ftp; + } + else + { + retrieve_url (url_parsed, t, &filename, &redirected_URL, NULL, + &dt, opt.recursive, iri, true); + } + + if (opt.delete_after && filename != NULL && file_exists_p (filename, NULL)) + { + DEBUGP (("Removing file due to --delete-after in main():\n")); + logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename); + if (unlink (filename)) + logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); + } + xfree (redirected_URL); + xfree (filename); + url_free (url_parsed); + } + + iri_free (iri); + + if (t != argv[optind]) + xfree (t); + } + + /* And then from the input file, if any. */ + if (opt.input_filename) + { + int count; + int status; + status = retrieve_from_file (opt.input_filename, opt.force_html, &count); + inform_exit_status (status); + if (!count) + logprintf (LOG_NOTQUIET, _("No URLs found in %s.\n"), + opt.input_filename); + } + +#ifdef HAVE_METALINK + /* Finally, from metlink file, if any. */ + if (opt.input_metalink) + { + metalink_error_t meta_err; + uerr_t retr_err; + metalink_t *metalink; + + meta_err = metalink_parse_file (opt.input_metalink, &metalink); + + if (meta_err) + { + logprintf (LOG_NOTQUIET, _("Unable to parse metalink file %s.\n"), + opt.input_metalink); + retr_err = METALINK_PARSE_ERROR; + } + else + { + /* We need to sort the resources if preferred location + was specified by the user. */ + if (opt.preferred_location && opt.preferred_location[0]) + { + metalink_file_t **mfile_ptr; + for (mfile_ptr = metalink->files; *mfile_ptr; mfile_ptr++) + { + metalink_resource_t **mres_ptr; + metalink_file_t *mfile = *mfile_ptr; + size_t mres_count = 0; + + for (mres_ptr = mfile->resources; *mres_ptr; mres_ptr++) + mres_count++; + + stable_sort (mfile->resources, + mres_count, + sizeof (metalink_resource_t *), + metalink_res_cmp); + } + } + retr_err = retrieve_from_metalink (metalink); + if (retr_err != RETROK) + { + logprintf (LOG_NOTQUIET, + _("Could not download all resources from %s.\n"), + quote (opt.input_metalink)); + } + metalink_delete (metalink); + } + inform_exit_status (retr_err); + } +#endif /* HAVE_METALINK */ + + /* Print broken links. */ + if (opt.recursive && opt.spider) + print_broken_links (); + + /* Print the downloaded sum. */ + if ((opt.recursive || opt.page_requisites + || nurls > 1 + || (opt.input_filename && total_downloaded_bytes != 0)) + && + total_downloaded_bytes != 0) + { + double end_time = ptimer_measure (timer); + char *wall_time = xstrdup (secs_to_human_time (end_time - start_time)); + char *download_time = xstrdup (secs_to_human_time (total_download_time)); + + ptimer_destroy (timer); timer = NULL; + + logprintf (LOG_NOTQUIET, + _("FINISHED --%s--\nTotal wall clock time: %s\n" + "Downloaded: %d files, %s in %s (%s)\n"), + datetime_str (time (NULL)), + wall_time, + numurls, + human_readable (total_downloaded_bytes, 10, 1), + download_time, + retr_rate (total_downloaded_bytes, total_download_time)); + xfree (wall_time); + xfree (download_time); + + /* Print quota warning, if exceeded. */ + if (opt.quota && total_downloaded_bytes > opt.quota) + logprintf (LOG_NOTQUIET, + _("Download quota of %s EXCEEDED!\n"), + human_readable (opt.quota, 10, 1)); + } + + if (opt.cookies_output) + save_cookies (); + +#ifdef HAVE_HSTS + if (opt.hsts && hsts_store) + save_hsts (); +#endif + + if ((opt.convert_links || opt.convert_file_only) && !opt.delete_after) + convert_all_links (); + + cleanup (); + + exit (get_exit_status ()); +} + +/* + * vim: et ts=2 sw=2 + */ diff --git a/src/metalink.c b/src/metalink.c new file mode 100644 index 0000000..81cd1b3 --- /dev/null +++ b/src/metalink.c @@ -0,0 +1,1599 @@ +/* Metalink module. + Copyright (C) 2015, 2018-2020 Free Software Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" +#ifdef HAVE_METALINK + +#include "metalink.h" +#include "retr.h" +#include "exits.h" +#include "utils.h" +#include "md2.h" +#include "md4.h" +#include "md5.h" +#include "sha1.h" +#include "sha256.h" +#include "sha512.h" +#include "filename.h" +#include "xmemdup0.h" +#include "xstrndup.h" +#include "c-strcase.h" +#include <errno.h> +#include <unistd.h> /* For unlink. */ +#include <metalink/metalink_parser.h> +#ifdef HAVE_GPGME +#include <gpgme.h> +#include <fcntl.h> /* For open and close. */ +#endif + +#ifdef TESTING +#include "../tests/unit-tests.h" +#endif + +/* Loop through all files in metalink structure and retrieve them. + Returns RETROK if all files were downloaded. + Returns last retrieval error (from retrieve_url) if some files + could not be downloaded. */ +uerr_t +retrieve_from_metalink (const metalink_t* metalink) +{ + metalink_file_t **mfile_ptr; + uerr_t last_retr_err = RETROK; /* Store last encountered retrieve error. */ + + FILE *_output_stream = output_stream; + bool _output_stream_regular = output_stream_regular; + char *_output_document = opt.output_document; + + /* metalink file counter */ + unsigned mfc = 0; + + /* metalink retrieval type */ + const char *metatpy = metalink->origin ? "Metalink/HTTP" : "Metalink/XML"; + + /* metalink mother source */ + char *metasrc = metalink->origin ? metalink->origin : opt.input_metalink; + + DEBUGP (("Retrieving from Metalink %s\n", quote (metasrc))); + + /* No files to download. */ + if (!metalink->files) + return RETROK; + + if (opt.output_document) + { + /* We cannot support output_document as we need to compute checksum + of downloaded file, and to remove it if the checksum is bad. */ + logputs (LOG_NOTQUIET, + _("-O not supported for metalink download. Ignoring.\n")); + } + + for (mfile_ptr = metalink->files; *mfile_ptr; mfile_ptr++) + { + metalink_file_t *mfile = *mfile_ptr; + metalink_resource_t **mres_ptr; + char *planname = NULL; + char *trsrname = NULL; + char *filename; + char *basename; + char *safename = NULL; + char *destname = NULL; + bool size_ok = false; + bool hash_ok = false; + + uerr_t retr_err = METALINK_MISSING_RESOURCE; + + /* -1 -> file should be rejected + 0 -> could not verify + 1 -> verified successfully */ + char sig_status = 0; + + bool skip_mfile = false; + + output_stream = NULL; + + mfc++; + + /* The directory prefix for opt.metalink_over_http is handled by + src/url.c (url_file_name), do not add it a second time. */ + if (!metalink->origin && opt.dir_prefix && strlen (opt.dir_prefix)) + planname = aprintf ("%s/%s", opt.dir_prefix, mfile->name); + else + planname = xstrdup (mfile->name); + + /* With Metalink/HTTP, trust the metalink file name (from cli). + With --trust-server-names, trust the Metalink/XML file name, + otherwise, use the basename of --input-metalink followed by + the metalink file counter as suffix. */ + if (metalink->origin || opt.trustservernames) + { + trsrname = xstrdup (mfile->name); + } + else + { + trsrname = xstrdup (get_metalink_basename (opt.input_metalink)); + append_suffix_number (&trsrname, ".#", mfc); + } + + /* Add the directory prefix for opt.input_metalink. */ + if (!metalink->origin && opt.dir_prefix && strlen (opt.dir_prefix)) + filename = aprintf ("%s/%s", opt.dir_prefix, trsrname); + else + filename = xstrdup (trsrname); + + /* Enforce libmetalink's metalink_check_safe_path(). */ + basename = get_metalink_basename (filename); + safename = metalink_check_safe_path (filename) ? filename : basename; + + DEBUGP (("Processing metalink file %s...\n", quote (mfile->name))); + DEBUGP (("\n")); + DEBUGP ((" %s\n", metatpy)); + DEBUGP (("\n")); + DEBUGP ((" --trust-server-names %s\n", opt.trustservernames ? "true" : "false")); + DEBUGP ((" --directory-prefix %s\n", quote (opt.dir_prefix ? opt.dir_prefix : ""))); + DEBUGP (("\n")); + DEBUGP ((" Counted metalink file %u\n", mfc)); + DEBUGP ((" Planned metalink file %s\n", quote (planname ? planname : ""))); + DEBUGP ((" Trusted metalink file %s\n", quote (trsrname ? trsrname : ""))); + DEBUGP ((" Current metalink file %s\n", quote (filename ? filename : ""))); + DEBUGP ((" Cleaned metalink file %s\n", quote (basename ? basename : ""))); + DEBUGP ((" Secured metalink file %s\n", quote (safename ? safename : ""))); + DEBUGP (("\n")); + + /* Verify if the planned metalink file name is safe. */ + if (!safename || strcmp (planname, safename)) + { + logprintf (LOG_NOTQUIET, + _("[--trust-server-names %s, --directory-prefix=%s]\n"), + (opt.trustservernames ? "true" : "false"), + quote (opt.dir_prefix ? opt.dir_prefix : "")); + logprintf (LOG_NOTQUIET, + _("Planned metalink file: %s\n"), + quote (planname ? planname : "")); + logprintf (LOG_NOTQUIET, + _("Secured metalink file: %s\n"), + quote (safename ? safename : "")); + if (!safename) + { + logprintf (LOG_NOTQUIET, + _("Rejecting metalink file. Unsafe name.\n")); + xfree (planname); + xfree (trsrname); + xfree (filename); + continue; + } + } + + /* Process the chosen application/metalink4+xml metaurl. */ + if (opt.metalink_index >= 0) + { + int _metalink_index = opt.metalink_index; + + metalink_metaurl_t **murl_ptr; + int abs_count = 0, meta_count = 0; + + uerr_t x_retr_err = METALINK_MISSING_RESOURCE; + + opt.metalink_index = -1; + + DEBUGP (("Searching application/metalink4+xml ordinal number %d...\n", _metalink_index)); + + if (mfile->metaurls && mfile->metaurls[0]) + for (murl_ptr = mfile->metaurls; *murl_ptr; murl_ptr++) + { + metalink_t* metaurl_xml; + metalink_error_t meta_err; + metalink_metaurl_t *murl = *murl_ptr; + + char *_dir_prefix = opt.dir_prefix; + char *_input_metalink = opt.input_metalink; + + char *metafile = NULL; + char *metadest = NULL; + char *metadir = NULL; + + abs_count++; + + if (strcmp (murl->mediatype, "application/metalink4+xml")) + continue; + + meta_count++; + + DEBUGP ((" Ordinal number %d: %s\n", meta_count, quote (murl->url))); + + if (_metalink_index > 0) + { + if (meta_count < _metalink_index) + continue; + else if (meta_count > _metalink_index) + break; + } + + logprintf (LOG_NOTQUIET, + _("Processing metaurl %s...\n"), quote (murl->url)); + + /* Metalink/XML download file name. */ + metafile = xstrdup (safename); + + if (opt.trustservernames) + replace_metalink_basename (&metafile, murl->name ? murl->name : murl->url); + else + append_suffix_number (&metafile, ".meta#", meta_count); + + if (!metalink_check_safe_path (metafile)) + { + logprintf (LOG_NOTQUIET, + _("Rejecting metaurl file %s. Unsafe name.\n"), + quote (metafile)); + xfree (metafile); + if (_metalink_index > 0) + break; + continue; + } + + /* For security reasons, always save metalink metaurl + files as new unique files. Keep them on failure. */ + x_retr_err = fetch_metalink_file (murl->url, false, false, + metafile, &metadest); + + /* On failure, try the next metalink metaurl. */ + if (x_retr_err != RETROK) + { + logprintf (LOG_VERBOSE, + _("Failed to download %s. Skipping metaurl.\n"), + quote (metadest ? metadest : metafile)); + inform_exit_status (x_retr_err); + xfree (metadest); + xfree (metafile); + if (_metalink_index > 0) + break; + continue; + } + + /* Parse Metalink/XML. */ + meta_err = metalink_parse_file (metadest, &metaurl_xml); + + /* On failure, try the next metalink metaurl. */ + if (meta_err) + { + logprintf (LOG_NOTQUIET, + _("Unable to parse metaurl file %s.\n"), quote (metadest)); + x_retr_err = METALINK_PARSE_ERROR; + inform_exit_status (x_retr_err); + xfree (metadest); + xfree (metafile); + if (_metalink_index > 0) + break; + continue; + } + + /* We need to sort the resources if preferred location + was specified by the user. */ + if (opt.preferred_location && opt.preferred_location[0]) + { + metalink_file_t **x_mfile_ptr; + for (x_mfile_ptr = metaurl_xml->files; *x_mfile_ptr; x_mfile_ptr++) + { + metalink_resource_t **x_mres_ptr; + metalink_file_t *x_mfile = *x_mfile_ptr; + size_t mres_count = 0; + + for (x_mres_ptr = x_mfile->resources; *x_mres_ptr; x_mres_ptr++) + mres_count++; + + stable_sort (x_mfile->resources, + mres_count, + sizeof (metalink_resource_t *), + metalink_res_cmp); + } + } + + /* Insert the current "Directory Options". */ + if (metalink->origin) + { + /* WARNING: Do not use lib/dirname.c (dir_name) to + get the directory name, it may append a dot '.' + character to the directory name. */ + metadir = xstrdup (planname); + replace_metalink_basename (&metadir, NULL); + } + else + { + metadir = xstrdup (opt.dir_prefix); + } + + opt.dir_prefix = metadir; + opt.input_metalink = metadest; + + x_retr_err = retrieve_from_metalink (metaurl_xml); + + if (x_retr_err != RETROK) + logprintf (LOG_NOTQUIET, + _("Could not download all resources from %s.\n"), + quote (metadest)); + + metalink_delete (metaurl_xml); + metaurl_xml = NULL; + + opt.input_metalink = _input_metalink; + opt.dir_prefix = _dir_prefix; + + xfree (metadir); + xfree (metadest); + xfree (metafile); + + break; + } + + if (x_retr_err != RETROK) + logprintf (LOG_NOTQUIET, _("Metaurls processing returned with error.\n")); + + xfree (destname); + xfree (filename); + xfree (trsrname); + xfree (planname); + + opt.output_document = _output_document; + output_stream_regular = _output_stream_regular; + output_stream = _output_stream; + + opt.metalink_index = _metalink_index; + + return x_retr_err; + } + + /* Resources are sorted by priority. */ + for (mres_ptr = mfile->resources; + *mres_ptr && mfile->checksums && !skip_mfile; mres_ptr++) + { + metalink_resource_t *mres = *mres_ptr; + metalink_checksum_t **mchksum_ptr, *mchksum; + struct iri *iri; + struct url *url; + file_stats_t flstats; + int url_err; + + clean_metalink_string (&mres->url); + + if (!RES_TYPE_SUPPORTED (mres->type)) + { + logprintf (LOG_VERBOSE, + _("Resource type %s not supported, ignoring...\n"), + quote (mres->type)); + continue; + } + + /* The file is fully downloaded, but some problems were + encountered (checksum failure?). The loop had been + continued to switch to the next url. */ + if (output_stream && retr_err == RETROK) + { + /* Do not rename/remove a continued file. Skip it. */ + if (opt.always_rest) + { + skip_mfile = true; + continue; + } + + fclose (output_stream); + output_stream = NULL; + badhash_or_remove (destname); + xfree (destname); + } + else if (!output_stream && destname) + { + xfree (destname); + } + + retr_err = METALINK_RETR_ERROR; + + /* Parse our resource URL. */ + iri = iri_new (); + set_uri_encoding (iri, opt.locale, true); + url = url_parse (mres->url, &url_err, iri, false); + + if (!url) + { + char *error = url_error (mres->url, url_err); + logprintf (LOG_NOTQUIET, "%s: %s.\n", mres->url, error); + xfree (error); + inform_exit_status (URLERROR); + iri_free (iri); + continue; + } + else + { + /* Avoid recursive Metalink from HTTP headers. */ + bool _metalink_http = opt.metalink_over_http; + + /* If output_stream is not NULL, then we have failed on + previous resource and are retrying. Thus, continue + with the next resource. Do not close output_stream + while iterating over the resources, or the download + progress will be lost. */ + if (output_stream) + { + DEBUGP (("Previous resource failed, continue with next resource.\n")); + } + else + { + /* Assure proper local file name regardless of the URL + of particular Metalink resource. + To do that we create the local file here and put + it as output_stream. We restore the original configuration + after we are finished with the file. */ + if (opt.always_rest) + /* continue previous download */ + output_stream = fopen (safename, "ab"); + else + /* create a file with an unique name */ + output_stream = unique_create (safename, true, &destname); + } + + output_stream_regular = true; + + /* + At this point, if output_stream is NULL, the file + couldn't be created/opened. + + This happens when the metalink:file has a "path/file" + name format and its directory tree cannot be created: + * stdio.h (fopen) + * src/utils.c (unique_create) + + RFC5854 requires a proper "path/file" format handling, + this can be achieved setting opt.output_document while + output_stream is left to NULL: + * src/http.c (open_output_stream): If output_stream is + NULL, create the opt.output_document "path/file" + */ + if (!destname) + destname = xstrdup (safename); + + /* Store the real file name for displaying in messages, + and for proper RFC5854 "path/file" handling. */ + opt.output_document = destname; + + opt.metalink_over_http = false; + DEBUGP (("Storing to %s\n", destname)); + retr_err = retrieve_url (url, mres->url, NULL, NULL, + NULL, NULL, opt.recursive, iri, false); + opt.metalink_over_http = _metalink_http; + + /* + Bug: output_stream is NULL, but retrieve_url() somehow + created destname. + + Bugfix: point output_stream to destname if it exists. + */ + memset(&flstats, 0, sizeof(flstats)); + if (!output_stream && file_exists_p (destname, &flstats)) + output_stream = fopen_stat (destname, "ab", &flstats); + } + url_free (url); + iri_free (iri); + + if (retr_err == RETROK) + { + FILE *local_file; + + /* Check the digest. */ + local_file = fopen (destname, "rb"); + if (!local_file) + { + logprintf (LOG_NOTQUIET, _("Could not open downloaded file.\n")); + continue; + } + + size_ok = false; + logprintf (LOG_VERBOSE, _("Computing size for %s\n"), quote (destname)); + + if (!mfile->size) + { + size_ok = true; + logprintf (LOG_VERBOSE, _("File size not declared. Skipping check.\n")); + } + else + { + wgint local_file_size = file_size (destname); + + if (local_file_size == -1) + { + logprintf (LOG_NOTQUIET, _("Could not get downloaded file's size.\n")); + fclose (local_file); + local_file = NULL; + continue; + } + + /* FIXME: what about int64? */ + DEBUGP (("Declared size: %lld\n", mfile->size)); + DEBUGP (("Computed size: %lld\n", (long long) local_file_size)); + + if (local_file_size != (wgint) mfile->size) + { + logprintf (LOG_NOTQUIET, _("Size mismatch for file %s.\n"), quote (destname)); + fclose (local_file); + local_file = NULL; + continue; + } + else + { + size_ok = true; + logputs (LOG_VERBOSE, _("Size matches.\n")); + } + } + + for (mchksum_ptr = mfile->checksums; *mchksum_ptr; mchksum_ptr++) + { + char md2[MD2_DIGEST_SIZE]; + char md2_txt[2 * MD2_DIGEST_SIZE + 1]; + + char md4[MD4_DIGEST_SIZE]; + char md4_txt[2 * MD4_DIGEST_SIZE + 1]; + + char md5[MD5_DIGEST_SIZE]; + char md5_txt[2 * MD5_DIGEST_SIZE + 1]; + + char sha1[SHA1_DIGEST_SIZE]; + char sha1_txt[2 * SHA1_DIGEST_SIZE + 1]; + + char sha224[SHA224_DIGEST_SIZE]; + char sha224_txt[2 * SHA224_DIGEST_SIZE + 1]; + + char sha256[SHA256_DIGEST_SIZE]; + char sha256_txt[2 * SHA256_DIGEST_SIZE + 1]; + + char sha384[SHA384_DIGEST_SIZE]; + char sha384_txt[2 * SHA384_DIGEST_SIZE + 1]; + + char sha512[SHA512_DIGEST_SIZE]; + char sha512_txt[2 * SHA512_DIGEST_SIZE + 1]; + + hash_ok = false; + mchksum = *mchksum_ptr; + + /* I have seen both variants... */ + if (c_strcasecmp (mchksum->type, "md2") + && c_strcasecmp (mchksum->type, "md4") + && c_strcasecmp (mchksum->type, "md5") + && c_strcasecmp (mchksum->type, "sha1") + && c_strcasecmp (mchksum->type, "sha-1") + && c_strcasecmp (mchksum->type, "sha224") + && c_strcasecmp (mchksum->type, "sha-224") + && c_strcasecmp (mchksum->type, "sha256") + && c_strcasecmp (mchksum->type, "sha-256") + && c_strcasecmp (mchksum->type, "sha384") + && c_strcasecmp (mchksum->type, "sha-384") + && c_strcasecmp (mchksum->type, "sha512") + && c_strcasecmp (mchksum->type, "sha-512")) + { + DEBUGP (("Ignoring unsupported checksum type %s.\n", + quote (mchksum->type))); + continue; + } + + logprintf (LOG_VERBOSE, _("Computing checksum for %s\n"), + quote (destname)); + + DEBUGP (("Declared hash: %s\n", mchksum->hash)); + + if (c_strcasecmp (mchksum->type, "md2") == 0) + { + md2_stream (local_file, md2); + wg_hex_to_string (md2_txt, md2, MD2_DIGEST_SIZE); + DEBUGP (("Computed hash: %s\n", md2_txt)); + if (!strcmp (md2_txt, mchksum->hash)) + hash_ok = true; + } + else if (c_strcasecmp (mchksum->type, "md4") == 0) + { + md4_stream (local_file, md4); + wg_hex_to_string (md4_txt, md4, MD4_DIGEST_SIZE); + DEBUGP (("Computed hash: %s\n", md4_txt)); + if (!strcmp (md4_txt, mchksum->hash)) + hash_ok = true; + } + else if (c_strcasecmp (mchksum->type, "md5") == 0) + { + md5_stream (local_file, md5); + wg_hex_to_string (md5_txt, md5, MD5_DIGEST_SIZE); + DEBUGP (("Computed hash: %s\n", md5_txt)); + if (!strcmp (md5_txt, mchksum->hash)) + hash_ok = true; + } + else if (c_strcasecmp (mchksum->type, "sha1") == 0 + || c_strcasecmp (mchksum->type, "sha-1") == 0) + { + sha1_stream (local_file, sha1); + wg_hex_to_string (sha1_txt, sha1, SHA1_DIGEST_SIZE); + DEBUGP (("Computed hash: %s\n", sha1_txt)); + if (!strcmp (sha1_txt, mchksum->hash)) + hash_ok = true; + } + else if (c_strcasecmp (mchksum->type, "sha224") == 0 + || c_strcasecmp (mchksum->type, "sha-224") == 0) + { + sha224_stream (local_file, sha224); + wg_hex_to_string (sha224_txt, sha224, SHA224_DIGEST_SIZE); + DEBUGP (("Computed hash: %s\n", sha224_txt)); + if (!strcmp (sha224_txt, mchksum->hash)) + hash_ok = true; + } + else if (c_strcasecmp (mchksum->type, "sha256") == 0 + || c_strcasecmp (mchksum->type, "sha-256") == 0) + { + sha256_stream (local_file, sha256); + wg_hex_to_string (sha256_txt, sha256, SHA256_DIGEST_SIZE); + DEBUGP (("Computed hash: %s\n", sha256_txt)); + if (!strcmp (sha256_txt, mchksum->hash)) + hash_ok = true; + } + else if (c_strcasecmp (mchksum->type, "sha384") == 0 + || c_strcasecmp (mchksum->type, "sha-384") == 0) + { + sha384_stream (local_file, sha384); + wg_hex_to_string (sha384_txt, sha384, SHA384_DIGEST_SIZE); + DEBUGP (("Computed hash: %s\n", sha384_txt)); + if (!strcmp (sha384_txt, mchksum->hash)) + hash_ok = true; + } + else if (c_strcasecmp (mchksum->type, "sha512") == 0 + || c_strcasecmp (mchksum->type, "sha-512") == 0) + { + sha512_stream (local_file, sha512); + wg_hex_to_string (sha512_txt, sha512, SHA512_DIGEST_SIZE); + DEBUGP (("Computed hash: %s\n", sha512_txt)); + if (!strcmp (sha512_txt, mchksum->hash)) + hash_ok = true; + } + + if (hash_ok) + { + logputs (LOG_VERBOSE, + _("Checksum matches.\n")); + } + else + { + logprintf (LOG_NOTQUIET, + _("Checksum mismatch for file %s.\n"), + quote (destname)); + } + + /* Stop as soon as we checked the supported checksum. */ + break; + } /* Iterate over available checksums. */ + fclose (local_file); + local_file = NULL; + + if (!hash_ok) + continue; + + sig_status = 0; /* Not verified. */ + +#ifdef HAVE_GPGME + /* Check the crypto signature. + + Note that the signatures from Metalink in XML will not be + parsed when using libmetalink version older than 0.1.3. + Metalink-over-HTTP is not affected by this problem. */ + if (mfile->signature) + { + metalink_signature_t *msig = mfile->signature; + gpgme_error_t gpgerr; + gpgme_ctx_t gpgctx; + gpgme_data_t gpgsigdata, gpgdata; + gpgme_verify_result_t gpgres; + gpgme_signature_t gpgsig; + int fd; + + /* Initialize the library - as name suggests. */ + gpgme_check_version (NULL); + + /* Open data file. */ + fd = open (destname, O_RDONLY); + if (fd == -1) + { + logputs (LOG_NOTQUIET, + _("Could not open downloaded file for signature " + "verification.\n")); + goto gpg_skip_verification; + } + + /* Assign file descriptor to GPG data structure. */ + gpgerr = gpgme_data_new_from_fd (&gpgdata, fd); + if (gpgerr != GPG_ERR_NO_ERROR) + { + logprintf (LOG_NOTQUIET, + "GPGME data_new_from_fd: %s\n", + gpgme_strerror (gpgerr)); + goto gpg_skip_verification; + } + + /* Prepare new GPGME context. */ + gpgerr = gpgme_new (&gpgctx); + if (gpgerr != GPG_ERR_NO_ERROR) + { + logprintf (LOG_NOTQUIET, + "GPGME new: %s\n", + gpgme_strerror (gpgerr)); + gpgme_data_release (gpgdata); + goto gpg_skip_verification; + } + + DEBUGP (("Verifying signature %s:\n%s\n", + quote (msig->mediatype), + msig->signature)); + + /* Check signature type. */ + if (strcmp (msig->mediatype, "application/pgp-signature")) + { + /* Unsupported signature type. */ + gpgme_release (gpgctx); + gpgme_data_release (gpgdata); + goto gpg_skip_verification; + } + + gpgerr = gpgme_set_protocol (gpgctx, GPGME_PROTOCOL_OpenPGP); + if (gpgerr != GPG_ERR_NO_ERROR) + { + logprintf (LOG_NOTQUIET, + "GPGME set_protocol: %s\n", + gpgme_strerror (gpgerr)); + gpgme_release (gpgctx); + gpgme_data_release (gpgdata); + goto gpg_skip_verification; + } + + /* Load the signature. */ + gpgerr = gpgme_data_new_from_mem (&gpgsigdata, + msig->signature, + strlen (msig->signature), + 0); + if (gpgerr != GPG_ERR_NO_ERROR) + { + logprintf (LOG_NOTQUIET, + _("GPGME data_new_from_mem: %s\n"), + gpgme_strerror (gpgerr)); + gpgme_release (gpgctx); + gpgme_data_release (gpgdata); + goto gpg_skip_verification; + } + + /* Verify the signature. */ + gpgerr = gpgme_op_verify (gpgctx, gpgsigdata, gpgdata, NULL); + if (gpgerr != GPG_ERR_NO_ERROR) + { + logprintf (LOG_NOTQUIET, + _("GPGME op_verify: %s\n"), + gpgme_strerror (gpgerr)); + gpgme_data_release (gpgsigdata); + gpgme_release (gpgctx); + gpgme_data_release (gpgdata); + goto gpg_skip_verification; + } + + /* Check the results. */ + gpgres = gpgme_op_verify_result (gpgctx); + if (!gpgres) + { + logputs (LOG_NOTQUIET, + _("GPGME op_verify_result: NULL\n")); + gpgme_data_release (gpgsigdata); + gpgme_release (gpgctx); + gpgme_data_release (gpgdata); + goto gpg_skip_verification; + } + + /* The list is null-terminated. */ + for (gpgsig = gpgres->signatures; gpgsig; gpgsig = gpgsig->next) + { + DEBUGP (("Checking signature %s\n", gpgsig->fpr)); + + if (gpgsig->summary + & (GPGME_SIGSUM_VALID | GPGME_SIGSUM_GREEN)) + { + logputs (LOG_VERBOSE, + _("Signature validation succeeded.\n")); + sig_status = 1; + break; + } + + if (gpgsig->summary & GPGME_SIGSUM_RED) + { + logputs (LOG_NOTQUIET, + _("Invalid signature. Rejecting resource.\n")); + sig_status = -1; + break; + } + + if (gpgsig->summary == 0 + && (gpgsig->status & 0xFFFF) == GPG_ERR_NO_ERROR) + { + logputs (LOG_VERBOSE, + _("Data matches signature, but signature " + "is not trusted.\n")); + } + + if ((gpgsig->status & 0xFFFF) != GPG_ERR_NO_ERROR) + { + logprintf (LOG_NOTQUIET, + "GPGME: %s\n", + gpgme_strerror (gpgsig->status & 0xFFFF)); + } + } + gpgme_data_release (gpgsigdata); + gpgme_release (gpgctx); + gpgme_data_release (gpgdata); +gpg_skip_verification: + if (fd != -1) + close (fd); + } /* endif (mfile->signature) */ +#endif + /* Stop if file was downloaded with success. */ + if (sig_status >= 0) + break; + } /* endif RETR_OK. */ + } /* Iterate over resources. */ + + if (!mfile->checksums) + { + logprintf (LOG_NOTQUIET, _("No checksums found.\n")); + retr_err = METALINK_CHKSUM_ERROR; + } + + if (retr_err != RETROK) + { + logprintf (LOG_VERBOSE, _("Failed to download %s. Skipping resource.\n"), + quote (destname ? destname : safename)); + } + else if (!size_ok) + { + retr_err = METALINK_SIZE_ERROR; + logprintf (LOG_NOTQUIET, + _("File %s retrieved but size does not match. " + "\n"), quote (destname)); + } + else if (!hash_ok) + { + retr_err = METALINK_CHKSUM_ERROR; + logprintf (LOG_NOTQUIET, + _("File %s retrieved but checksum does not match. " + "\n"), quote (destname)); + } +#ifdef HAVE_GPGME + /* Signature will be only validated if hash check was successful. */ + else if (sig_status < 0) + { + retr_err = METALINK_SIG_ERROR; + logprintf (LOG_NOTQUIET, + _("File %s retrieved but signature does not match. " + "\n"), quote (destname)); + } +#endif + last_retr_err = retr_err == RETROK ? last_retr_err : retr_err; + + /* Rename the file if error encountered; remove if option specified. + Note: the file has been downloaded using *_loop. Therefore, it + is not necessary to keep the file for continuated download. */ + if (((retr_err != RETROK && !opt.always_rest) || opt.delete_after) + && destname != NULL && file_exists_p (destname, NULL)) + { + badhash_or_remove (destname); + } + if (output_stream) + { + fclose (output_stream); + output_stream = NULL; + } + xfree (destname); + xfree (filename); + xfree (trsrname); + xfree (planname); + } /* Iterate over files. */ + + /* Restore original values. */ + opt.output_document = _output_document; + output_stream_regular = _output_stream_regular; + output_stream = _output_stream; + + return last_retr_err; +} + +/* + Replace/remove the basename of a file name. + + The file name is permanently modified. + + Always set NAME to a string, even an empty one. + + Use REF's basename as replacement. If REF is NULL or if it doesn't + provide a valid basename candidate, then remove NAME's basename. +*/ +void +replace_metalink_basename (char **name, char *ref) +{ + int n; + char *p, *new, *basename; + + if (!name) + return; + + /* Strip old basename. */ + if (*name) + { + basename = last_component (*name); + + if (basename == *name) + xfree (*name); + else + *basename = '\0'; + } + + /* New basename from file name reference. */ + if (ref) + ref = last_component (ref); + + /* Replace the old basename. */ + new = aprintf ("%s%s", *name ? *name : "", ref ? ref : ""); + xfree (*name); + *name = new; + + /* Remove prefix drive letters if required, i.e. when in w32 + environments. */ + p = new; + while (p[0] != '\0') + { + while ((n = FILE_SYSTEM_PREFIX_LEN (p)) > 0) + p += n; + + if (p != new) + { + while (ISSLASH (p[0])) + ++p; + new = p; + continue; + } + + break; + } + + if (*name != new) + { + new = xstrdup (new); + xfree (*name); + *name = new; + } +} + +/* + Strip the directory components from the given name. + + Return a pointer to the end of the leading directory components. + Return NULL if the resulting name is unsafe or invalid. + + Due to security issues posed by saving files with unsafe names, here + the use of libmetalink's metalink_check_safe_path() is enforced. If + this appears redundant because the given name was already verified, + just remember to never underestimate unsafe file names. +*/ +char * +get_metalink_basename (char *name) +{ + int n; + char *basename; + + if (!name) + return NULL; + + basename = last_component (name); + + while ((n = FILE_SYSTEM_PREFIX_LEN (basename)) > 0) + basename += n; + + return metalink_check_safe_path (basename) ? basename : NULL; +} + +/* + Append a separator and a numeric suffix to a string. + + The string is permanently modified. +*/ +void +append_suffix_number (char **str, const char *sep, wgint num) +{ + char *new, buf[24]; + + number_to_string (buf, num); + new = aprintf ("%s%s%s", *str ? *str : "", sep ? sep : "", buf); + xfree (*str); + *str = new; +} + +/* + Remove the string's trailing/leading whitespaces and line breaks. + + The string is permanently modified. +*/ +void +clean_metalink_string (char **str) +{ + int c; + size_t len; + char *new, *beg, *end; + + if (!str || !*str) + return; + + beg = *str; + + while ((c = *beg) && (c == '\n' || c == '\r' || c == '\t' || c == ' ')) + beg++; + + end = beg; + + /* To not truncate a string containing spaces, search the first '\r' + or '\n' which ipotetically marks the end of the string. */ + while ((c = *end) && (c != '\r') && (c != '\n')) + end++; + + /* If we are at the end of the string, search the first legit + character going backward. */ + if (*end == '\0') + while ((c = *(end - 1)) && (c == '\n' || c == '\r' || c == '\t' || c == ' ')) + end--; + + len = end - beg; + + new = xmemdup0 (beg, len); + xfree (*str); + *str = new; +} + +/* + Remove the quotation surrounding a string. + + The string is permanently modified. + */ +void +dequote_metalink_string (char **str) +{ + char *new; + size_t str_len; + + if (!str || !*str || ((*str)[0] != '\"' && (*str)[0] != '\'')) + return; + + str_len = strlen (*str); /* current string length */ + + /* Verify if the current string is surrounded by quotes. */ + if (str_len < 2 || (*str)[0] != (*str)[str_len - 1]) + return; + + /* Dequoted string. */ + new = xmemdup0 (*str + 1, str_len - 2); + xfree (*str); + *str = new; +} + +/* Append the suffix ".badhash" to the file NAME, except without + overwriting an existing file with that name and suffix. */ +void +badhash_suffix (char *name) +{ + char *bhash, *uname; + + bhash = concat_strings (name, ".badhash", (char *)0); + uname = unique_name (bhash); + + logprintf (LOG_VERBOSE, _("Renaming %s to %s.\n"), + quote_n (0, name), quote_n (1, uname)); + + if (link (name, uname)) + logprintf (LOG_NOTQUIET, "link: %s\n", strerror (errno)); + else if (unlink (name)) + logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); + + xfree (bhash); + xfree (uname); +} + +/* Append the suffix ".badhash" to the file NAME, except without + overwriting an existing file with that name and suffix. + + Remove the file NAME if the option --delete-after is specified, or + if the option --keep-badhash isn't set. */ +void +badhash_or_remove (char *name) +{ + if (opt.delete_after || !opt.keep_badhash) + { + logprintf (LOG_VERBOSE, _("Removing %s.\n"), quote (name)); + if (unlink (name)) + logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); + } + else + { + badhash_suffix(name); + } +} + +/* + Simple file fetch. + + Set DESTNAME to the name of the saved file. + + Resume previous download if RESUME is true. To disable + Metalink/HTTP, set METALINK_HTTP to false. +*/ +uerr_t +fetch_metalink_file (const char *url_str, + bool resume, bool metalink_http, + const char *filename, char **destname) +{ + FILE *_output_stream = output_stream; + bool _output_stream_regular = output_stream_regular; + char *_output_document = opt.output_document; + bool _metalink_http = opt.metalink_over_http; + + char *local_file = NULL; + + uerr_t retr_err = URLERROR; + + struct iri *iri; + struct url *url; + int url_err; + + /* Parse the URL. */ + iri = iri_new (); + set_uri_encoding (iri, opt.locale, true); + url = url_parse (url_str, &url_err, iri, false); + + if (!url) + { + char *error = url_error (url_str, url_err); + logprintf (LOG_NOTQUIET, "%s: %s.\n", url_str, error); + inform_exit_status (retr_err); + iri_free (iri); + xfree (error); + return retr_err; + } + + output_stream = NULL; + + if (resume) + /* continue previous download */ + output_stream = fopen (filename, "ab"); + else + /* create a file with an unique name */ + output_stream = unique_create (filename, true, &local_file); + + output_stream_regular = true; + + /* + If output_stream is NULL, the file couldn't be created/opened. + This could be due to the impossibility to create a directory tree: + * stdio.h (fopen) + * src/utils.c (unique_create) + + A call to retrieve_url() can indirectly create a directory tree, + when opt.output_document is set to the destination file name and + output_stream is left to NULL: + * src/http.c (open_output_stream): If output_stream is NULL, + create the destination opt.output_document "path/file" + */ + if (!local_file) + local_file = xstrdup (filename); + + /* Store the real file name for displaying in messages, and for + proper "path/file" handling. */ + opt.output_document = local_file; + + opt.metalink_over_http = metalink_http; + + DEBUGP (("Storing to %s\n", local_file)); + retr_err = retrieve_url (url, url_str, NULL, NULL, + NULL, NULL, opt.recursive, iri, false); + + if (retr_err == RETROK) + { + if (destname) + *destname = local_file; + else + xfree (local_file); + } + + if (output_stream) + { + fclose (output_stream); + output_stream = NULL; + } + + opt.metalink_over_http = _metalink_http; + opt.output_document = _output_document; + output_stream_regular = _output_stream_regular; + output_stream = _output_stream; + + inform_exit_status (retr_err); + + iri_free (iri); + url_free (url); + + return retr_err; +} + +int metalink_res_cmp (const void* v1, const void* v2) +{ + const metalink_resource_t *res1 = *(metalink_resource_t **) v1, + *res2 = *(metalink_resource_t **) v2; + if (res1->preference != res2->preference) + return res2->preference - res1->preference; + if (res1->priority != res2->priority) + return res1->priority - res2->priority; + if (opt.preferred_location) + { + int cmp = 0; + if (res1->location && + !c_strcasecmp (opt.preferred_location, res1->location)) + cmp -= 1; + if (res2->location && + !c_strcasecmp (opt.preferred_location, res2->location)) + cmp += 1; + return cmp; + } + return 0; +} + +int metalink_meta_cmp (const void* v1, const void* v2) +{ + const metalink_metaurl_t *meta1 = *(metalink_metaurl_t **) v1, + *meta2 = *(metalink_metaurl_t **) v2; + if (meta1->priority != meta2->priority) + return meta1->priority - meta2->priority; + return 0; +} + +/* + Find value of given key. This is intended for Link header, but will + work with any header that uses ';' as field separator and '=' as key-value + separator. + + Link = "Link" ":" #link-value + link-value = "<" URI-Reference ">" *( ";" link-param ) + link-param = ( ( "rel" "=" relation-types ) + | ( "anchor" "=" <"> URI-Reference <"> ) + | ( "rev" "=" relation-types ) + | ( "hreflang" "=" Language-Tag ) + | ( "media" "=" ( MediaDesc | ( <"> MediaDesc <"> ) ) ) + | ( "title" "=" quoted-string ) + | ( "title*" "=" ext-value ) + | ( "type" "=" ( media-type | quoted-mt ) ) + | ( link-extension ) ) + link-extension = ( parmname [ "=" ( ptoken | quoted-string ) ] ) + | ( ext-name-star "=" ext-value ) + ext-name-star = parmname "*" ; reserved for RFC2231-profiled + ; extensions. Whitespace NOT + ; allowed in between. + ptoken = 1*ptokenchar + ptokenchar = "!" | "#" | "$" | "%" | "&" | "'" | "(" + | ")" | "*" | "+" | "-" | "." | "/" | DIGIT + | ":" | "<" | "=" | ">" | "?" | "@" | ALPHA + | "[" | "]" | "^" | "_" | "`" | "{" | "|" + | "}" | "~" + media-type = type-name "/" subtype-name + quoted-mt = <"> media-type <"> + relation-types = relation-type + | <"> relation-type *( 1*SP relation-type ) <"> + relation-type = reg-rel-type | ext-rel-type + reg-rel-type = LOALPHA *( LOALPHA | DIGIT | "." | "-" ) + ext-rel-type = URI + + See more: rfc5988 +*/ +bool +find_key_value (const char *start, const char *end, const char *key, char **value) +{ + const char *eq; + size_t key_len = strlen (key); + const char *val_beg, *val_end; + const char *key_beg; + + key_beg = start; + + while (key_beg + key_len + 1 < end) + { + /* Skip whitespaces. */ + while (key_beg + key_len + 1 < end && c_isspace (*key_beg)) + key_beg++; + if (strncmp (key_beg, key, key_len)) + { + /* Find next token. */ + while (key_beg + key_len + 1 < end && *key_beg != ';') + key_beg++; + key_beg++; + continue; + } + else + { + /* Find equals sign. */ + eq = key_beg + key_len; + while (eq < end && c_isspace (*eq)) + eq++; + if (eq == end) + return false; + if (*eq != '=') + { + key_beg++; + continue; + } + + val_beg = eq + 1; + while (val_beg < end && c_isspace (*val_beg)) + val_beg++; + if (val_beg == end) + return false; + val_end = val_beg + 1; + while (val_end < end && *val_end != ';' && !c_isspace (*val_end)) + val_end++; + *value = xstrndup (val_beg, val_end - val_beg); + dequote_metalink_string (value); + return true; + } + } + *value = NULL; + return false; +} + +/* This is to check if given token exists in HTTP header. Tokens are + separated by ';'. */ +bool +has_key (const char *start, const char *end, const char *key) +{ + const char *pos; /* Here would the token start. */ + size_t key_len = strlen (key); + + pos = start; + while (pos + key_len <= end) + { + /* Skip whitespaces at beginning. */ + while (pos + key_len <= end && c_isspace (*pos)) + pos++; + + /* Does the prefix of pos match our key? */ + if (strncmp (key, pos, key_len)) + { + /* This was not a match. + Skip all characters until beginning of next token. */ + while (pos + key_len <= end && *pos != ';') + pos++; + pos++; + continue; + } + + /* key is prefix of pos. Is it the exact token or just a prefix? */ + pos += key_len; + while (pos < end && c_isspace (*pos)) + pos++; + if (pos == end || *pos == ';') + return true; + + /* This was not a match (just a prefix). + Skip all characters until beginning of next token. */ + while (pos + key_len <= end && *pos != ';') + pos++; + pos++; + } + return false; +} + +/* Find all key=value pairs delimited with ';' or ','. This is intended for + Digest header parsing. + The usage is: + + const char *pos; + for (pos = header_beg; pos = find_key_values (pos, header_end, &key, &val); pos++) + { + ... + } + + */ +const char * +find_key_values (const char *start, const char *end, char **key, char **value) +{ + const char *key_start, *key_end; + const char *eq; + const char *val_start, *val_end; + + eq = start; + while (eq < end && *eq != '=') + { + /* Skip tokens without =value part. */ + if (*eq == ';' || *eq == ',') + start = eq + 1; + eq++; + } + + if (eq >= end) + return NULL; + + key_start = start; + while (key_start < eq && c_isspace (*key_start)) + key_start++; + + key_end = eq - 1; + while (key_end > key_start && c_isspace (*key_end)) + key_end--; + key_end++; + + val_start = eq + 1; + while (val_start < end && c_isspace (*val_start)) + val_start++; + + val_end = val_start; + + while (val_end < end && *val_end != ';' && + *val_end != ',' && !c_isspace (*val_end)) + val_end++; + + *key = xstrndup (key_start, key_end - key_start); + *value = xstrndup (val_start, val_end - val_start); + dequote_metalink_string (value); + + /* Skip trailing whitespaces. */ + while (val_end < end && c_isspace (*val_end)) + val_end++; + + return val_end; +} + +#ifdef TESTING +const char * +test_find_key_values (void) +{ + static const char *header_data = "key1=val1;key2=\"val2\" ;key3=val3; key4=val4"\ + " ; key5=val5;key6 ='val6';key7= val7; "\ + "key8 = val8 ; key9 = \"val9\" "\ + " ,key10= 'val10',key11,key12=val12"; + static const struct + { + const char *key; + const char *val; + } test_array[] = + { + { "key1", "val1" }, + { "key2", "val2" }, + { "key3", "val3" }, + { "key4", "val4" }, + { "key5", "val5" }, + { "key6", "val6" }, + { "key7", "val7" }, + { "key8", "val8" }, + { "key9", "val9" }, + { "key10", "val10" }, + { "key12", "val12" } + }; + const char *pos; + char *key, *value; + size_t i = 0; + + for (pos = header_data; (pos = find_key_values (pos, + header_data + strlen (header_data), + &key, &value)); pos++) + { + mu_assert ("test_find_key_values: wrong result", + !strcmp (test_array[i].val, value) && + !strcmp (test_array[i].key, key)); + xfree (key); + xfree (value); + i++; + } + + return NULL; +} + +const char * +test_find_key_value (void) +{ + static const char *header_data = "key1=val1;key2=val2 ;key3='val3'; key4=val4"\ + " ; key5='val5';key6 =val6;key7= \"val7\"; "\ + "key8 = \"val8\" ; key9 = val9 "; + static const struct + { + const char *key; + const char *val; + bool result; + } test_array[] = + { + { "key1", "val1", true }, + { "key2", "val2", true }, + { "key3", "val3", true }, + { "key4", "val4", true }, + { "key5", "val5", true }, + { "key6", "val6", true }, + { "key7", "val7", true }, + { "key8", "val8", true }, + { "key9", "val9", true }, + { "key10", NULL, false }, + { "ey1", NULL, false }, + { "dey1", NULL, false } + }; + size_t i; + + for (i=0; i < countof (test_array); ++i) + { + bool result; + char *value; + + result = find_key_value (header_data, + header_data + strlen(header_data), + test_array[i].key, &value); + + mu_assert ("test_find_key_value: wrong result", + result == test_array[i].result && + ((!test_array[i].result && !value) || + !strcmp (value, test_array[i].val))); + + xfree (value); + } + + return NULL; +} + +const char * +test_has_key (void) +{ + static const char *header_data = "key1=val2;token1;xyz; token2;xyz;token3 ;"\ + "xyz; token4 ;xyz; token5 "; + struct + { + const char *token; + bool result; + } test_array[] = + { + { "key1=val2", true }, + { "token1", true }, + { "token2", true }, + { "token3", true }, + { "token4", true }, + { "token5", true }, + { "token6", false }, + { "oken1", false }, + { "poken1", false }, + { "key1=val2", true } + }; + size_t i; + + for (i = 0; i < countof (test_array); ++i) + mu_assert ("test_has_key: wrong result", + has_key (header_data, header_data + strlen (header_data), + test_array[i].token) == test_array[i].result); + + return NULL; +} +#endif + +#endif /* HAVE_METALINK */ diff --git a/src/metalink.h b/src/metalink.h new file mode 100644 index 0000000..b20a421 --- /dev/null +++ b/src/metalink.h @@ -0,0 +1,75 @@ +/* Declarations for metalink.c. + Copyright (C) 2015, 2018-2020 Free Software Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ +#if ! defined METALINK_H && defined HAVE_METALINK +#define METALINK_H + +#include <metalink/metalink_types.h> +#include "dirname.h" +#include "wget.h" + +#ifdef HAVE_SSL +# define RES_TYPE_SUPPORTED(x)\ + ((!x) || !strcmp (x, "http") || !strcmp (x, "https") || !strcmp (x, "ftp") || !strcmp (x, "ftps")) +#else +# define RES_TYPE_SUPPORTED(x)\ + ((!x) || !strcmp (x, "ftp") || !strcmp (x, "http")) +#endif + +#define DEFAULT_PRI 999999 +#define VALID_PRI_RANGE(x) ((x) > 0 && (x) < 1000000) + +uerr_t retrieve_from_metalink (const metalink_t *metalink); + +int metalink_res_cmp (const void *res1, const void *res2); +int metalink_meta_cmp (const void* meta1, const void* meta2); + +int metalink_check_safe_path (const char *path); + +void replace_metalink_basename (char **name, char *ref); +char *get_metalink_basename (char *name); +void append_suffix_number (char **str, const char *sep, wgint num); +void clean_metalink_string (char **str); +void dequote_metalink_string (char **str); +void badhash_suffix (char *name); +void badhash_or_remove (char *name); +uerr_t fetch_metalink_file (const char *url_str, + bool resume, bool metalink_http, + const char *filename, char **destname); + +bool find_key_value (const char *start, + const char *end, + const char *key, + char **value); +bool has_key (const char *start, const char *end, const char *key); +const char *find_key_values (const char *start, + const char *end, + char **key, + char **value); + +#endif /* METALINK_H */ diff --git a/src/mswindows.c b/src/mswindows.c new file mode 100644 index 0000000..42d9f5f --- /dev/null +++ b/src/mswindows.c @@ -0,0 +1,652 @@ +/* mswindows.c -- Windows-specific support + Copyright (C) 1996-2011, 2014-2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#define INHIBIT_WRAP /* avoid wrapping of socket, bind, ... */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <math.h> + + +#include "utils.h" +#include "url.h" +#include "exits.h" + +#ifndef ES_SYSTEM_REQUIRED +#define ES_SYSTEM_REQUIRED 0x00000001 +#endif + +#ifndef ES_CONTINUOUS +#define ES_CONTINUOUS 0x80000000 +#endif + + +/* Windows version of xsleep in utils.c. */ + +void +xsleep (double seconds) +{ +#if defined(HAVE_USLEEP) && defined(HAVE_SLEEP) + if (seconds >= 1) + { + /* Explained in utils.c. */ + sleep (seconds); + seconds -= (long) seconds; + } + usleep (seconds * 1000000); +#else /* not HAVE_USLEEP */ + SleepEx ((DWORD) (seconds * 1000 + .5), FALSE); +#endif /* not HAVE_USLEEP */ +} + +void +windows_main (char **exec_name) +{ + char *p; + + /* Remove .EXE from filename if it has one. */ + *exec_name = xstrdup (*exec_name); + p = strrchr (*exec_name, '.'); + if (p) + *p = '\0'; +} + +static void +ws_cleanup (void) +{ + xfree (exec_name); + WSACleanup (); +} + +#if defined(CTRLBREAK_BACKGND) || defined(CTRLC_BACKGND) +static void +ws_hangup (const char *reason) +{ + fprintf (stderr, _("Continuing in background.\n")); + redirect_output (true, reason); + + /* Detach process from the current console. Under Windows 9x, if we + were launched from a 16-bit process (which is usually the case; + command.com is 16-bit) the parent process should resume right away. + Under NT or if launched from a 32-process under 9x, this is a futile + gesture as the parent will wait for us to terminate before resuming. */ + FreeConsole (); +} +#endif + +/* Construct the name for a named section (a.k.a. `file mapping') object. + The returned string is dynamically allocated and needs to be xfree()'d. */ +static char * +make_section_name (DWORD pid) +{ + return aprintf ("gnu_wget_fake_fork_%lu", pid); +} + +/* This structure is used to hold all the data that is exchanged between + parent and child. */ +struct fake_fork_info +{ + HANDLE event; + bool logfile_changed; + char lfilename[MAX_PATH + 1]; +}; + +/* Determines if we are the child and if so performs the child logic. + Return values: + < 0 error + 0 parent + > 0 child +*/ +static int +fake_fork_child (void) +{ + HANDLE section, event; + struct fake_fork_info *info; + char *name; + + name = make_section_name (GetCurrentProcessId ()); + section = OpenFileMapping (FILE_MAP_WRITE, FALSE, name); + xfree (name); + /* It seems that Windows 9x and NT set last-error inconsistently when + OpenFileMapping() fails; so we assume it failed because the section + object does not exist. */ + if (!section) + return 0; /* We are the parent. */ + + info = MapViewOfFile (section, FILE_MAP_WRITE, 0, 0, 0); + if (!info) + { + CloseHandle (section); + return -1; + } + + event = info->event; + + info->logfile_changed = false; + if (!opt.lfilename && (!opt.quiet || opt.server_response)) + { + /* See utils:fork_to_background for explanation. */ + FILE *new_log_fp = unique_create (DEFAULT_LOGFILE, false, &opt.lfilename); + if (new_log_fp) + { + info->logfile_changed = true; + snprintf (info->lfilename, sizeof (info->lfilename), "%s", + opt.lfilename); + fclose (new_log_fp); + } + } + + UnmapViewOfFile (info); + CloseHandle (section); + + /* Inform the parent that we've done our part. */ + if (!SetEvent (event)) + return -1; + + CloseHandle (event); + return 1; /* We are the child. */ +} + +/* Windows doesn't support the fork() call; so we fake it by invoking + another copy of Wget with the same arguments with which we were + invoked. The child copy of Wget should perform the same initialization + sequence as the parent; so we should have two processes that are + essentially identical. We create a specially named section object that + allows the child to distinguish itself from the parent and is used to + exchange information between the two processes. We use an event object + for synchronization. */ +static void +fake_fork (void) +{ + char exe[MAX_PATH + 1]; + DWORD exe_len, le; + SECURITY_ATTRIBUTES sa; + HANDLE section, event, h[2]; + STARTUPINFO si; + PROCESS_INFORMATION pi; + struct fake_fork_info *info; + char *name; + BOOL rv; + + section = pi.hProcess = pi.hThread = NULL; + + /* Get the fully qualified name of our executable. This is more reliable + than using argv[0]. */ + exe_len = GetModuleFileName (GetModuleHandle (NULL), exe, sizeof (exe)); + if (!exe_len || (exe_len >= sizeof (exe))) + return; + + sa.nLength = sizeof (sa); + sa.lpSecurityDescriptor = NULL; + sa.bInheritHandle = TRUE; + + /* Create an anonymous inheritable event object that starts out + non-signaled. */ + event = CreateEvent (&sa, FALSE, FALSE, NULL); + if (!event) + return; + + /* Create the child process detached form the current console and in a + suspended state. */ + xzero (si); + si.cb = sizeof (si); + rv = CreateProcess (exe, GetCommandLine (), NULL, NULL, TRUE, + CREATE_SUSPENDED | DETACHED_PROCESS, + NULL, NULL, &si, &pi); + if (!rv) + goto cleanup; + + /* Create a named section object with a name based on the process id of + the child. */ + name = make_section_name (pi.dwProcessId); + section = + CreateFileMapping (INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, + sizeof (struct fake_fork_info), name); + le = GetLastError(); + xfree (name); + /* Fail if the section object already exists (should not happen). */ + if (!section || (le == ERROR_ALREADY_EXISTS)) + { + rv = FALSE; + goto cleanup; + } + + /* Copy the event handle into the section object. */ + info = MapViewOfFile (section, FILE_MAP_WRITE, 0, 0, 0); + if (!info) + { + rv = FALSE; + goto cleanup; + } + + info->event = event; + + UnmapViewOfFile (info); + + /* Start the child process. */ + rv = ResumeThread (pi.hThread); + if (!rv) + { + TerminateProcess (pi.hProcess, (DWORD) -1); + goto cleanup; + } + + /* Wait for the child to signal to us that it has done its part. If it + terminates before signaling us it's an error. */ + + h[0] = event; + h[1] = pi.hProcess; + rv = WAIT_OBJECT_0 == WaitForMultipleObjects (2, h, FALSE, 5 * 60 * 1000); + if (!rv) + goto cleanup; + + info = MapViewOfFile (section, FILE_MAP_READ, 0, 0, 0); + if (!info) + { + rv = FALSE; + goto cleanup; + } + + /* Ensure string is properly terminated. */ + if (info->logfile_changed && + !memchr (info->lfilename, '\0', sizeof (info->lfilename))) + { + rv = FALSE; + goto cleanup; + } + + printf (_("Continuing in background, pid %lu.\n"), pi.dwProcessId); + if (info->logfile_changed) + printf (_("Output will be written to %s.\n"), quote (info->lfilename)); + + UnmapViewOfFile (info); + +cleanup: + + if (event) + CloseHandle (event); + if (section) + CloseHandle (section); + if (pi.hThread) + CloseHandle (pi.hThread); + if (pi.hProcess) + CloseHandle (pi.hProcess); + + /* We're the parent. If all is well, terminate. */ + if (rv) + exit (WGET_EXIT_SUCCESS); + + /* We failed, return. */ +} + +/* This is the corresponding Windows implementation of the + fork_to_background() function in utils.c. */ +bool +fork_to_background (void) +{ + int rv; + + rv = fake_fork_child (); + if (rv < 0) + { + fprintf (stderr, _("fake_fork_child() failed\n")); + abort (); + } + else if (rv == 0) + { + /* We're the parent. */ + fake_fork (); + /* If fake_fork() returns, it failed. */ + fprintf (stderr, _("fake_fork() failed\n")); + abort (); + } + /* If we get here, we're the child. */ + return false; +} + +static BOOL WINAPI +ws_handler (DWORD dwEvent) +{ + switch (dwEvent) + { +#ifdef CTRLC_BACKGND + case CTRL_C_EVENT: + ws_hangup ("CTRL+C"); + return TRUE; +#endif +#ifdef CTRLBREAK_BACKGND + case CTRL_BREAK_EVENT: + ws_hangup ("CTRL+Break"); + return TRUE; +#endif + default: + return FALSE; + } +} + +static char *title_buf = NULL; +static char *curr_url = NULL; +static int old_percentage = -1; + +/* Updates the console title with the URL of the current file being + transferred. */ +void +ws_changetitle (const char *url) +{ + xfree (title_buf); + xfree (curr_url); + title_buf = xmalloc (strlen (url) + 20); + curr_url = xstrdup (url); + old_percentage = -1; + sprintf (title_buf, "Wget %s", curr_url); + SetConsoleTitle (title_buf); +} + +/* Updates the console title with the percentage of the current file + transferred. */ +void +ws_percenttitle (double percentage_float) +{ + int percentage; + + if (!title_buf || !curr_url) + return; + + percentage = (int) percentage_float; + + /* Clamp percentage value. */ + if (percentage < 0) + percentage = 0; + if (percentage > 100) + percentage = 100; + + /* Only update the title when the percentage has changed. */ + if (percentage == old_percentage) + return; + + old_percentage = percentage; + + sprintf (title_buf, "Wget [%d%%] %s", percentage, curr_url); + SetConsoleTitle (title_buf); +} + +/* Returns a pointer to the fully qualified name of the directory that + contains the Wget binary (wget.exe). The returned path does not have a + trailing path separator. Returns NULL on failure. */ +char * +ws_mypath (void) +{ + static char *wspathsave = NULL; + + if (!wspathsave) + { + char buf[MAX_PATH + 1]; + char *p; + DWORD len; + + len = GetModuleFileName (GetModuleHandle (NULL), buf, sizeof (buf)); + if (!len || (len >= sizeof (buf))) + return NULL; + + p = strrchr (buf, PATH_SEPARATOR); + if (!p) + return NULL; + + *p = '\0'; + wspathsave = xstrdup (buf); + } + + return wspathsave; +} + +/* Prevent Windows entering sleep/hibernation-mode while Wget is doing + a lengthy transfer. Windows does not, by default, consider network + activity in console-programs as activity! Works on Win-98/ME/2K + and up. */ +static void +set_sleep_mode (void) +{ + typedef DWORD (WINAPI *func_t) (DWORD); + func_t set_exec_state; + + set_exec_state = + (func_t) GetProcAddress (GetModuleHandle ("KERNEL32.DLL"), + "SetThreadExecutionState"); + + if (set_exec_state) + set_exec_state (ES_SYSTEM_REQUIRED | ES_CONTINUOUS); +} + +/* Perform Windows specific initialization. */ +void +ws_startup (void) +{ + WSADATA data; + WORD requested = MAKEWORD (1, 1); + int err = WSAStartup (requested, &data); + if (err != 0) + { + fprintf (stderr, _("%s: Couldn't find usable socket driver.\n"), + exec_name); + exit (WGET_EXIT_GENERIC_ERROR); + } + + if (data.wVersion < requested) + { + fprintf (stderr, _("%s: Couldn't find usable socket driver.\n"), + exec_name); + WSACleanup (); + exit (WGET_EXIT_GENERIC_ERROR); + } + + atexit (ws_cleanup); + set_sleep_mode (); + SetConsoleCtrlHandler (ws_handler, TRUE); +} + +/* run_with_timeout Windows implementation. */ + +/* Stack size 0 uses default thread stack-size (reserve+commit). + Determined by what's in the PE header. */ +#define THREAD_STACK_SIZE 0 + +struct thread_data +{ + void (*fun) (void *); + void *arg; + DWORD ws_error; +}; + +/* The callback that runs FUN(ARG) in a separate thread. This + function exists for two reasons: a) to not require FUN to be + declared WINAPI/__stdcall[1], and b) to retrieve Winsock errors, + which are per-thread. The latter is useful when FUN calls Winsock + functions, which is how run_with_timeout is used in Wget. + + [1] MSVC can use __fastcall globally (cl /Gr) and on Watcom this is + the default (wcc386 -3r). */ + +static DWORD WINAPI +thread_helper (void *arg) +{ + struct thread_data *td = (struct thread_data *) arg; + + /* Initialize Winsock error to what it was in the parent. That way + the subsequent call to WSAGetLastError will return the same value + if td->fun doesn't change Winsock error state. */ + WSASetLastError (td->ws_error); + + td->fun (td->arg); + + /* Return Winsock error to the caller, in case FUN ran Winsock + code. */ + td->ws_error = WSAGetLastError (); + return 0; +} + +/* Call FUN(ARG), but don't allow it to run for more than TIMEOUT + seconds. Returns true if the function was interrupted with a + timeout, false otherwise. + + This works by running FUN in a separate thread and terminating the + thread if it doesn't finish in the specified time. */ + +bool +run_with_timeout (double seconds, void (*fun) (void *), void *arg) +{ + HANDLE thread_hnd; + struct thread_data thread_arg; + DWORD thread_id; + bool rc; + + DEBUGP (("seconds %.2f, ", seconds)); + + if (seconds == 0) + { + blocking_fallback: + fun (arg); + return false; + } + + thread_arg.fun = fun; + thread_arg.arg = arg; + thread_arg.ws_error = WSAGetLastError (); + thread_hnd = CreateThread (NULL, THREAD_STACK_SIZE, thread_helper, + &thread_arg, 0, &thread_id); + if (!thread_hnd) + { + DEBUGP (("CreateThread() failed; [%#lx]\n", + (unsigned long) GetLastError ())); + goto blocking_fallback; + } + + if (WaitForSingleObject (thread_hnd, (DWORD)(1000 * seconds)) + == WAIT_OBJECT_0) + { + /* Propagate error state (which is per-thread) to this thread, + so the caller can inspect it. */ + WSASetLastError (thread_arg.ws_error); + DEBUGP (("Winsock error: %d\n", WSAGetLastError ())); + rc = false; + } + else + { + TerminateThread (thread_hnd, 1); + rc = true; + } + + CloseHandle (thread_hnd); /* Clear-up after TerminateThread(). */ + thread_hnd = NULL; + return rc; +} + + +#ifdef ENABLE_IPV6 +/* An inet_ntop implementation that uses WSAAddressToString. + Prototype complies with POSIX 1003.1-2004. This is only used under + IPv6 because Wget prints IPv4 addresses using inet_ntoa. */ + +const char * +inet_ntop (int af, const void *src, char *dst, socklen_t cnt) +{ + /* struct sockaddr can't accommodate struct sockaddr_in6. */ + union { + struct sockaddr_in6 sin6; + struct sockaddr_in sin; + } sa; + DWORD dstlen = cnt; + size_t srcsize; + + xzero (sa); + switch (af) + { + case AF_INET: + sa.sin.sin_family = AF_INET; + sa.sin.sin_addr = *(struct in_addr *) src; + srcsize = sizeof (sa.sin); + break; + case AF_INET6: + sa.sin6.sin6_family = AF_INET6; + sa.sin6.sin6_addr = *(struct in6_addr *) src; + srcsize = sizeof (sa.sin6); + break; + default: + abort (); + } + + if (WSAAddressToString ((struct sockaddr *) &sa, srcsize, NULL, dst, &dstlen) != 0) + { + errno = WSAGetLastError(); + return NULL; + } + return (const char *) dst; +} +#endif + + +void +set_windows_fd_as_blocking_socket (int fd) +{ + /* 04/2011 + gnulib select() converts blocking sockets to nonblocking in windows + discussed here: + http://old.nabble.com/blocking-socket-is-nonblocking-after-calling-gnulib- + select%28%29-in-windows-td31432857.html + + wget uses blocking sockets so we must convert them back to blocking. + */ + int ret = 0; + int wsagle = 0; + const int zero = 0; + + do + { + if(wsagle == WSAEINPROGRESS) + Sleep(1); /* use windows sleep */ + + WSASetLastError (0); + ret = ioctl (fd, FIONBIO, &zero); + wsagle = WSAGetLastError (); + } + while (ret && (wsagle == WSAEINPROGRESS)); + + if(ret) + { + fprintf (stderr, + _("ioctl() failed. The socket could not be set as blocking.\n") ); + DEBUGP (("Winsock error: %d\n", WSAGetLastError ())); + abort (); + } + return; +} diff --git a/src/mswindows.h b/src/mswindows.h new file mode 100644 index 0000000..a8f9ba0 --- /dev/null +++ b/src/mswindows.h @@ -0,0 +1,78 @@ +/* Declarations for windows + Copyright (C) 1996-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef MSWINDOWS_H +#define MSWINDOWS_H + +#ifndef WGET_H +# error This file should not be included directly. +#endif + +/* Prevent inclusion of <winsock*.h> in <windows.h>. */ +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +#include <windows.h> + +#include <winsock2.h> +#include <ws2tcpip.h> + +#ifndef EAI_SYSTEM +# define EAI_SYSTEM -1 /* value doesn't matter */ +#endif + +/* Declares file access functions, such as open, create, access, and + chmod. Unix declares these in unistd.h and fcntl.h. */ +#include <io.h> + +/* Declares getpid(). */ +#include <process.h> + +/* Declares inet_ntop() and inet_pton(). */ +#include <arpa/inet.h> + +#include <stdio.h> + +#define PATH_SEPARATOR '\\' + +/* ioctl needed by set_windows_fd_as_blocking_socket() */ +#include <sys/ioctl.h> + +/* Public functions. */ + +void ws_startup (void); +void ws_changetitle (const char *); +void ws_percenttitle (double); +char *ws_mypath (void); +void windows_main (char **); +void set_windows_fd_as_blocking_socket (int); + +#endif /* MSWINDOWS_H */ diff --git a/src/netrc.c b/src/netrc.c new file mode 100644 index 0000000..a9232ed --- /dev/null +++ b/src/netrc.c @@ -0,0 +1,569 @@ +/* Read and parse the .netrc file to get hosts, accounts, and passwords. + Copyright (C) 1996, 2007-2011, 2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +/* This file used to be kept in synch with the code in Fetchmail, but + the latter has diverged since. */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> + +#include "utils.h" +#include "netrc.h" +#include "init.h" + +#ifdef WINDOWS +# define NETRC_FILE_NAME "_netrc" +#else +# define NETRC_FILE_NAME ".netrc" +#endif + +typedef struct _acc_t +{ + char *host; /* NULL if this is the default machine + entry. */ + char *acc; + char *passwd; /* NULL if there is no password. */ + struct _acc_t *next; +} acc_t; + +static acc_t *parse_netrc (const char *); +static acc_t *parse_netrc_fp (const char *, FILE *); + +static acc_t *netrc_list; +static int processed_netrc; + +#if defined DEBUG_MALLOC || defined TESTING +static void free_netrc(acc_t *); + +void +netrc_cleanup (void) +{ + free_netrc (netrc_list); + processed_netrc = 0; +} +#endif + +/* Return the correct user and password, given the host, user (as + given in the URL), and password (as given in the URL). May return + NULL. + + If SLACK_DEFAULT is set, allow looking for a "default" account. + You will typically turn it off for HTTP. */ +void +search_netrc (const char *host, const char **acc, const char **passwd, + int slack_default, FILE *fp_netrc) +{ + acc_t *l; + + if (!opt.netrc) + return; + /* Find ~/.netrc. */ + if (!processed_netrc) + { +#ifdef __VMS + + int err; + struct stat buf; + char *path = "SYS$LOGIN:.netrc"; + + netrc_list = NULL; + processed_netrc = 1; + + err = stat (path, &buf); + if (err == 0) + netrc_list = parse_netrc (path); + +#else /* def __VMS */ + + netrc_list = NULL; + processed_netrc = 1; + + if (fp_netrc) + netrc_list = parse_netrc_fp (".netrc", fp_netrc); + else if (opt.homedir) + { + struct stat buf; + char *path = aprintf ("%s/%s", opt.homedir, NETRC_FILE_NAME); + if (stat (path, &buf) == 0) + netrc_list = parse_netrc (path); + xfree (path); + } + +#endif /* def __VMS [else] */ + } + /* If nothing to do... */ + if (!netrc_list) + return; + /* Acc and password found; all OK. */ + if (*acc && *passwd) + return; + /* Some data not given -- try finding the host. */ + for (l = netrc_list; l; l = l->next) + { + if (!l->host) + continue; + else if (!strcasecmp (l->host, host)) + break; + } + if (l) + { + if (*acc) + { + /* Looking for password in .netrc. */ + if (!strcmp (l->acc, *acc)) + *passwd = l->passwd; /* usernames match; password OK */ + else + *passwd = NULL; /* usernames don't match */ + } + else /* NOT *acc */ + { + /* If password was given, use it. The account is l->acc. */ + *acc = l->acc; + if (l->passwd) + *passwd = l->passwd; + } + return; + } + else + { + if (!slack_default) + return; + if (*acc) + return; + /* Try looking for the default account. */ + for (l = netrc_list; l; l = l->next) + if (!l->host) + break; + if (!l) + return; + *acc = l->acc; + if (!*passwd) + *passwd = l->passwd; + return; + } +} + + +#ifdef STANDALONE + +/* Normally, these functions would be defined by your package. */ +# define xmalloc malloc +# define xfree(p) do { free ((void *) (p)); p = NULL; } while (0) +# define xstrdup strdup + +# define xrealloc realloc + +#endif /* STANDALONE */ + +/* Maybe add NEWENTRY to the account information list, LIST. NEWENTRY is + set to a ready-to-use acc_t, in any event. */ +static void +maybe_add_to_list (acc_t **newentry, acc_t **list) +{ + acc_t *a, *l; + a = *newentry; + l = *list; + + /* We need an account name in order to add the entry to the list. */ + if (a && ! a->acc) + { + /* Free any allocated space. */ + xfree (a->host); + xfree (a->acc); + xfree (a->passwd); + } + else + { + if (a) + { + /* Add the current machine into our list. */ + a->next = l; + l = a; + } + + /* Allocate a new acc_t structure. */ + a = xmalloc (sizeof (acc_t)); + } + + /* Zero the structure, so that it is ready to use. */ + memset (a, 0, sizeof(*a)); + + /* Return the new pointers. */ + *newentry = a; + *list = l; + return; +} + +/* Helper function for the parser, shifts contents of + null-terminated string once character to the left. + Used in processing \ and " constructs in the netrc file */ +static void +shift_left(char *string) +{ + char *p; + + for (p=string; *p; ++p) + *p = *(p+1); +} + +/* Parse a .netrc file (as described in the ftp(1) manual page). */ +static acc_t * +parse_netrc_fp (const char *path, FILE *fp) +{ + char *line = NULL, *p, *tok; + const char *premature_token = NULL; + acc_t *current = NULL, *retval = NULL; + int ln = 0, qmark; + size_t bufsize = 0; + + /* The latest token we've seen in the file. */ + enum + { + tok_nothing, tok_account, tok_login, tok_macdef, tok_machine, tok_password, tok_port, tok_force + } last_token = tok_nothing; + + /* While there are lines in the file... */ + while (getline (&line, &bufsize, fp) > 0) + { + ln ++; + + /* Parse the line. */ + p = line; + qmark = 0; + + /* Skip leading whitespace. */ + while (*p && c_isspace (*p)) + p ++; + + /* If the line is empty, then end any macro definition. */ + if (last_token == tok_macdef && !*p) + /* End of macro if the line is empty. */ + last_token = tok_nothing; + + /* If we are defining macros, then skip parsing the line. */ + while (*p && last_token != tok_macdef) + { + /* Skip any whitespace. */ + while (*p && c_isspace (*p)) + p ++; + + /* Discard end-of-line comments; also, stop processing if + the above `while' merely skipped trailing whitespace. */ + if (*p == '#' || !*p) + break; + + /* If the token starts with quotation mark, note this fact, + and squash the quotation character */ + if (*p == '"'){ + qmark = 1; + shift_left (p); + } + + tok = p; + + /* Find the end of the token, handling quotes and escapes. */ + while (*p && (qmark ? *p != '"' : !c_isspace (*p))){ + if (*p == '\\') + shift_left (p); + p ++; + } + + /* If field was quoted, squash the trailing quotation mark + and reset qmark flag. */ + if (qmark) + { + shift_left (p); + qmark = 0; + } + + /* Null-terminate the token, if it isn't already. */ + if (*p) + *p ++ = '\0'; + + switch (last_token) + { + case tok_login: + if (current) + { + xfree (current->acc); + current->acc = xstrdup (tok); + } + else + premature_token = "login"; + break; + + case tok_machine: + /* Start a new machine entry. */ + maybe_add_to_list (¤t, &retval); + current->host = xstrdup (tok); + break; + + case tok_password: + if (current) + { + xfree (current->passwd); + current->passwd = xstrdup (tok); + } + else + premature_token = "password"; + break; + + /* We handle most of tok_macdef above. */ + case tok_macdef: + if (!current) + premature_token = "macdef"; + break; + + /* We don't handle the account keyword at all. */ + case tok_account: + if (!current) + premature_token = "account"; + break; + + /* We don't handle the port keyword at all. */ + case tok_port: + if (!current) + premature_token = "port"; + break; + + /* We don't handle the force keyword at all. */ + case tok_force: + if (!current) + premature_token = "force"; + break; + + /* We handle tok_nothing below this switch. */ + case tok_nothing: + break; + } + + if (premature_token) + { + fprintf (stderr, _("\ +%s: %s:%d: warning: %s token appears before any machine name\n"), + exec_name, path, ln, quote (premature_token)); + premature_token = NULL; + } + + if (last_token != tok_nothing) + /* We got a value, so reset the token state. */ + last_token = tok_nothing; + else + { + /* Fetch the next token. */ + if (!strcmp (tok, "account")) + last_token = tok_account; + + else if (!strcmp (tok, "default")) + maybe_add_to_list (¤t, &retval); + + else if (!strcmp (tok, "login")) + last_token = tok_login; + + else if (!strcmp (tok, "macdef")) + last_token = tok_macdef; + + else if (!strcmp (tok, "machine")) + last_token = tok_machine; + + else if (!strcmp (tok, "password")) + last_token = tok_password; + + /* GNU extensions 'port' and 'force', not operational + * see https://www.gnu.org/software/emacs/manual/html_node/gnus/NNTP.html#index-nntp_002dauthinfo_002dfunction-2003 + * see https://savannah.gnu.org/bugs/index.php?52066 + */ + else if (!strcmp (tok, "port")) + last_token = tok_port; + + else if (!strcmp (tok, "force")) + last_token = tok_force; + + else + fprintf (stderr, _("%s: %s:%d: unknown token \"%s\"\n"), + exec_name, path, ln, tok); + } + } + } + + xfree (line); + + /* Finalize the last machine entry we found. */ + maybe_add_to_list (¤t, &retval); + xfree (current); + + /* Reverse the order of the list so that it appears in file order. */ + current = retval; + retval = NULL; + while (current) + { + acc_t *saved_reference; + + /* Change the direction of the pointers. */ + saved_reference = current->next; + current->next = retval; + + /* Advance to the next node. */ + retval = current; + current = saved_reference; + } + + return retval; +} + +static acc_t * +parse_netrc (const char *path) +{ + FILE *fp; + acc_t *acc; + + fp = fopen (path, "r"); + if (!fp) + { + fprintf (stderr, _("%s: Cannot read %s (%s).\n"), exec_name, + path, strerror (errno)); + return NULL; + } + + acc = parse_netrc_fp (path, fp); + fclose(fp); + + return acc; +} + +#if defined DEBUG_MALLOC || defined TESTING +/* Free a netrc list. */ +static void +free_netrc(acc_t *l) +{ + acc_t *t; + + while (l) + { + t = l->next; + xfree (l->acc); + xfree (l->passwd); + xfree (l->host); + xfree (l); + l = t; + } +} +#endif + +#ifdef STANDALONE +#include <sys/types.h> +#include <sys/stat.h> +#include "exits.h" + +const char *program_argstring = NULL; /* Needed by warc.c */ + +int +main (int argc, char **argv) +{ + struct stat sb; + char *program_name, *file, *target; + acc_t *head, *a; + + if (argc < 2 || argc > 3) + { + fprintf (stderr, _("Usage: %s NETRC [HOSTNAME]\n"), argv[0]); + exit (WGET_EXIT_GENERIC_ERROR); + } + + program_name = argv[0]; + file = argv[1]; + target = argv[2]; + +#ifdef ENABLE_NLS + /* Set the current locale. */ + setlocale (LC_ALL, ""); + /* Set the text message domain. */ + bindtextdomain ("wget", LOCALEDIR); + textdomain ("wget"); +#endif /* ENABLE_NLS */ + + if (stat (file, &sb)) + { + fprintf (stderr, _("%s: cannot stat %s: %s\n"), argv[0], file, + strerror (errno)); + exit (WGET_EXIT_GENERIC_ERROR); + } + + head = parse_netrc (file); + a = head; + while (a) + { + /* Skip if we have a target and this isn't it. */ + if (target && a->host && strcmp (target, a->host)) + { + a = a->next; + continue; + } + + if (!target) + { + /* Print the host name if we have no target. */ + if (a->host) + fputs (a->host, stdout); + else + fputs ("DEFAULT", stdout); + + fputc (' ', stdout); + } + + /* Print the account name. */ + fputs (a->acc, stdout); + + if (a->passwd) + { + /* Print the password, if there is any. */ + fputc (' ', stdout); + fputs (a->passwd, stdout); + } + + fputc ('\n', stdout); + + /* Exit if we found the target. */ + if (target) + exit (WGET_EXIT_SUCCESS); + a = a->next; + } + + /* Exit with failure if we had a target, success otherwise. */ + if (target) + exit (WGET_EXIT_GENERIC_ERROR); + + exit (WGET_EXIT_SUCCESS); +} +#endif /* STANDALONE */ diff --git a/src/netrc.h b/src/netrc.h new file mode 100644 index 0000000..903b8fa --- /dev/null +++ b/src/netrc.h @@ -0,0 +1,39 @@ +/* Declarations for netrc.c + Copyright (C) 1996, 1996-1997, 2007-2011, 2015, 2018-2020 Free + Software Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef NETRC_H +#define NETRC_H + +#include <stdio.h> + +void search_netrc (const char *, const char **, const char **, int, FILE *); +void netrc_cleanup(void); + +#endif /* NETRC_H */ diff --git a/src/openssl.c b/src/openssl.c new file mode 100644 index 0000000..d6099ba --- /dev/null +++ b/src/openssl.c @@ -0,0 +1,1260 @@ +/* SSL support via OpenSSL library. + Copyright (C) 2000-2012, 2015, 2018-2020 Free Software Foundation, + Inc. + Originally contributed by Christian Fraenkel. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <assert.h> +#include <errno.h> +#include <unistd.h> +#include <string.h> +#include <xalloc.h> + +#include <openssl/ssl.h> +#include <openssl/x509v3.h> +#include <openssl/err.h> +#include <openssl/rand.h> +#include <openssl/bio.h> +#if OPENSSL_VERSION_NUMBER >= 0x00907000 +#include <openssl/conf.h> +#ifndef OPENSSL_NO_ENGINE +#include <openssl/engine.h> +#endif +#endif + +#include <sys/ioctl.h> + +#include "utils.h" +#include "connect.h" +#include "ptimer.h" +#include "url.h" +#include "ssl.h" + +#include <fcntl.h> + +#ifdef WINDOWS +# include <w32sock.h> +#endif + +/* Application-wide SSL context. This is common to all SSL + connections. */ +static SSL_CTX *ssl_ctx; + +/* Initialize the SSL's PRNG using various methods. */ + +static void +init_prng (void) +{ + char namebuf[256]; + const char *random_file; + + /* Seed from a file specified by the user. This will be the file + specified with --random-file, $RANDFILE, if set, or ~/.rnd, if it + exists. */ + if (opt.random_file) + random_file = opt.random_file; + else + { + /* Get the random file name using RAND_file_name. */ + namebuf[0] = '\0'; + random_file = RAND_file_name (namebuf, sizeof (namebuf)); + if (!file_exists_p (random_file, NULL)) + random_file = NULL; + } + + if (random_file && *random_file) + /* Seed at most 16k (apparently arbitrary value borrowed from + curl) from random file. */ + { + int _err = RAND_load_file (random_file, 16384); + if(_err == -1) + /* later the thread error queue will be cleared */ + if ( (_err = ERR_peek_last_error ()) ) + logprintf (LOG_VERBOSE, "WARNING: Could not load random file: %s, %s\n", opt.random_file, ERR_reason_error_string(_err)); + } + +#ifdef HAVE_RAND_EGD + /* Get random data from EGD if opt.egd_file was used. */ + if (opt.egd_file && *opt.egd_file) + RAND_egd (opt.egd_file); +#endif + +#ifdef WINDOWS + /* Under Windows, we can try to seed the PRNG using screen content. + This may or may not work, depending on whether we'll calling Wget + interactively. */ + + RAND_screen (); + if (RAND_status ()) + return; +#endif + +#if 0 /* don't do this by default */ + { + int maxrand = 500; + + /* Still not random enough, presumably because neither /dev/random + nor EGD were available. Try to seed OpenSSL's PRNG with libc + PRNG. This is cryptographically weak and defeats the purpose + of using OpenSSL, which is why it is highly discouraged. */ + + logprintf (LOG_NOTQUIET, _("WARNING: using a weak random seed.\n")); + + while (RAND_status () == 0 && maxrand-- > 0) + { + unsigned char rnd = random_number (256); + RAND_seed (&rnd, sizeof (rnd)); + } + } +#endif +} + +/* Print errors in the OpenSSL error stack. */ + +static void +print_errors (void) +{ + unsigned long err; + while ((err = ERR_get_error ()) != 0) + logprintf (LOG_NOTQUIET, "OpenSSL: %s\n", ERR_error_string (err, NULL)); +} + +/* Convert keyfile type as used by options.h to a type as accepted by + SSL_CTX_use_certificate_file and SSL_CTX_use_PrivateKey_file. + + (options.h intentionally doesn't use values from openssl/ssl.h so + it doesn't depend specifically on OpenSSL for SSL functionality.) */ + +static int +key_type_to_ssl_type (enum keyfile_type type) +{ + switch (type) + { + case keyfile_pem: + return SSL_FILETYPE_PEM; + case keyfile_asn1: + return SSL_FILETYPE_ASN1; + default: + abort (); + } +} + +/* SSL has been initialized */ +static int ssl_true_initialized = 0; + +/* Create an SSL Context and set default paths etc. Called the first + time an HTTP download is attempted. + + Returns true on success, false otherwise. */ + +bool +ssl_init (void) +{ + SSL_METHOD const *meth; + long ssl_options = 0; + char *ciphers_string = NULL; +#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L) + int ssl_proto_version = 0; +#endif + +#if OPENSSL_VERSION_NUMBER >= 0x00907000 + if (ssl_true_initialized == 0) + { +#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L) + OPENSSL_init_ssl (OPENSSL_INIT_LOAD_CONFIG | OPENSSL_INIT_ENGINE_ALL_BUILTIN, NULL); +#else + OPENSSL_config (NULL); +#endif + ssl_true_initialized = 1; + } +#endif + + if (ssl_ctx) + /* The SSL has already been initialized. */ + return true; + + /* Init the PRNG. If that fails, bail out. */ + init_prng (); + if (RAND_status () != 1) + { + logprintf (LOG_NOTQUIET, + _("Could not seed PRNG; consider using --random-file.\n")); + goto error; + } + +#if defined(LIBRESSL_VERSION_NUMBER) || (OPENSSL_VERSION_NUMBER < 0x10100000L) + SSL_library_init (); + SSL_load_error_strings (); + SSLeay_add_all_algorithms (); + SSLeay_add_ssl_algorithms (); +#endif + + switch (opt.secure_protocol) + { +#if !defined OPENSSL_NO_SSL2 && OPENSSL_VERSION_NUMBER < 0x10100000L + case secure_protocol_sslv2: + meth = SSLv2_client_method (); + break; +#endif + +#ifndef OPENSSL_NO_SSL3_METHOD + case secure_protocol_sslv3: + meth = SSLv3_client_method (); + break; +#endif + + case secure_protocol_auto: + case secure_protocol_pfs: + meth = SSLv23_client_method (); + ssl_options |= SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3; + break; + case secure_protocol_tlsv1: +#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L) + meth = TLS_client_method(); + ssl_proto_version = TLS1_VERSION; +#else + meth = TLSv1_client_method (); +#endif + break; + +#if OPENSSL_VERSION_NUMBER >= 0x10001000 + case secure_protocol_tlsv1_1: +#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L) + meth = TLS_client_method(); + ssl_proto_version = TLS1_1_VERSION; +#else + meth = TLSv1_1_client_method (); +#endif + break; + + case secure_protocol_tlsv1_2: +#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L) + meth = TLS_client_method(); + ssl_proto_version = TLS1_2_VERSION; +#else + meth = TLSv1_2_client_method (); +#endif + break; + + case secure_protocol_tlsv1_3: +#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L) && defined TLS1_3_VERSION + meth = TLS_client_method(); + ssl_proto_version = TLS1_3_VERSION; +#else + logprintf (LOG_NOTQUIET, _("Your OpenSSL version is too old to support TLS 1.3\n")); + goto error; +#endif + break; +#else + case secure_protocol_tlsv1_1: + logprintf (LOG_NOTQUIET, _("Your OpenSSL version is too old to support TLSv1.1\n")); + goto error; + + case secure_protocol_tlsv1_2: + logprintf (LOG_NOTQUIET, _("Your OpenSSL version is too old to support TLSv1.2\n")); + goto error; + +#endif + + default: + logprintf (LOG_NOTQUIET, _("OpenSSL: unimplemented 'secure-protocol' option value %d\n"), opt.secure_protocol); + logprintf (LOG_NOTQUIET, _("Please report this issue to bug-wget@gnu.org\n")); + abort (); + } + + /* The type cast below accommodates older OpenSSL versions (0.9.8) + where SSL_CTX_new() is declared without a "const" argument. */ + ssl_ctx = SSL_CTX_new ((SSL_METHOD *)meth); + if (!ssl_ctx) + goto error; + + if (ssl_options) + SSL_CTX_set_options (ssl_ctx, ssl_options); + +#if ((OPENSSL_VERSION_NUMBER >= 0x10101000L) && \ + !defined(LIBRESSL_VERSION_NUMBER) && \ + !defined(OPENSSL_IS_BORINGSSL)) + SSL_CTX_set_post_handshake_auth (ssl_ctx, 1); +#endif + +#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10100000L) + if (ssl_proto_version) + SSL_CTX_set_min_proto_version(ssl_ctx, ssl_proto_version); +#endif + + /* OpenSSL ciphers: https://www.openssl.org/docs/apps/ciphers.html + * + * Rules: + * 1. --ciphers overrides everything + * 2. We allow RSA key exchange by default (secure_protocol_auto) + * 3. We disallow RSA key exchange if PFS was requested (secure_protocol_pfs) + */ + if (!opt.tls_ciphers_string) + { + if (opt.secure_protocol == secure_protocol_auto) + ciphers_string = "HIGH:!aNULL:!RC4:!MD5:!SRP:!PSK"; + else if (opt.secure_protocol == secure_protocol_pfs) + ciphers_string = "HIGH:!aNULL:!RC4:!MD5:!SRP:!PSK:!kRSA"; + } + else + { + ciphers_string = opt.tls_ciphers_string; + } + + if (ciphers_string && !SSL_CTX_set_cipher_list(ssl_ctx, ciphers_string)) + { + logprintf(LOG_NOTQUIET, _("OpenSSL: Invalid cipher list: %s\n"), ciphers_string); + goto error; + } + + SSL_CTX_set_default_verify_paths (ssl_ctx); + SSL_CTX_load_verify_locations (ssl_ctx, opt.ca_cert, opt.ca_directory); + +#ifdef X509_V_FLAG_PARTIAL_CHAIN + /* Set X509_V_FLAG_PARTIAL_CHAIN to allow the client to anchor trust in + * a non-self-signed certificate. This defies RFC 4158 (Path Building) + * which defines a trust anchor in terms of a self-signed certificate. + * However, it substantially reduces attack surface by pruning the tree + * of unneeded trust points. For example, the cross-certified + * Let's Encrypt X3 CA, which protects gnu.org and appears as an + * intermediate CA to clients, can be used as a trust anchor without + * the entire IdentTrust PKI. + */ + X509_VERIFY_PARAM *param = X509_VERIFY_PARAM_new(); + if (param) + { + /* We only want X509_V_FLAG_PARTIAL_CHAIN, but the OpenSSL docs + * say to use X509_V_FLAG_TRUSTED_FIRST also. It looks like + * X509_V_FLAG_TRUSTED_FIRST applies to a collection of trust + * anchors and not a single trust anchor. + */ + (void) X509_VERIFY_PARAM_set_flags (param, X509_V_FLAG_TRUSTED_FIRST | X509_V_FLAG_PARTIAL_CHAIN); + if (SSL_CTX_set1_param (ssl_ctx, param) == 0) + logprintf(LOG_NOTQUIET, _("OpenSSL: Failed set trust to partial chain\n")); + /* We continue on error */ + X509_VERIFY_PARAM_free (param); + } + else + { + logprintf(LOG_NOTQUIET, _("OpenSSL: Failed to allocate verification param\n")); + /* We continue on error */ + } +#endif + + if (opt.crl_file) + { + X509_STORE *store = SSL_CTX_get_cert_store (ssl_ctx); + X509_LOOKUP *lookup; + + if (!(lookup = X509_STORE_add_lookup (store, X509_LOOKUP_file ())) + || (!X509_load_crl_file (lookup, opt.crl_file, X509_FILETYPE_PEM))) + goto error; + + X509_STORE_set_flags (store, X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL); + } + + /* SSL_VERIFY_NONE instructs OpenSSL not to abort SSL_connect if the + certificate is invalid. We verify the certificate separately in + ssl_check_certificate, which provides much better diagnostics + than examining the error stack after a failed SSL_connect. */ + SSL_CTX_set_verify (ssl_ctx, SSL_VERIFY_NONE, NULL); + + /* Use the private key from the cert file unless otherwise specified. */ + if (opt.cert_file && !opt.private_key) + { + opt.private_key = xstrdup (opt.cert_file); + opt.private_key_type = opt.cert_type; + } + + /* Use cert from private key file unless otherwise specified. */ + if (opt.private_key && !opt.cert_file) + { + opt.cert_file = xstrdup (opt.private_key); + opt.cert_type = opt.private_key_type; + } + + if (opt.cert_file) + if (SSL_CTX_use_certificate_file (ssl_ctx, opt.cert_file, + key_type_to_ssl_type (opt.cert_type)) + != 1) + goto error; + if (opt.private_key) + if (SSL_CTX_use_PrivateKey_file (ssl_ctx, opt.private_key, + key_type_to_ssl_type (opt.private_key_type)) + != 1) + goto error; + + /* Since fd_write unconditionally assumes partial writes (and + handles them correctly), allow them in OpenSSL. */ + SSL_CTX_set_mode (ssl_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE); + + /* The OpenSSL library can handle renegotiations automatically, so + tell it to do so. */ + SSL_CTX_set_mode (ssl_ctx, SSL_MODE_AUTO_RETRY); + + return true; + + error: + if (ssl_ctx) + SSL_CTX_free (ssl_ctx); + print_errors (); + return false; +} + +void +ssl_cleanup (void) +{ +} + +struct openssl_transport_context +{ + SSL *conn; /* SSL connection handle */ + SSL_SESSION *sess; /* SSL session info */ + char *last_error; /* last error printed with openssl_errstr */ +}; + +typedef int (*ssl_fn_t)(SSL *, void *, int); + +#ifdef OPENSSL_RUN_WITHTIMEOUT + +struct scwt_context +{ + SSL *ssl; + int result; +}; + +static void +ssl_connect_with_timeout_callback(void *arg) +{ + struct scwt_context *ctx = (struct scwt_context *)arg; + ctx->result = SSL_connect(ctx->ssl); +} + +static int +ssl_connect_with_timeout(int fd _GL_UNUSED, SSL *conn, double timeout) +{ + struct scwt_context scwt_ctx; + scwt_ctx.ssl = conn; + errno = 0; + if (run_with_timeout(timeout, ssl_connect_with_timeout_callback, + &scwt_ctx)) + { + errno = ETIMEDOUT; + return -1; + } + return scwt_ctx.result; +} + +struct openssl_read_args +{ + int fd; + struct openssl_transport_context *ctx; + ssl_fn_t fn; + char *buf; + int bufsize; + int retval; +}; + +static void +openssl_read_peek_callback(void *arg) +{ + struct openssl_read_args *args = (struct openssl_read_args *) arg; + struct openssl_transport_context *ctx = args->ctx; + ssl_fn_t fn = args->fn; + SSL *conn = ctx->conn; + char *buf = args->buf; + int bufsize = args->bufsize; + int ret; + + do + { + ret = fn (conn, buf, bufsize); + } + while (ret == -1 && SSL_get_error (conn, ret) == SSL_ERROR_SYSCALL && errno == EINTR); + args->retval = ret; +} + +static int +openssl_read_peek (int fd, char *buf, int bufsize, void *arg, double timeout, ssl_fn_t fn) +{ + struct openssl_transport_context *ctx = arg; + int ret = SSL_pending (ctx->conn); + + if (ret) + ret = fn (ctx->conn, buf, MIN (bufsize, ret)); + else + { + struct openssl_read_args args; + args.fd = fd; + args.buf = buf; + args.bufsize = bufsize; + args.fn = fn; + args.ctx = ctx; + + if (timeout == -1) + timeout = opt.read_timeout; + + if (run_with_timeout(timeout, openssl_read_peek_callback, &args)) + { + errno = ETIMEDOUT; + ret = -1; + } + else + ret = args.retval; + } + return ret; +} + +#else /* OPENSSL_RUN_WITHTIMEOUT */ + +#ifdef F_GETFL +#define NONBLOCK_DECL int flags = 0; +#define FD_SET_NONBLOCKED(_fd) \ + flags = fcntl (_fd, F_GETFL, 0); \ + if (flags < 0) \ + return flags; \ + if (fcntl (_fd, F_SETFL, flags | O_NONBLOCK)) \ + return -1; +#define FD_SET_BLOCKED(_fd) \ + if (fcntl (_fd, F_SETFL, flags) < 0) \ + return -1; +#else +#define NONBLOCK_DECL +#define FD_SET_NONBLOCKED(_fd) \ + {\ + const int one = 1;\ + if (ioctl (_fd, FIONBIO, &one) < 0)\ + return -1;\ + } +#define FD_SET_BLOCKED(_fd) \ + {\ + const int zero = 0;\ + if (ioctl (_fd, FIONBIO, &zero) < 0)\ + return -1;\ + } +#endif /* F_GETFL */ + +#define TIMER_INIT(_fd, _ret, _timeout) \ + { \ + NONBLOCK_DECL \ + int timed_out = 0; \ + FD_SET_NONBLOCKED(_fd) \ + struct ptimer *timer = ptimer_new (); \ + if (timer == NULL) \ + _ret = -1; \ + else \ + { \ + double next_timeout = _timeout; + +#define TIMER_FREE(_fd) \ + ptimer_destroy (timer); \ + } \ + FD_SET_BLOCKED(_fd) \ + if (timed_out) \ + { \ + errno = ETIMEDOUT; \ + } \ + } + +#define TIMER_WAIT(_fd, _conn, _ret, _timeout) \ + { \ + int wait_for; \ + int err = SSL_get_error(_conn, _ret); \ + if (err == SSL_ERROR_WANT_READ) \ + wait_for = WAIT_FOR_READ; \ + else if (err == SSL_ERROR_WANT_WRITE) \ + wait_for = WAIT_FOR_WRITE; \ + else \ + break; \ + err = select_fd_nb (_fd, next_timeout, wait_for); \ + if (err <= 0) \ + { \ + if (err == 0) \ +timedout: \ + timed_out = 1; \ + _ret = -1; \ + break; \ + } \ + next_timeout = _timeout - ptimer_measure (timer); \ + if (next_timeout <= 0) \ + goto timedout; \ + } + +static int +ssl_connect_with_timeout(int fd, SSL *conn, double timeout) +{ + int ret; + + errno = 0; + if (timeout == 0) + ret = SSL_connect(conn); + else + { + TIMER_INIT(fd, ret, timeout) + ERR_clear_error(); + while( (ret = SSL_connect(conn)) < 0 ) + TIMER_WAIT(fd, conn, ret, timeout) + TIMER_FREE(fd) + } + + return ret; +} + +static int +openssl_read_peek (int fd, char *buf, int bufsize, void *arg, double timeout, ssl_fn_t fn) +{ + struct openssl_transport_context *ctx = arg; + int ret = SSL_pending (ctx->conn); + + if (timeout == -1) + timeout = opt.read_timeout; + /* If we have data available for immediate read, simply return that, + or do blocked read when timeout == 0 */ + if (ret || timeout == 0) + do + { + ret = fn (ctx->conn, buf, (ret ? MIN (bufsize, ret) : bufsize)); + } + while (ret == -1 && SSL_get_error (ctx->conn, ret) == SSL_ERROR_SYSCALL && errno == EINTR); + else + { + TIMER_INIT(fd, ret, timeout) + while( (ret = fn (ctx->conn, buf, bufsize)) <= 0 ) + TIMER_WAIT(fd, ctx->conn, ret, timeout) + TIMER_FREE(fd) + } + + return ret; +} + +#endif /* OPENSSL_RUN_WITHTIMEOUT */ + +static int +openssl_read (int fd, char *buf, int bufsize, void *arg, double timeout) +{ + return openssl_read_peek (fd, buf, bufsize, arg, timeout, SSL_read); +} + +static int +openssl_write (int fd _GL_UNUSED, char *buf, int bufsize, void *arg) +{ + int ret = 0; + struct openssl_transport_context *ctx = arg; + SSL *conn = ctx->conn; + do + ret = SSL_write (conn, buf, bufsize); + while (ret == -1 && SSL_get_error (conn, ret) == SSL_ERROR_SYSCALL && errno == EINTR); + return ret; +} + +static int +openssl_poll (int fd, double timeout, int wait_for, void *arg) +{ + struct openssl_transport_context *ctx = arg; + SSL *conn = ctx->conn; + if ((wait_for & WAIT_FOR_READ) && SSL_pending (conn)) + return 1; + /* if (timeout == 0) + return 1; */ + if (timeout == -1) + timeout = opt.read_timeout; + return select_fd (fd, timeout, wait_for); +} + +static int +openssl_peek (int fd, char *buf, int bufsize, void *arg, double timeout) +{ + return openssl_read_peek (fd, buf, bufsize, arg, timeout, SSL_peek); +} + +static const char * +openssl_errstr (int fd _GL_UNUSED, void *arg) +{ + struct openssl_transport_context *ctx = arg; + unsigned long errcode; + char *errmsg = NULL; + int msglen = 0; + + /* If there are no SSL-specific errors, just return NULL. */ + if ((errcode = ERR_get_error ()) == 0) + return NULL; + + /* Get rid of previous contents of ctx->last_error, if any. */ + xfree (ctx->last_error); + + /* Iterate over OpenSSL's error stack and accumulate errors in the + last_error buffer, separated by "; ". This is better than using + a static buffer, which *always* takes up space (and has to be + large, to fit more than one error message), whereas these + allocations are only performed when there is an actual error. */ + + for (;;) + { + const char *str = ERR_error_string (errcode, NULL); + int len = strlen (str); + + /* Allocate space for the existing message, plus two more chars + for the "; " separator and one for the terminating \0. */ + errmsg = xrealloc (errmsg, msglen + len + 2 + 1); + memcpy (errmsg + msglen, str, len); + msglen += len; + + /* Get next error and bail out if there are no more. */ + errcode = ERR_get_error (); + if (errcode == 0) + break; + + errmsg[msglen++] = ';'; + errmsg[msglen++] = ' '; + } + errmsg[msglen] = '\0'; + + /* Store the error in ctx->last_error where openssl_close will + eventually find it and free it. */ + ctx->last_error = errmsg; + + return errmsg; +} + +static void +openssl_close (int fd, void *arg) +{ + struct openssl_transport_context *ctx = arg; + SSL *conn = ctx->conn; + + SSL_shutdown (conn); + SSL_free (conn); + xfree (ctx->last_error); + xfree (ctx); + + close (fd); + + DEBUGP (("Closed %d/SSL 0x%0*lx\n", fd, PTR_FORMAT (conn))); +} + +/* openssl_transport is the singleton that describes the SSL transport + methods provided by this file. */ + +static struct transport_implementation openssl_transport = { + openssl_read, openssl_write, openssl_poll, + openssl_peek, openssl_errstr, openssl_close +}; + +static const char * +_sni_hostname(const char *hostname) +{ + size_t len = strlen(hostname); + + char *sni_hostname = xmemdup(hostname, len + 1); + + /* Remove trailing dot(s) to fix #47408. + * Regarding RFC 6066 (SNI): The hostname is represented as a byte + * string using ASCII encoding without a trailing dot. */ + while (len && sni_hostname[--len] == '.') + sni_hostname[len] = 0; + + return sni_hostname; +} + +/* Perform the SSL handshake on file descriptor FD, which is assumed + to be connected to an SSL server. The SSL handle provided by + OpenSSL is registered with the file descriptor FD using + fd_register_transport, so that subsequent calls to fd_read, + fd_write, etc., will use the corresponding SSL functions. + + Returns true on success, false on failure. */ + +bool +ssl_connect_wget (int fd, const char *hostname, int *continue_session) +{ + SSL *conn; + struct openssl_transport_context *ctx; + + DEBUGP (("Initiating SSL handshake.\n")); + + assert (ssl_ctx != NULL); + conn = SSL_new (ssl_ctx); + if (!conn) + goto error; +#if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT) + /* If the SSL library was built with support for ServerNameIndication + then use it whenever we have a hostname. If not, don't, ever. */ + if (! is_valid_ip_address (hostname)) + { + const char *sni_hostname = _sni_hostname(hostname); + + long rc = SSL_set_tlsext_host_name (conn, sni_hostname); + xfree(sni_hostname); + + if (rc == 0) + { + DEBUGP (("Failed to set TLS server-name indication.")); + goto error; + } + } +#endif + + if (continue_session) + { + /* attempt to resume a previous SSL session */ + ctx = (struct openssl_transport_context *) fd_transport_context (*continue_session); + if (!ctx || !ctx->sess || !SSL_set_session (conn, ctx->sess)) + goto error; + } + +#ifndef FD_TO_SOCKET +# define FD_TO_SOCKET(X) (X) +#endif + if (!SSL_set_fd (conn, FD_TO_SOCKET (fd))) + goto error; + SSL_set_connect_state (conn); + + /* Re-seed the PRNG before the SSL handshake */ + init_prng (); + if (RAND_status () != 1) + { + logprintf(LOG_NOTQUIET, + _("WARNING: Could not seed PRNG. Consider using --random-file.\n")); + goto error; + } + + if (ssl_connect_with_timeout(fd, conn, opt.read_timeout) <= 0 + || !SSL_is_init_finished(conn)) + goto timedout; + + ctx = xnew0 (struct openssl_transport_context); + ctx->conn = conn; + ctx->sess = SSL_get0_session (conn); + if (!ctx->sess) + logprintf (LOG_NOTQUIET, "WARNING: Could not save SSL session data for socket %d\n", fd); + + /* Register FD with Wget's transport layer, i.e. arrange that our + functions are used for reading, writing, and polling. */ + fd_register_transport (fd, &openssl_transport, ctx); + DEBUGP (("Handshake successful; connected socket %d to SSL handle 0x%0*lx\n", + fd, PTR_FORMAT (conn))); + + ERR_clear_error (); + return true; + + timedout: + if (errno == ETIMEDOUT) + DEBUGP (("SSL handshake timed out.\n")); + else + error: + DEBUGP (("SSL handshake failed.\n")); + print_errors (); + if (conn) + SSL_free (conn); + return false; +} + +#define ASTERISK_EXCLUDES_DOT /* mandated by rfc2818 */ + +/* Return true is STRING (case-insensitively) matches PATTERN, false + otherwise. The recognized wildcard character is "*", which matches + any character in STRING except ".". Any number of the "*" wildcard + may be present in the pattern. + + This is used to match of hosts as indicated in rfc2818: "Names may + contain the wildcard character * which is considered to match any + single domain name component or component fragment. E.g., *.a.com + matches foo.a.com but not bar.foo.a.com. f*.com matches foo.com but + not bar.com [or foo.bar.com]." + + If the pattern contain no wildcards, pattern_match(a, b) is + equivalent to !strcasecmp(a, b). */ + +static bool +pattern_match (const char *pattern, const char *string) +{ + const char *p = pattern, *n = string; + char c; + for (; (c = c_tolower (*p++)) != '\0'; n++) + if (c == '*') + { + for (c = c_tolower (*p); c == '*'; c = c_tolower (*++p)) + ; + for (; *n != '\0'; n++) + if (c_tolower (*n) == c && pattern_match (p, n)) + return true; +#ifdef ASTERISK_EXCLUDES_DOT + else if (*n == '.') + return false; +#endif + return c == '\0'; + } + else + { + if (c != c_tolower (*n)) + return false; + } + return *n == '\0'; +} + +static char *_get_rfc2253_formatted (X509_NAME *name) +{ + int len; + char *out = NULL; + BIO* b; + + if ((b = BIO_new (BIO_s_mem ()))) + { + if (X509_NAME_print_ex (b, name, 0, XN_FLAG_RFC2253) >= 0 + && (len = BIO_number_written (b)) > 0) + { + out = xmalloc (len + 1); + BIO_read (b, out, len); + out[len] = 0; + } + BIO_free (b); + } + + return out ? out : xstrdup(""); +} + +/* + * Heavily modified from: + * https://www.owasp.org/index.php/Certificate_and_Public_Key_Pinning#OpenSSL + */ +static bool +pkp_pin_peer_pubkey (X509* cert, const char *pinnedpubkey) +{ + /* Scratch */ + int len1 = 0, len2 = 0; + char *buff1 = NULL, *temp = NULL; + + /* Result is returned to caller */ + bool result = false; + + /* if a path wasn't specified, don't pin */ + if (!pinnedpubkey) + return true; + + if (!cert) + return result; + + /* Begin Gyrations to get the subjectPublicKeyInfo */ + /* Thanks to Viktor Dukhovni on the OpenSSL mailing list */ + + /* https://groups.google.com/group/mailing.openssl.users/browse_thread + /thread/d61858dae102c6c7 */ + len1 = i2d_X509_PUBKEY (X509_get_X509_PUBKEY (cert), NULL); + if (len1 < 1) + goto cleanup; /* failed */ + + /* https://www.openssl.org/docs/crypto/buffer.html */ + buff1 = temp = OPENSSL_malloc (len1); + if (!buff1) + goto cleanup; /* failed */ + + /* https://www.openssl.org/docs/crypto/d2i_X509.html */ + len2 = i2d_X509_PUBKEY (X509_get_X509_PUBKEY (cert), (unsigned char **) &temp); + + /* + * These checks are verifying we got back the same values as when we + * sized the buffer. It's pretty weak since they should always be the + * same. But it gives us something to test. + */ + if ((len1 != len2) || !temp || ((temp - buff1) != len1)) + goto cleanup; /* failed */ + + /* End Gyrations */ + + /* The one good exit point */ + result = wg_pin_peer_pubkey (pinnedpubkey, buff1, len1); + + cleanup: + /* https://www.openssl.org/docs/crypto/buffer.html */ + if (NULL != buff1) + OPENSSL_free (buff1); + + return result; +} + +/* Verify the validity of the certificate presented by the server. + Also check that the "common name" of the server, as presented by + its certificate, corresponds to HOST. (HOST typically comes from + the URL and is what the user thinks he's connecting to.) + + This assumes that ssl_connect_wget has successfully finished, i.e. that + the SSL handshake has been performed and that FD is connected to an + SSL handle. + + If opt.check_cert is true (the default), this returns 1 if the + certificate is valid, 0 otherwise. If opt.check_cert is 0, the + function always returns 1, but should still be called because it + warns the user about any problems with the certificate. */ + +bool +ssl_check_certificate (int fd, const char *host) +{ + X509 *cert; + GENERAL_NAMES *subjectAltNames; + char common_name[256]; + long vresult; + bool success = true; + bool alt_name_checked = false; + bool pinsuccess = opt.pinnedpubkey == NULL; + + /* If the user has specified --no-check-cert, we still want to warn + him about problems with the server's certificate. */ + const char *severity = opt.check_cert ? _("ERROR") : _("WARNING"); + + struct openssl_transport_context *ctx = fd_transport_context (fd); + SSL *conn = ctx->conn; + assert (conn != NULL); + + /* The user explicitly said to not check for the certificate. */ + if (opt.check_cert == CHECK_CERT_QUIET && pinsuccess) + return success; + + cert = SSL_get_peer_certificate (conn); + if (!cert) + { + logprintf (LOG_NOTQUIET, _("%s: No certificate presented by %s.\n"), + severity, quotearg_style (escape_quoting_style, host)); + success = false; + goto no_cert; /* must bail out since CERT is NULL */ + } + + IF_DEBUG + { + char *subject = _get_rfc2253_formatted (X509_get_subject_name (cert)); + char *issuer = _get_rfc2253_formatted (X509_get_issuer_name (cert)); + DEBUGP (("certificate:\n subject: %s\n issuer: %s\n", + quotearg_n_style (0, escape_quoting_style, subject), + quotearg_n_style (1, escape_quoting_style, issuer))); + xfree (subject); + xfree (issuer); + } + + vresult = SSL_get_verify_result (conn); + if (vresult != X509_V_OK) + { + char *issuer = _get_rfc2253_formatted (X509_get_issuer_name (cert)); + logprintf (LOG_NOTQUIET, + _("%s: cannot verify %s's certificate, issued by %s:\n"), + severity, quotearg_n_style (0, escape_quoting_style, host), + quote_n (1, issuer)); + xfree(issuer); + + /* Try to print more user-friendly (and translated) messages for + the frequent verification errors. */ + switch (vresult) + { + case X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY: + logprintf (LOG_NOTQUIET, + _(" Unable to locally verify the issuer's authority.\n")); + break; + case X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN: + case X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT: + logprintf (LOG_NOTQUIET, + _(" Self-signed certificate encountered.\n")); + break; + case X509_V_ERR_CERT_NOT_YET_VALID: + logprintf (LOG_NOTQUIET, _(" Issued certificate not yet valid.\n")); + break; + case X509_V_ERR_CERT_HAS_EXPIRED: + logprintf (LOG_NOTQUIET, _(" Issued certificate has expired.\n")); + break; + default: + /* For the less frequent error strings, simply provide the + OpenSSL error message. */ + logprintf (LOG_NOTQUIET, " %s\n", + X509_verify_cert_error_string (vresult)); + } + success = false; + /* Fall through, so that the user is warned about *all* issues + with the cert (important with --no-check-certificate.) */ + } + + /* Check that HOST matches the common name in the certificate. + #### The following remains to be done: + + - When matching against common names, it should loop over all + common names and choose the most specific one, i.e. the last + one, not the first one, which the current code picks. + + - Ensure that ASN1 strings from the certificate are encoded as + UTF-8 which can be meaningfully compared to HOST. */ + + subjectAltNames = X509_get_ext_d2i (cert, NID_subject_alt_name, NULL, NULL); + + if (subjectAltNames) + { + /* Test subject alternative names */ + + /* SNI hostname must not have a trailing dot */ + const char *sni_hostname = _sni_hostname(host); + + /* Do we want to check for dNSNAmes or ipAddresses (see RFC 2818)? + * Signal it by host_in_octet_string. */ + ASN1_OCTET_STRING *host_in_octet_string = a2i_IPADDRESS (sni_hostname); + + int numaltnames = sk_GENERAL_NAME_num (subjectAltNames); + int i; + for (i=0; i < numaltnames; i++) + { + const GENERAL_NAME *name = + sk_GENERAL_NAME_value (subjectAltNames, i); + if (name) + { + if (host_in_octet_string) + { + if (name->type == GEN_IPADD) + { + /* Check for ipAddress */ + /* TODO: Should we convert between IPv4-mapped IPv6 + * addresses and IPv4 addresses? */ + alt_name_checked = true; + if (!ASN1_STRING_cmp (host_in_octet_string, + name->d.iPAddress)) + break; + } + } + else if (name->type == GEN_DNS) + { + /* dNSName should be IA5String (i.e. ASCII), however who + * does trust CA? Convert it into UTF-8 for sure. */ + unsigned char *name_in_utf8 = NULL; + + /* Check for dNSName */ + alt_name_checked = true; + + if (0 <= ASN1_STRING_to_UTF8 (&name_in_utf8, name->d.dNSName)) + { + /* Compare and check for NULL attack in ASN1_STRING */ + if (pattern_match ((char *)name_in_utf8, sni_hostname) && + (strlen ((char *)name_in_utf8) == + (size_t) ASN1_STRING_length (name->d.dNSName))) + { + OPENSSL_free (name_in_utf8); + break; + } + OPENSSL_free (name_in_utf8); + } + } + } + } + sk_GENERAL_NAME_pop_free(subjectAltNames, GENERAL_NAME_free); + if (host_in_octet_string) + ASN1_OCTET_STRING_free(host_in_octet_string); + + if (alt_name_checked == true && i >= numaltnames) + { + logprintf (LOG_NOTQUIET, + _("%s: no certificate subject alternative name matches\n" + "\trequested host name %s.\n"), + severity, quote_n (1, sni_hostname)); + success = false; + } + + xfree(sni_hostname); + } + + if (alt_name_checked == false) + { + /* Test commomName */ + X509_NAME *xname = X509_get_subject_name(cert); + common_name[0] = '\0'; + X509_NAME_get_text_by_NID (xname, NID_commonName, common_name, + sizeof (common_name)); + + if (!pattern_match (common_name, host)) + { + logprintf (LOG_NOTQUIET, _("\ + %s: certificate common name %s doesn't match requested host name %s.\n"), + severity, quote_n (0, common_name), quote_n (1, host)); + success = false; + } + else + { + /* We now determine the length of the ASN1 string. If it + * differs from common_name's length, then there is a \0 + * before the string terminates. This can be an instance of a + * null-prefix attack. + * + * https://www.blackhat.com/html/bh-usa-09/bh-usa-09-archives.html#Marlinspike + * */ + + int i = -1, j; + X509_NAME_ENTRY *xentry; + ASN1_STRING *sdata; + + if (xname) { + for (;;) + { + j = X509_NAME_get_index_by_NID (xname, NID_commonName, i); + if (j == -1) break; + i = j; + } + } + + xentry = X509_NAME_get_entry(xname,i); + sdata = X509_NAME_ENTRY_get_data(xentry); + if (strlen (common_name) != (size_t) ASN1_STRING_length (sdata)) + { + logprintf (LOG_NOTQUIET, _("\ + %s: certificate common name is invalid (contains a NUL character).\n\ + This may be an indication that the host is not who it claims to be\n\ + (that is, it is not the real %s).\n"), + severity, quote (host)); + success = false; + } + } + } + + pinsuccess = pkp_pin_peer_pubkey (cert, opt.pinnedpubkey); + if (!pinsuccess) + { + logprintf (LOG_ALWAYS, _("The public key does not match pinned public key!\n")); + success = false; + } + + + if (success) + DEBUGP (("X509 certificate successfully verified and matches host %s\n", + quotearg_style (escape_quoting_style, host))); + X509_free (cert); + + no_cert: + if (opt.check_cert == CHECK_CERT_ON && !success) + logprintf (LOG_NOTQUIET, _("\ +To connect to %s insecurely, use `--no-check-certificate'.\n"), + quotearg_style (escape_quoting_style, host)); + + /* never return true if pinsuccess fails */ + return !pinsuccess ? false : (opt.check_cert == CHECK_CERT_ON ? success : true); +} + +/* + * vim: tabstop=2 shiftwidth=2 softtabstop=2 + */ diff --git a/src/options.h b/src/options.h new file mode 100644 index 0000000..9a02f3a --- /dev/null +++ b/src/options.h @@ -0,0 +1,351 @@ +/* struct options. + Copyright (C) 1996-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +enum CHECK_CERT_MODES +{ + CHECK_CERT_OFF, + CHECK_CERT_ON, + CHECK_CERT_QUIET +}; + +struct options +{ + int verbose; /* Are we verbose? (First set to -1, + hence not boolean.) */ + bool quiet; /* Are we quiet? */ + int ntry; /* Number of tries per URL */ + bool retry_connrefused; /* Treat CONNREFUSED as non-fatal. */ + bool retry_on_host_error; /* Treat host errors as non-fatal. */ + char *retry_on_http_error; /* Treat given HTTP errors as non-fatal. */ + bool background; /* Whether we should work in background. */ + bool ignore_length; /* Do we heed content-length at all? */ + bool recursive; /* Are we recursive? */ + bool spanhost; /* Do we span across hosts in + recursion? */ + int max_redirect; /* Maximum number of times we'll allow + a page to redirect. */ + bool relative_only; /* Follow only relative links. */ + bool no_parent; /* Restrict access to the parent + directory. */ + int reclevel; /* Maximum level of recursion */ + bool dirstruct; /* Do we build the directory structure + as we go along? */ + bool no_dirstruct; /* Do we hate dirstruct? */ + int cut_dirs; /* Number of directory components to cut. */ + bool add_hostdir; /* Do we add hostname directory? */ + bool protocol_directories; /* Whether to prepend "http"/"ftp" to dirs. */ + bool noclobber; /* Disables clobbering of existing data. */ + bool unlink_requested; /* remove file before clobbering */ + char *dir_prefix; /* The top of directory tree */ + char *lfilename; /* Log filename */ + char *input_filename; /* Input filename */ +#ifdef HAVE_METALINK + char *input_metalink; /* Input metalink file */ + int metalink_index; /* Metalink application/metalink4+xml metaurl ordinal number. */ + bool metalink_over_http; /* Use Metalink if present in HTTP response */ + char *preferred_location; /* Preferred location for Metalink resources */ +#endif + char *choose_config; /* Specified config file */ + bool noconfig; /* Ignore all config files? */ + bool force_html; /* Is the input file an HTML file? */ + + char *default_page; /* Alternative default page (index file) */ + + bool spider; /* Is Wget in spider mode? */ + + char **accepts; /* List of patterns to accept. */ + char **rejects; /* List of patterns to reject. */ + const char **excludes; /* List of excluded FTP directories. */ + const char **includes; /* List of FTP directories to + follow. */ + bool ignore_case; /* Whether to ignore case when + matching dirs and files */ + + char *acceptregex_s; /* Patterns to accept (a regex string). */ + char *rejectregex_s; /* Patterns to reject (a regex string). */ + void *acceptregex; /* Patterns to accept (a regex struct). */ + void *rejectregex; /* Patterns to reject (a regex struct). */ + enum { +#if defined HAVE_LIBPCRE || HAVE_LIBPCRE2 + regex_type_pcre, +#endif + regex_type_posix + } regex_type; /* The regex library. */ + void *(*regex_compile_fun)(const char *); /* Function to compile a regex. */ + bool (*regex_match_fun)(const void *, const char *); /* Function to match a string to a regex. */ + +#ifdef HAVE_LIBCARES + char *bind_dns_address; + char *dns_servers; +#endif + + char **domains; /* See host.c */ + char **exclude_domains; + bool dns_cache; /* whether we cache DNS lookups. */ + + char **follow_tags; /* List of HTML tags to recursively follow. */ + char **ignore_tags; /* List of HTML tags to ignore if recursing. */ + + bool follow_ftp; /* Are FTP URL-s followed in recursive + retrieving? */ + bool retr_symlinks; /* Whether we retrieve symlinks in + FTP. */ + char *output_document; /* The output file to which the + documents will be printed. */ + char *warc_filename; /* WARC output filename */ + char *warc_tempdir; /* WARC temp dir */ + char *warc_cdx_dedup_filename;/* CDX file to be used for deduplication. */ + wgint warc_maxsize; /* WARC max archive size */ + bool warc_compression_enabled;/* For GZIP compression. */ + bool warc_digests_enabled; /* For SHA1 digests. */ + bool warc_cdx_enabled; /* Create CDX files? */ + bool warc_keep_log; /* Store the log file in a WARC record. */ + char **warc_user_headers; /* User-defined WARC header(s). */ + + bool enable_xattr; /* Store metadata in POSIX extended attributes. */ + + char *user; /* Generic username */ + char *passwd; /* Generic password */ + bool ask_passwd; /* Ask for password? */ + char *use_askpass; /* value to use for use-askpass if WGET_ASKPASS is not set */ + + bool always_rest; /* Always use REST. */ + wgint start_pos; /* Start position of a download. */ + char *ftp_user; /* FTP username */ + char *ftp_passwd; /* FTP password */ + bool netrc; /* Whether to read .netrc. */ + bool ftp_glob; /* FTP globbing */ + bool ftp_pasv; /* Passive FTP. */ + + char *http_user; /* HTTP username. */ + char *http_passwd; /* HTTP password. */ + char **user_headers; /* User-defined header(s). */ + bool http_keep_alive; /* whether we use keep-alive */ + + bool use_proxy; /* Do we use proxy? */ + bool allow_cache; /* Do we allow server-side caching? */ + char *http_proxy, *ftp_proxy, *https_proxy; + char **no_proxy; + char *base_href; + char *progress_type; /* progress indicator type. */ + int show_progress; /* Show only the progress bar */ + bool noscroll; /* Don't scroll the filename in the progressbar */ + char *proxy_user; /*oli*/ + char *proxy_passwd; + + double read_timeout; /* The read/write timeout. */ + double dns_timeout; /* The DNS timeout. */ + double connect_timeout; /* The connect timeout. */ + + bool random_wait; /* vary from 0 .. wait secs by random()? */ + double wait; /* The wait period between retrievals. */ + double waitretry; /* The wait period between retries. - HEH */ + bool use_robots; /* Do we heed robots.txt? */ + + wgint limit_rate; /* Limit the download rate to this + many bps. */ + SUM_SIZE_INT quota; /* Maximum file size to download and + store. */ + + bool server_response; /* Do we print server response? */ + bool save_headers; /* Do we save headers together with + file? */ + bool content_on_error; /* Do we output the content when the HTTP + status code indicates a server error */ + + bool debug; /* Debugging on/off */ + +#ifdef USE_WATT32 + bool wdebug; /* Watt-32 tcp/ip debugging on/off */ +#endif + + bool timestamping; /* Whether to use time-stamping. */ + bool if_modified_since; /* Whether to use conditional get requests. */ + + bool backup_converted; /* Do we save pre-converted files as *.orig? */ + int backups; /* Are numeric backups made? */ + + char *useragent; /* User-Agent string, which can be set + to something other than Wget. */ + char *referer; /* Naughty Referer, which can be + set to something other than + NULL. */ + bool convert_links; /* Will the links be converted + locally? */ + bool convert_file_only; /* Convert only the file portion of the URI (i.e. basename). + Leave everything else untouched. */ + + bool remove_listing; /* Do we remove .listing files + generated by FTP? */ + bool htmlify; /* Do we HTML-ify the OS-dependent + listings? */ + + char *dot_style; + wgint dot_bytes; /* How many bytes in a printing + dot. */ + int dots_in_line; /* How many dots in one line. */ + int dot_spacing; /* How many dots between spacings. */ + + bool delete_after; /* Whether the files will be deleted + after download. */ + + bool adjust_extension; /* Use ".html" extension on all text/html? */ + + bool page_requisites; /* Whether we need to download all files + necessary to display a page properly. */ + char *bind_address; /* What local IP address to bind to. */ + +#ifdef HAVE_SSL + enum { + secure_protocol_auto, + secure_protocol_sslv2, + secure_protocol_sslv3, + secure_protocol_tlsv1, + secure_protocol_tlsv1_1, + secure_protocol_tlsv1_2, + secure_protocol_tlsv1_3, + secure_protocol_pfs + } secure_protocol; /* type of secure protocol to use. */ + int check_cert; /* whether to validate the server's cert */ + char *cert_file; /* external client certificate to use. */ + char *private_key; /* private key file (if not internal). */ + enum keyfile_type { + keyfile_pem, + keyfile_asn1 + } cert_type; /* type of client certificate file */ + enum keyfile_type + private_key_type; /* type of private key file */ + + char *ca_directory; /* CA directory (hash files) */ + char *ca_cert; /* CA certificate file to use */ + char *crl_file; /* file with CRLs */ + + char *pinnedpubkey; /* Public key (PEM/DER) file, or any number + of base64 encoded sha256 hashes preceded by + \'sha256//\' and separated by \';\', to verify + peer against */ + + char *random_file; /* file with random data to seed the PRNG */ + char *egd_file; /* file name of the egd daemon socket */ + bool https_only; /* whether to follow HTTPS only */ + bool ftps_resume_ssl; + bool ftps_fallback_to_ftp; + bool ftps_implicit; + bool ftps_clear_data_connection; + + char *tls_ciphers_string; +#endif /* HAVE_SSL */ + + bool cookies; /* whether cookies are used. */ + char *cookies_input; /* file we're loading the cookies from. */ + char *cookies_output; /* file we're saving the cookies to. */ + bool keep_badhash; /* Keep files with checksum mismatch. */ + bool keep_session_cookies; /* whether session cookies should be + saved and loaded. */ + + char *post_data; /* POST query string */ + char *post_file_name; /* File to post */ + char *method; /* HTTP Method to use in Header */ + char *body_data; /* HTTP Method Data String */ + char *body_file; /* HTTP Method File */ + + enum { + restrict_unix, + restrict_vms, + restrict_windows + } restrict_files_os; /* file name restriction ruleset. */ + bool restrict_files_ctrl; /* non-zero if control chars in URLs + are restricted from appearing in + generated file names. */ + bool restrict_files_nonascii; /* non-zero if bytes with values greater + than 127 are restricted. */ + enum { + restrict_no_case_restriction, + restrict_lowercase, + restrict_uppercase + } restrict_files_case; /* file name case restriction. */ + + bool strict_comments; /* whether strict SGML comments are + enforced. */ + + bool preserve_perm; /* whether remote permissions are used + or that what is set by umask. */ + +#ifdef ENABLE_IPV6 + bool ipv4_only; /* IPv4 connections have been requested. */ + bool ipv6_only; /* IPv4 connections have been requested. */ +#endif + enum { + prefer_ipv4, + prefer_ipv6, + prefer_none + } prefer_family; /* preferred address family when more + than one type is available */ + + bool content_disposition; /* Honor HTTP Content-Disposition header. */ + bool auth_without_challenge; /* Issue Basic authentication creds without + waiting for a challenge. */ + + bool enable_iri; + char *encoding_remote; + const char *locale; + + bool trustservernames; +#ifdef __VMS + int ftp_stmlf; /* Force Stream_LF format for binary FTP. */ +#endif /* def __VMS */ + + bool useservertimestamps; /* Update downloaded files' timestamps to + match those on server? */ + + bool show_all_dns_entries; /* Show all the DNS entries when resolving a + name. */ + bool report_bps; /*Output bandwidth in bits format*/ + +#ifdef HAVE_LIBZ + enum compression_options { + compression_auto, + compression_gzip, + compression_none + } compression; /* type of HTTP compression to use */ +#endif + + char *rejected_log; /* The file to log rejected URLS to. */ + +#ifdef HAVE_HSTS + bool hsts; + char *hsts_file; +#endif + + const char *homedir; /* the homedir of the running process */ + const char *wgetrcfile; /* the wgetrc file to be loaded */ +}; + +extern struct options opt; diff --git a/src/progress.c b/src/progress.c new file mode 100644 index 0000000..d8b57e3 --- /dev/null +++ b/src/progress.c @@ -0,0 +1,1462 @@ +/* Download progress. + Copyright (C) 2001-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <unistd.h> +#include <signal.h> +#include <wchar.h> +#include <mbiter.h> + +#include "progress.h" +#include "utils.h" +#include "retr.h" +#include "c-strcase.h" + +struct progress_implementation { + const char *name; + bool interactive; + void *(*create) (const char *, wgint, wgint); + void (*update) (void *, wgint, double); + void (*draw) (void *); + void (*finish) (void *, double); + void (*set_params) (const char *); +}; + +/* Necessary forward declarations. */ + +static void *dot_create (const char *, wgint, wgint); +static void dot_update (void *, wgint, double); +static void dot_finish (void *, double); +static void dot_draw (void *); +static void dot_set_params (const char *); + +static void *bar_create (const char *, wgint, wgint); +static void bar_update (void *, wgint, double); +static void bar_draw (void *); +static void bar_finish (void *, double); +static void bar_set_params (const char *); + +static struct progress_implementation implementations[] = { + { "dot", 0, dot_create, dot_update, dot_draw, dot_finish, dot_set_params }, + { "bar", 1, bar_create, bar_update, bar_draw, bar_finish, bar_set_params } +}; +static struct progress_implementation *current_impl; +static int current_impl_locked; + +/* Progress implementation used by default. Can be overridden in + wgetrc or by the fallback one. */ + +#define DEFAULT_PROGRESS_IMPLEMENTATION "bar" + +/* Fallback progress implementation should be something that works + under all display types. If you put something other than "dot" + here, remember that bar_set_params tries to switch to this if we're + not running on a TTY. So changing this to "bar" could cause + infloop. */ + +#define FALLBACK_PROGRESS_IMPLEMENTATION "dot" + +/* Return true if NAME names a valid progress bar implementation. The + characters after the first : will be ignored. */ + +bool +valid_progress_implementation_p (const char *name) +{ + size_t i; + struct progress_implementation *pi = implementations; + char *colon = strchr (name, ':'); + size_t namelen = colon ? (size_t) (colon - name) : strlen (name); + + for (i = 0; i < countof (implementations); i++, pi++) + if (!strncmp (pi->name, name, namelen)) + return true; + return false; +} + +/* Set the progress implementation to NAME. */ + +void +set_progress_implementation (const char *name) +{ + size_t i, namelen; + struct progress_implementation *pi = implementations; + const char *colon; + + if (!name) + name = DEFAULT_PROGRESS_IMPLEMENTATION; + + colon = strchr (name, ':'); + namelen = colon ? (size_t) (colon - name) : strlen (name); + + for (i = 0; i < countof (implementations); i++, pi++) + if (!strncmp (pi->name, name, namelen)) + { + current_impl = pi; + current_impl_locked = 0; + + if (colon) + /* We call pi->set_params even if colon is NULL because we + want to give the implementation a chance to set up some + things it needs to run. */ + ++colon; + + if (pi->set_params) + pi->set_params (colon); + return; + } + abort (); +} + +static int output_redirected; + +void +progress_schedule_redirect (void) +{ + output_redirected = 1; +} + +/* Create a progress gauge. INITIAL is the number of bytes the + download starts from (zero if the download starts from scratch). + TOTAL is the expected total number of bytes in this download. If + TOTAL is zero, it means that the download size is not known in + advance. */ + +void * +progress_create (const char *f_download, wgint initial, wgint total) +{ + /* Check if the log status has changed under our feet. */ + if (output_redirected) + { + if (!current_impl_locked) + set_progress_implementation (FALLBACK_PROGRESS_IMPLEMENTATION); + output_redirected = 0; + } + + return current_impl->create (f_download, initial, total); +} + +/* Return true if the progress gauge is "interactive", i.e. if it can + profit from being called regularly even in absence of data. The + progress bar is interactive because it regularly updates the ETA + and current update. */ + +bool +progress_interactive_p (void *progress _GL_UNUSED) +{ + return current_impl->interactive; +} + +/* Inform the progress gauge of newly received bytes. DLTIME is the + time since the beginning of the download. */ + +void +progress_update (void *progress, wgint howmuch, double dltime) +{ + // sanitize input + if (dltime >= INT_MAX) + dltime = INT_MAX - 1; + else if (dltime < 0) + dltime = 0; + + if (howmuch < 0) + howmuch = 0; + + current_impl->update (progress, howmuch, dltime); + current_impl->draw (progress); +} + +/* Tell the progress gauge to clean up. Calling this will free the + PROGRESS object, the further use of which is not allowed. */ + +void +progress_finish (void *progress, double dltime) +{ + // sanitize input + if (dltime >= INT_MAX) + dltime = INT_MAX - 1; + else if (dltime < 0) + dltime = 0; + + current_impl->finish (progress, dltime); +} + +/* Dot-printing. */ + +struct dot_progress { + wgint initial_length; /* how many bytes have been downloaded + previously. */ + wgint total_length; /* expected total byte count when the + download finishes */ + + wgint accumulated; /* number of bytes accumulated after + the last printed dot */ + + double dltime; /* download time so far */ + wgint rows; /* number of rows printed so far */ + int dots; /* number of dots printed in this row */ + + double last_timer_value; +}; + +/* Dot-progress backend for progress_create. */ + +static void * +dot_create (const char *f_download _GL_UNUSED, wgint initial, wgint total) +{ + struct dot_progress *dp = xnew0 (struct dot_progress); + dp->initial_length = initial; + dp->total_length = total; + + if (dp->initial_length) + { + int dot_bytes = opt.dot_bytes; + const wgint ROW_BYTES = opt.dot_bytes * opt.dots_in_line; + + int remainder = dp->initial_length % ROW_BYTES; + wgint skipped = dp->initial_length - remainder; + + if (skipped) + { + wgint skipped_k = skipped / 1024; /* skipped amount in K */ + int skipped_k_len = numdigit (skipped_k); + if (skipped_k_len < 6) + skipped_k_len = 6; + + /* Align the [ skipping ... ] line with the dots. To do + that, insert the number of spaces equal to the number of + digits in the skipped amount in K. */ + logprintf (LOG_PROGRESS, _("\n%*s[ skipping %sK ]"), + 2 + skipped_k_len, "", + number_to_static_string (skipped_k)); + } + + logprintf (LOG_PROGRESS, "\n%6sK", + number_to_static_string (skipped / 1024)); + for (; remainder >= dot_bytes; remainder -= dot_bytes) + { + if (dp->dots % opt.dot_spacing == 0) + logputs (LOG_PROGRESS, " "); + logputs (LOG_PROGRESS, ","); + ++dp->dots; + } + assert (dp->dots < opt.dots_in_line); + + dp->accumulated = remainder; + dp->rows = skipped / ROW_BYTES; + } + + return dp; +} + +static const char *eta_to_human_short (int, bool); + +/* ADD_DOT_ROWS_THRS - minimal (1 << ADD_DOT_ROWS_THRS) ROWS to be added + to the current row if dp->accumulated too much. + Allows to reduce dot_draw io, times. + According to the way progress_update is currently has being called, this + should happens only when fuzzing, or (paranoia) if somehow buffer will + be too large. + Can be disabled by default if this is not fuzzing build. */ +#ifndef ADD_DOT_ROWS_THRS +#if FUZZING +#define ADD_DOT_ROWS_THRS 2 +#else +#define ADD_DOT_ROWS_THRS 2 +#endif +#endif /* ADD_DOT_ROWS_THRS */ + +/* Prints the stats (percentage of completion, speed, ETA) for current + row. DLTIME is the time spent downloading the data in current + row. + + #### This function is somewhat uglified by the fact that current + row and last row have somewhat different stats requirements. It + might be worthwhile to split it to two different functions. */ + +static void +#if ADD_DOT_ROWS_THRS +print_row_stats (struct dot_progress *dp, double dltime, bool last, wgint added_rows) +#else +print_row_stats (struct dot_progress *dp, double dltime, bool last) +#endif +{ + const wgint ROW_BYTES = opt.dot_bytes * opt.dots_in_line; + + /* bytes_displayed is the number of bytes indicated to the user by + dots printed so far, includes the initially "skipped" amount */ + wgint bytes_displayed = dp->rows * ROW_BYTES + dp->dots * opt.dot_bytes; + + if (last) + /* For last row also count bytes accumulated after last dot */ + bytes_displayed += dp->accumulated; + + if (bytes_displayed < 0) + bytes_displayed = 0; + + if (dp->total_length) + { + /* Round to floor value to provide gauge how much data *has* + been retrieved. 12.8% will round to 12% because the 13% mark + has not yet been reached. 100% is only shown when done. */ + int percentage = 100.0 * bytes_displayed / dp->total_length; + logprintf (LOG_PROGRESS, "%3d%%", percentage); + } + + { + static char names[] = {' ', 'K', 'M', 'G', 'T'}; + int units; + double rate; + wgint bytes_this_row; + if (!last) +#if ADD_DOT_ROWS_THRS + bytes_this_row = ROW_BYTES * added_rows; +#else + bytes_this_row = ROW_BYTES; +#endif + else + /* For last row also include bytes accumulated after last dot. */ + bytes_this_row = dp->dots * opt.dot_bytes + dp->accumulated; + /* Don't count the portion of the row belonging to initial_length */ + if (dp->rows == dp->initial_length / ROW_BYTES) + bytes_this_row -= dp->initial_length % ROW_BYTES; + rate = calc_rate (bytes_this_row, dltime - dp->last_timer_value, &units); + logprintf (LOG_PROGRESS, " %4.*f%c", + rate >= 99.95 ? 0 : rate >= 9.995 ? 1 : 2, + rate, names[units]); + dp->last_timer_value = dltime; + } + + if (!last) + { + /* Display ETA based on average speed. Inspired by Vladi + Belperchinov-Shabanski's "wget-new-percentage" patch. */ + if (dp->total_length) + { + wgint bytes_remaining = dp->total_length > bytes_displayed ? dp->total_length - bytes_displayed : 0; + /* The quantity downloaded in this download run. */ + wgint bytes_sofar = bytes_displayed > dp->initial_length ? bytes_displayed - dp->initial_length : 1; + double eta = dltime * bytes_remaining / bytes_sofar; + if (eta < 0) + eta = 0; + if (eta < INT_MAX - 1) + logprintf (LOG_PROGRESS, " %s", + eta_to_human_short ((int) (eta + 0.5), true)); + } + } + else + { + /* When done, print the total download time */ + if (dltime >= 10) + logprintf (LOG_PROGRESS, "=%s", + eta_to_human_short ((int) (dltime + 0.5), true)); + else + logprintf (LOG_PROGRESS, "=%ss", print_decimal (dltime)); + } +} + +/* Dot-progress backend for progress_update. */ + +static void +dot_update (void *progress, wgint howmuch, double dltime) +{ + // sanitize input + if (dltime >= INT_MAX) + dltime = INT_MAX - 1; + else if (dltime < 0) + dltime = 0; + + if (howmuch < 0) + howmuch = 0; + + struct dot_progress *dp = progress; + dp->accumulated += howmuch; + dp->dltime = dltime; +} + +static void +dot_draw (void *progress) +{ + struct dot_progress *dp = progress; + int dot_bytes = opt.dot_bytes; + wgint ROW_BYTES = opt.dot_bytes * opt.dots_in_line; + + log_set_flush (false); + + while (dp->accumulated >= dot_bytes) + { + dp->accumulated -= dot_bytes; + if (dp->dots == 0) + logprintf (LOG_PROGRESS, "\n%6sK", + number_to_static_string (dp->rows * ROW_BYTES / 1024)); + + if (dp->dots % opt.dot_spacing == 0) + logputs (LOG_PROGRESS, " "); + logputs (LOG_PROGRESS, "."); + + ++dp->dots; + if (dp->dots >= opt.dots_in_line) + { + dp->dots = 0; +#if ADD_DOT_ROWS_THRS + { + wgint added_rows = 1; + if (dp->accumulated >= (ROW_BYTES << ADD_DOT_ROWS_THRS)) + { + added_rows += dp->accumulated / ROW_BYTES; + dp->accumulated %= ROW_BYTES; + } + if (WGINT_MAX - dp->rows >= added_rows) + dp->rows += added_rows; + else + dp->rows = WGINT_MAX; + print_row_stats (dp, dp->dltime, false, added_rows); + } +#else + if (dp->rows < WGINT_MAX) + ++dp->rows; + print_row_stats (dp, dp->dltime, false); +#endif /* ADD_DOT_ROWS_THRS */ + } + } + + log_set_flush (true); +} + +/* Dot-progress backend for progress_finish. */ + +static void +dot_finish (void *progress, double dltime) +{ + struct dot_progress *dp = progress; + wgint ROW_BYTES = opt.dot_bytes * opt.dots_in_line; + int i; + + log_set_flush (false); + + if (dp->dots == 0) + logprintf (LOG_PROGRESS, "\n%6sK", + number_to_static_string (dp->rows * ROW_BYTES / 1024)); + for (i = dp->dots; i < opt.dots_in_line; i++) + { + if (i % opt.dot_spacing == 0) + logputs (LOG_PROGRESS, " "); + logputs (LOG_PROGRESS, " "); + } + + // sanitize input + if (dltime >= INT_MAX) + dltime = INT_MAX - 1; + else if (dltime < 0) + dltime = 0; +#if ADD_DOT_ROWS_THRS + print_row_stats (dp, dltime, true, 1); +#else + print_row_stats (dp, dltime, true); +#endif + logputs (LOG_VERBOSE, "\n\n"); + log_set_flush (false); + + xfree (dp); +} + +/* This function interprets the progress "parameters". For example, + if Wget is invoked with --progress=dot:mega, it will set the + "dot-style" to "mega". Valid styles are default, binary, mega, and + giga. */ + +static void +dot_set_params (const char *params) +{ + current_impl->interactive = false; + if (!params || !*params) + params = opt.dot_style; + + if (!params) + return; + + /* We use this to set the retrieval style. */ + if (!c_strcasecmp (params, "default")) + { + /* Default style: 1K dots, 10 dots in a cluster, 50 dots in a + line. */ + opt.dot_bytes = 1024; + opt.dot_spacing = 10; + opt.dots_in_line = 50; + } + else if (!c_strcasecmp (params, "binary")) + { + /* "Binary" retrieval: 8K dots, 16 dots in a cluster, 48 dots + (384K) in a line. */ + opt.dot_bytes = 8192; + opt.dot_spacing = 16; + opt.dots_in_line = 48; + } + else if (!c_strcasecmp (params, "mega")) + { + /* "Mega" retrieval, for retrieving very long files; each dot is + 64K, 8 dots in a cluster, 6 clusters (3M) in a line. */ + opt.dot_bytes = 65536L; + opt.dot_spacing = 8; + opt.dots_in_line = 48; + } + else if (!c_strcasecmp (params, "giga")) + { + /* "Giga" retrieval, for retrieving very very *very* long files; + each dot is 1M, 8 dots in a cluster, 4 clusters (32M) in a + line. */ + opt.dot_bytes = (1L << 20); + opt.dot_spacing = 8; + opt.dots_in_line = 32; + } + else + fprintf (stderr, + _("Invalid dot style specification %s; leaving unchanged.\n"), + quote (params)); +} + +/* "Thermometer" (bar) progress. */ + +/* Assumed screen width if we can't find the real value. */ +#define DEFAULT_SCREEN_WIDTH 80 + +/* Minimum screen width we'll try to work with. If this is too small, + create_image will overflow the buffer. + width = width/4 + 1 + PROGRESS_PERCENT_LEN + PROGRESS_FILESIZE_LEN + + PROGRESS_DWNLOAD_RATE + PROGRESS_ETA_LEN => widh = 38*4/3 (+1) */ +#define MINIMUM_SCREEN_WIDTH 51 + +/* The last known screen width. This can be updated by the code that + detects that SIGWINCH was received (but it's never updated from the + signal handler). */ +static int screen_width; + +/* A flag that, when set, means SIGWINCH was received. */ +static volatile sig_atomic_t received_sigwinch; + +/* Size of the download speed history ring. */ +#define DLSPEED_HISTORY_SIZE 20 + +/* The minimum time length of a history sample. By default, each + sample is at least 150ms long, which means that, over the course of + 20 samples, "current" download speed spans at least 3s into the + past. */ +#define DLSPEED_SAMPLE_MIN 0.15 + +/* The time after which the download starts to be considered + "stalled", i.e. the current bandwidth is not printed and the recent + download speeds are scratched. */ +#define STALL_START_TIME 5 + +/* Time between screen refreshes will not be shorter than this, so + that Wget doesn't swamp the TTY with output. */ +#define REFRESH_INTERVAL 0.2 + +/* Don't refresh the ETA too often to avoid jerkiness in predictions. + This allows ETA to change approximately once per second. */ +#define ETA_REFRESH_INTERVAL 0.99 + +struct bar_progress { + char *f_download; /* Filename of the downloaded file */ + wgint initial_length; /* how many bytes have been downloaded + previously. */ + wgint total_length; /* expected total byte count when the + download finishes */ + wgint count; /* bytes downloaded so far */ + + double last_screen_update; /* time of the last screen update, + measured since the beginning of + download. */ + + double dltime; /* download time so far */ + int width; /* screen width we're using at the + time the progress gauge was + created. this is different from + the screen_width global variable in + that the latter can be changed by a + signal. */ + char *buffer; /* buffer where the bar "image" is + stored. */ + int tick; /* counter used for drawing the + progress bar where the total size + is not known. */ + + /* The following variables (kept in a struct for namespace reasons) + keep track of recent download speeds. See bar_update() for + details. */ + struct bar_progress_hist { + int pos; + double times[DLSPEED_HISTORY_SIZE]; + wgint bytes[DLSPEED_HISTORY_SIZE]; + + /* The sum of times and bytes respectively, maintained for + efficiency. */ + double total_time; + wgint total_bytes; + } hist; + + double recent_start; /* timestamp of beginning of current + position. */ + wgint recent_bytes; /* bytes downloaded so far. */ + + bool stalled; /* set when no data arrives for longer + than STALL_START_TIME, then reset + when new data arrives. */ + + /* create_image() uses these to make sure that ETA information + doesn't flicker. */ + double last_eta_time; /* time of the last update to download + speed and ETA, measured since the + beginning of download. */ + int last_eta_value; +}; + +static void create_image (struct bar_progress *, double, bool); +static void display_image (char *); + +#if USE_NLS_PROGRESS_BAR +static size_t +prepare_filename (char *dest, const char *src) +{ + size_t ret = 1; + if (src) + { + mbi_iterator_t iter; + mbchar_t mbc; + mbi_init (iter, src, strlen (src)); + while (mbi_avail (iter)) + { + size_t i; + mbc = mbi_cur(iter); + /* replace invalid || unprintable || zero-width mbc ws hexdgt code */ + if (!mb_isprint (mbc) || !mb_width (mbc)) + for (i=0; i < mb_len (mbc); i++) + { + if (dest) + dest += sprintf (dest, "%%%02x", (unsigned char) *(mb_ptr(mbc) + i)); + ret += 3; + } + else + { + if (dest) + for (i=0; i < mb_len (mbc); i++) + *dest++ = *(mb_ptr (mbc) + i); + ret += mb_len (mbc); + } + mbi_advance (iter); + } + } + if (dest) + *dest = 0; + return ret; +} +#else +#include <ctype.h> +static size_t +prepare_filename (char *dest, const char *src) +{ + size_t ret = 1; + if (src) + while (*src) + { + /* isprint with some lang return false for some chars */ + if(!iscntrl (*src)) + { + if (dest) + *dest++ = *src; + ret++; + } + else + { + if (dest) + dest += sprintf (dest, "%%%02x", (unsigned char) *src ); + ret += 3; + } + src++; + } + if (dest) + *dest = 0; + return ret; +} +#endif /* USE_NLS_PROGRESS_BAR */ + +static void * +bar_create (const char *f_download, wgint initial, wgint total) +{ + struct bar_progress *bp = xnew0 (struct bar_progress); + + /* In theory, our callers should take care of this pathological + case, but it can sometimes happen. */ + if (initial > total) + total = initial; + + bp->initial_length = initial; + bp->total_length = total; + /* Zero-width mbc must be replaced to avoid buffer overflow. + Another way is to allocate a buffer that allows contain + full f_download, but in this case some escape sequences may break "bar" */ + bp->f_download = xmalloc (prepare_filename (NULL, f_download)); + prepare_filename (bp->f_download, f_download); + + /* Initialize screen_width if this hasn't been done or if it might + have changed, as indicated by receiving SIGWINCH. */ + if (!screen_width || received_sigwinch) + { + screen_width = determine_screen_width (); + if (!screen_width) + screen_width = DEFAULT_SCREEN_WIDTH; + else if (screen_width < MINIMUM_SCREEN_WIDTH) + screen_width = MINIMUM_SCREEN_WIDTH; + received_sigwinch = 0; + } + + /* - 1 because we don't want to use the last screen column. */ + bp->width = screen_width - 1; + /* + enough space for the terminating zero, and hopefully enough room + * for multibyte characters. */ +#define BUF_LEN (bp->width * 2 + 100) + bp->buffer = xcalloc (BUF_LEN, 1); + + logputs (LOG_VERBOSE, "\n"); + + create_image (bp, 0, false); + display_image (bp->buffer); + + return bp; +} + +static void update_speed_ring (struct bar_progress *, wgint, double); + +static void +bar_update (void *progress, wgint howmuch, double dltime) +{ + struct bar_progress *bp = progress; + + bp->dltime = dltime; + if (WGINT_MAX - (bp->count + bp->initial_length) >= howmuch) + bp->count += howmuch; + else + bp->count = WGINT_MAX - bp->initial_length; + if (bp->total_length > 0 + && bp->count + bp->initial_length > bp->total_length) + /* We could be downloading more than total_length, e.g. when the + server sends an incorrect Content-Length header. In that case, + adjust bp->total_length to the new reality, so that the code in + create_image() that depends on total size being smaller or + equal to the expected size doesn't abort. */ + bp->total_length = bp->initial_length + bp->count; + + update_speed_ring (bp, howmuch, dltime); +} + +static void +bar_draw (void *progress) +{ + bool force_screen_update = false; + struct bar_progress *bp = progress; + + /* If SIGWINCH (the window size change signal) been received, + determine the new screen size and update the screen. */ + if (received_sigwinch) + { + int old_width = screen_width; + screen_width = determine_screen_width (); + if (!screen_width) + screen_width = DEFAULT_SCREEN_WIDTH; + else if (screen_width < MINIMUM_SCREEN_WIDTH) + screen_width = MINIMUM_SCREEN_WIDTH; + if (screen_width != old_width) + { + bp->width = screen_width - 1; + bp->buffer = xrealloc (bp->buffer, BUF_LEN); + force_screen_update = true; + } + received_sigwinch = 0; + } + + if (bp->dltime - bp->last_screen_update < REFRESH_INTERVAL && !force_screen_update) + /* Don't update more often than five times per second. */ + return; + + create_image (bp, bp->dltime, false); + display_image (bp->buffer); + bp->last_screen_update = bp->dltime; +} + +static void +bar_finish (void *progress, double dltime) +{ + struct bar_progress *bp = progress; + + if (bp->total_length > 0 + && bp->count + bp->initial_length > bp->total_length) + /* See bar_update() for explanation. */ + bp->total_length = bp->initial_length + bp->count; + + create_image (bp, dltime, true); + display_image (bp->buffer); + + logputs (LOG_VERBOSE, "\n"); + logputs (LOG_PROGRESS, "\n"); + + xfree (bp->f_download); + xfree (bp->buffer); + xfree (bp); +} + +/* This code attempts to maintain the notion of a "current" download + speed, over the course of no less than 3s. (Shorter intervals + produce very erratic results.) + + To do so, it samples the speed in 150ms intervals and stores the + recorded samples in a FIFO history ring. The ring stores no more + than 20 intervals, hence the history covers the period of at least + three seconds and at most 20 reads into the past. This method + should produce reasonable results for downloads ranging from very + slow to very fast. + + The idea is that for fast downloads, we get the speed over exactly + the last three seconds. For slow downloads (where a network read + takes more than 150ms to complete), we get the speed over a larger + time period, as large as it takes to complete twenty reads. This + is good because slow downloads tend to fluctuate more and a + 3-second average would be too erratic. */ + +static void +update_speed_ring (struct bar_progress *bp, wgint howmuch, double dltime) +{ + struct bar_progress_hist *hist = &bp->hist; + double recent_age = dltime - bp->recent_start; + + /* Update the download count. */ + bp->recent_bytes += howmuch; + + /* For very small time intervals, we return after having updated the + "recent" download count. When its age reaches or exceeds minimum + sample time, it will be recorded in the history ring. */ + if (recent_age < DLSPEED_SAMPLE_MIN) + return; + + if (howmuch == 0) + { + /* If we're not downloading anything, we might be stalling, + i.e. not downloading anything for an extended period of time. + Since 0-reads do not enter the history ring, recent_age + effectively measures the time since last read. */ + if (recent_age >= STALL_START_TIME) + { + /* If we're stalling, reset the ring contents because it's + stale and because it will make bar_update stop printing + the (bogus) current bandwidth. */ + bp->stalled = true; + xzero (*hist); + bp->recent_bytes = 0; + } + return; + } + + /* We now have a non-zero amount of to store to the speed ring. */ + + /* If the stall status was acquired, reset it. */ + if (bp->stalled) + { + bp->stalled = false; + /* "recent_age" includes the entire stalled period, which + could be very long. Don't update the speed ring with that + value because the current bandwidth would start too small. + Start with an arbitrary (but more reasonable) time value and + let it level out. */ + recent_age = 1; + } + + /* Store "recent" bytes and download time to history ring at the + position POS. */ + + /* To correctly maintain the totals, first invalidate existing data + (least recent in time) at this position. */ + hist->total_time -= hist->times[hist->pos]; + hist->total_bytes -= hist->bytes[hist->pos]; + + /* Now store the new data and update the totals. */ + hist->times[hist->pos] = recent_age; + hist->bytes[hist->pos] = bp->recent_bytes; + hist->total_time += recent_age; + hist->total_bytes += bp->recent_bytes; + + /* Start a new "recent" period. */ + bp->recent_start = dltime; + bp->recent_bytes = 0; + + /* Advance the current ring position. */ + if (++hist->pos == DLSPEED_HISTORY_SIZE) + hist->pos = 0; + +#if 0 + /* Sledgehammer check to verify that the totals are accurate. */ + { + int i; + double sumt = 0, sumb = 0; + for (i = 0; i < DLSPEED_HISTORY_SIZE; i++) + { + sumt += hist->times[i]; + sumb += hist->bytes[i]; + } + assert (sumb == hist->total_bytes); + /* We can't use assert(sumt==hist->total_time) because some + precision is lost by adding and subtracting floating-point + numbers. But during a download this precision should not be + detectable, i.e. no larger than 1ns. */ + double diff = sumt - hist->total_time; + if (diff < 0) diff = -diff; + assert (diff < 1e-9); + } +#endif +} + +#if USE_NLS_PROGRESS_BAR +static int +count_cols (const char *mbs) +{ + mbchar_t mbc; + mbi_iterator_t iter; + int cols = 0; + mbi_init (iter, mbs, strlen(mbs)); + while (mbi_avail (iter)) + { + mbc = mbi_cur (iter); + cols += mb_width (mbc); + mbi_advance (iter); + } + return cols; +} + +static int +cols_to_bytes (const char *mbs, const int cols, int *ncols) +{ + int p_cols = 0, bytes = 0; + mbchar_t mbc; + mbi_iterator_t iter; + mbi_init (iter, mbs, strlen(mbs)); + while (p_cols < cols && mbi_avail (iter)) + { + mbc = mbi_cur (iter); + p_cols += mb_width (mbc); + /* The multibyte character has exceeded the total number of columns we + * have available. The remaining bytes will be padded with a space. */ + if (p_cols > cols) + { + p_cols -= mb_width (mbc); + break; + } + bytes += mb_len (mbc); + mbi_advance (iter); + } + *ncols = p_cols; + return bytes; +} +#else +static int count_cols (const char *mbs) { return (int) strlen(mbs); } + +static int +cols_to_bytes (const char *mbs, const int cols, int *ncols) +{ + int len = strlen(mbs); + int ret = len < cols ? len : cols; + *ncols = ret; + return ret; +} +#endif + +static const char * +get_eta (int *bcd) +{ + /* TRANSLATORS: "ETA" is English-centric, but this must + be short, ideally 3 chars. Abbreviate if necessary. */ + static const char eta_str[] = N_(" eta %s"); + static const char *eta_trans; + static int bytes_cols_diff; + if (eta_trans == NULL) + { + int nbytes; + int ncols; + +#if USE_NLS_PROGRESS_BAR + eta_trans = _(eta_str); +#else + eta_trans = eta_str; +#endif + + /* Determine the number of bytes used in the translated string, + * versus the number of columns used. This is to figure out how + * many spaces to add at the end to pad to the full line width. + * + * We'll store the difference between the number of bytes and + * number of columns, so that removing this from the string length + * will reveal the total number of columns in the progress bar. */ + nbytes = strlen (eta_trans); + ncols = count_cols (eta_trans); + bytes_cols_diff = nbytes - ncols; + } + + if (bcd != NULL) + *bcd = bytes_cols_diff; + + return eta_trans; +} + +#define APPEND_LITERAL(s) do { \ + memcpy (p, s, sizeof (s) - 1); \ + p += sizeof (s) - 1; \ +} while (0) + +static void +create_image (struct bar_progress *bp, double dl_total_time, bool done) +{ + const int MAX_FILENAME_COLS = bp->width / 4; + char *p = bp->buffer; + wgint size = bp->initial_length + bp->count; + + struct bar_progress_hist *hist = &bp->hist; + int orig_filename_cols = count_cols (bp->f_download); + + int padding; + + /* The progress bar should look like this: + file xx% [=======> ] nnn.nnK 12.34KB/s eta 36m 51s + + Calculate the geometry. The idea is to assign as much room as + possible to the progress bar. The other idea is to never let + things "jitter", i.e. pad elements that vary in size so that + their variance does not affect the placement of other elements. + It would be especially bad for the progress bar to be resized + randomly. + + "file " - Downloaded filename - MAX_FILENAME_COLS chars + 1 + "xx% " or "100%" - percentage - 4 chars + "[]" - progress bar decorations - 2 chars + " nnn.nnK" - downloaded bytes - 7 chars + 1 + " 12.5KB/s" - download rate - 8 chars + 1 + " eta 36m 51s" - ETA - 14 chars + + "=====>..." - progress bar - the rest + */ + + /* TODO: Ask the Turkish Translators to fix their translation for the "done" + * mode of progress bar. Use one less character. Once that is done, redice + * PROGRESS_ETA_LEN by 1. + */ +#define PROGRESS_FILENAME_LEN MAX_FILENAME_COLS + 1 +#define PROGRESS_PERCENT_LEN 4 +#define PROGRESS_DECORAT_LEN 2 +#define PROGRESS_FILESIZE_LEN 7 + 1 +#define PROGRESS_DWNLOAD_RATE 8 + 2 +#define PROGRESS_ETA_LEN 15 + + int progress_size = bp->width - (PROGRESS_FILENAME_LEN + PROGRESS_PERCENT_LEN + + PROGRESS_DECORAT_LEN + PROGRESS_FILESIZE_LEN + + PROGRESS_DWNLOAD_RATE + PROGRESS_ETA_LEN); + + /* The difference between the number of bytes used, + and the number of columns used. */ + int bytes_cols_diff = 0; + int cols_diff; + const char *down_size; + + if (progress_size < 5) + progress_size = 0; + + // sanitize input + if (dl_total_time >= INT_MAX) + dl_total_time = INT_MAX - 1; + else if (dl_total_time < 0) + dl_total_time = 0; + + if (orig_filename_cols < MAX_FILENAME_COLS) + { + p += sprintf (p, "%s", bp->f_download); + padding = MAX_FILENAME_COLS - orig_filename_cols + 1; + memset (p, ' ', padding); + p += padding; + } + else + { +/* + memcpy(p, bp->f_download, MAX_FILENAME_COLS); + p += MAX_FILENAME_COLS; + *p++ = ' '; + } +*/ + int offset_cols; + int bytes_in_filename, offset_bytes, col; + int *cols_ret = &col; + +#define MIN_SCROLL_TEXT 5 + if ((orig_filename_cols > MAX_FILENAME_COLS + MIN_SCROLL_TEXT) && + !opt.noscroll && + !done) + { + offset_cols = ((int) bp->tick + orig_filename_cols + MAX_FILENAME_COLS / 2) + % (orig_filename_cols + MAX_FILENAME_COLS); + if (offset_cols > orig_filename_cols) + { + padding = MAX_FILENAME_COLS - (offset_cols - orig_filename_cols); + memset(p, ' ', padding); + p += padding; + offset_cols = 0; + } + else + padding = 0; + } + else + { + padding = 0; + offset_cols = 0; + } + offset_bytes = cols_to_bytes (bp->f_download, offset_cols, cols_ret); + bytes_in_filename = cols_to_bytes (bp->f_download + offset_bytes, + MAX_FILENAME_COLS - padding, + cols_ret); + memcpy (p, bp->f_download + offset_bytes, bytes_in_filename); + p += bytes_in_filename; + padding = MAX_FILENAME_COLS - (padding + *cols_ret); + memset (p, ' ', padding + 1); + p += padding + 1; + } + + + /* "xx% " */ + if (bp->total_length > 0) + { + int percentage = 100.0 * size / bp->total_length; + assert (percentage <= 100); + p += sprintf (p, "%3d%%", percentage); + } + else + { + memset (p, ' ', PROGRESS_PERCENT_LEN); + p += PROGRESS_PERCENT_LEN; + } + + /* The progress bar: "[====> ]" or "[++==> ]". */ + if (progress_size && bp->total_length > 0) + { + /* Size of the initial portion. */ + int insz = (double)bp->initial_length / bp->total_length * progress_size; + + /* Size of the downloaded portion. */ + int dlsz = (double)size / bp->total_length * progress_size; + + char *begin; + + assert (dlsz <= progress_size); + assert (insz <= dlsz); + + *p++ = '['; + begin = p; + + /* Print the initial portion of the download with '+' chars, the + rest with '=' and one '>'. */ + memset (p, '+', insz); + p += insz; + + dlsz -= insz; + if (dlsz > 0) + { + memset (p, '=', dlsz-1); + p += dlsz - 1; + *p++ = '>'; + } + + memset (p, ' ', (progress_size - (p - begin))); + p += (progress_size - (p - begin)); + *p++ = ']'; + } + else if (progress_size) + { + /* If we can't draw a real progress bar, then at least show + *something* to the user. */ + int ind = bp->tick % (progress_size * 2 - 6); + int i, pos; + + /* Make the star move in two directions. */ + if (ind < progress_size - 2) + pos = ind + 1; + else + pos = progress_size - (ind - progress_size + 5); + + *p++ = '['; + for (i = 0; i < progress_size; i++) + { + if (i == pos - 1) *p++ = '<'; + else if (i == pos ) *p++ = '='; + else if (i == pos + 1) *p++ = '>'; + else + *p++ = ' '; + } + *p++ = ']'; + + } + ++bp->tick; + + /* " 234.56M" */ + down_size = human_readable (size, 1000, 2); + cols_diff = PROGRESS_FILESIZE_LEN - count_cols (down_size); + if (cols_diff > 0) + { + memset (p, ' ', cols_diff); + p += cols_diff; + } + p += sprintf (p, "%s", down_size); + + /* " 12.52Kb/s or 12.52KB/s" */ + if (hist->total_time > 0 && hist->total_bytes) + { + static const char *short_units[] = { " B/s", "KB/s", "MB/s", "GB/s", "TB/s" }; + static const char *short_units_bits[] = { " b/s", "Kb/s", "Mb/s", "Gb/s", "Tb/s" }; + int units = 0; + /* Calculate the download speed using the history ring and + recent data that hasn't made it to the ring yet. */ + wgint dlquant = hist->total_bytes + bp->recent_bytes; + double dltime = hist->total_time + (dl_total_time - bp->recent_start); + double dlspeed = calc_rate (dlquant, dltime, &units); + p += sprintf (p, " %4.*f%s", dlspeed >= 99.95 ? 0 : dlspeed >= 9.995 ? 1 : 2, + dlspeed, !opt.report_bps ? short_units[units] : short_units_bits[units]); + } + else + APPEND_LITERAL (" --.-KB/s"); + + if (!done) + { + /* " eta ..m ..s"; wait for three seconds before displaying the ETA. + That's because the ETA value needs a while to become + reliable. */ + if (bp->total_length > 0 && bp->count > 0 && dl_total_time > 3) + { + int eta; + + /* Don't change the value of ETA more than approximately once + per second; doing so would cause flashing without providing + any value to the user. */ + if (bp->total_length != size + && bp->last_eta_value != 0 + && dl_total_time - bp->last_eta_time < ETA_REFRESH_INTERVAL) + eta = bp->last_eta_value; + else + { + /* Calculate ETA using the average download speed to predict + the future speed. If you want to use a speed averaged + over a more recent period, replace dl_total_time with + hist->total_time and bp->count with hist->total_bytes. + I found that doing that results in a very jerky and + ultimately unreliable ETA. */ + wgint bytes_remaining = bp->total_length - size; + double eta_ = dl_total_time * bytes_remaining / bp->count; + if (eta_ >= INT_MAX - 1) + goto skip_eta; + eta = (int) (eta_ + 0.5); + bp->last_eta_value = eta; + bp->last_eta_time = dl_total_time; + } + + p += sprintf (p, get_eta(&bytes_cols_diff), + eta_to_human_short (eta, false)); + } + else if (bp->total_length > 0) + { + skip_eta: + memset (p, ' ', PROGRESS_ETA_LEN); + p += PROGRESS_ETA_LEN; + } + } + else + { + /* When the download is done, print the elapsed time. */ + int nbytes; + int ncols; + + /* TRANSLATORS: The meaning is "elapsed time", and it is shown + * next to the progress bar once the download is done. + * This should not take up more room than + * available here (6 columns). Abbreviate if necessary. */ + strcpy (p, _(" in ")); + nbytes = strlen (p); + ncols = count_cols (p); + bytes_cols_diff = nbytes - ncols; + if (dl_total_time >= 10) + ncols += sprintf (p + nbytes, "%s", eta_to_human_short ((int) (dl_total_time + 0.5), false)); + else + ncols += sprintf (p + nbytes, "%ss", print_decimal (dl_total_time)); + p += ncols + bytes_cols_diff; + if (ncols < PROGRESS_ETA_LEN) + { + memset (p, ' ', PROGRESS_ETA_LEN - ncols); + p += PROGRESS_ETA_LEN - ncols; + } + } + + *p = '\0'; + + padding = bp->width - count_cols (bp->buffer); + assert (padding >= 0 && "Padding length became non-positive!"); + if (padding > 0) + { +// if (padding > BUF_LEN - (p - bp->buffer) - 1) +// padding = BUF_LEN - (p - bp->buffer) - 1; + memset (p, ' ', padding); + p += padding; + *p = '\0'; + } + + /* 2014-11-14 Darshit Shah <darnir@gmail.com> + * Assert that the length of the progress bar is lesser than the size of the + * screen with which we are dealing. This assertion *MUST* always be removed + * from the release code since we do not want Wget to crash and burn when the + * assertion fails. Instead Wget should continue downloading and display a + * horrible and irritating progress bar that spams the screen with newlines. + * + * By default, all assertions are disabled in a Wget build and are enabled + * only with the --enable-assert configure option. + */ + assert (count_cols (bp->buffer) == bp->width); +} + +/* Print the contents of the buffer as a one-line ASCII "image" so + that it can be overwritten next time. */ + +static void +display_image (char *buf) +{ + bool old = log_set_save_context (false); + logputs (LOG_PROGRESS, "\r"); + logputs (LOG_PROGRESS, buf); + log_set_save_context (old); +} + +static void +bar_set_params (const char *params) +{ +/* if run_with_timeout() will be used for read, needs to disable interactive bar, + or on every timeout(1s) we will have 'retry' with error "decryption failed" */ +#if (defined(HAVE_LIBSSL) || defined(HAVE_LIBSSL32)) && defined(OPENSSL_RUN_WITHTIMEOUT) + current_impl->interactive = false; +#else + current_impl->interactive = true; +#endif + if (params) + { + for (const char *param = params; *param; ) + { + if (!strncmp (param, "force", 5)) + current_impl_locked = 1; + else if (!strncmp (param, "noscroll", 8)) + opt.noscroll = true; + + if (*(param = strchrnul(param, ':'))) + param++; + } + } + + if (((opt.lfilename && opt.show_progress != 1) +#ifdef HAVE_ISATTY + /* The progress bar doesn't make sense if the output is not a + TTY -- when logging to file, it is better to review the + dots. */ + || !isatty (fileno (stderr)) +#endif + ) + && !current_impl_locked) + { + /* We're not printing to a TTY, so revert to the fallback + display. #### We're recursively calling + set_progress_implementation here, which is slightly kludgy. + It would be nicer if we provided that function a return value + indicating a failure of some sort. */ + set_progress_implementation (FALLBACK_PROGRESS_IMPLEMENTATION); + return; + } +} + +#ifdef SIGWINCH +void +progress_handle_sigwinch (int sig _GL_UNUSED) +{ + received_sigwinch = 1; + signal (SIGWINCH, progress_handle_sigwinch); +} +#endif + +/* Provide a short human-readable rendition of the ETA. This is like + secs_to_human_time in main.c, except the output doesn't include + fractions (which would look silly in by nature imprecise ETA) and + takes less room. If the time is measured in hours, hours and + minutes (but not seconds) are shown; if measured in days, then days + and hours are shown. This ensures brevity while still displaying + as much as possible. + + If CONDENSED is true, the separator between minutes and seconds + (and hours and minutes, etc.) is not included, shortening the + display by one additional character. This is used for dot + progress. + + The display never occupies more than 7 characters of screen + space. */ + +static const char * +eta_to_human_short (int secs, bool condensed) +{ + static char buf[16]; /* enough space to be on the safe side */ + static int last = -1; + const char *space = condensed ? "" : " "; + + /* Trivial optimization. create_image can call us every 200 msecs + (see bar_update) for fast downloads, but ETA will only change + once per 900 msecs. */ + if (secs == last) + return buf; + last = secs; + + if (secs < 100) + sprintf (buf, "%ds", secs); + else if (secs < 100 * 60) + sprintf (buf, "%dm%s%ds", secs / 60, space, secs % 60); + else if (secs < 48 * 3600) + sprintf (buf, "%dh%s%dm", secs / 3600, space, (secs / 60) % 60); + else if (secs < 100 * 86400) + sprintf (buf, "%dd%s%dh", secs / 86400, space, (secs / 3600) % 24); + else + /* even (2^31-1)/86400 doesn't overflow BUF. */ + sprintf (buf, "%dd", secs / 86400); + + return buf; +} diff --git a/src/progress.h b/src/progress.h new file mode 100644 index 0000000..c05e9f3 --- /dev/null +++ b/src/progress.h @@ -0,0 +1,45 @@ +/* Download progress. + Copyright (C) 2001-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +\(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef PROGRESS_H +#define PROGRESS_H + +bool valid_progress_implementation_p (const char *); +void set_progress_implementation (const char *); +void progress_schedule_redirect (void); + +void *progress_create (const char *, wgint, wgint); +bool progress_interactive_p (void *); +void progress_update (void *, wgint, double); +void progress_finish (void *, double); + +void progress_handle_sigwinch (int); + +#endif /* PROGRESS_H */ diff --git a/src/ptimer.c b/src/ptimer.c new file mode 100644 index 0000000..2e5efbc --- /dev/null +++ b/src/ptimer.c @@ -0,0 +1,413 @@ +/* Portable timers. + Copyright (C) 2005-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +/* This file implements "portable timers" (ptimers), objects that + measure elapsed time using the primitives most appropriate for the + underlying operating system. The entry points are: + + ptimer_new -- creates a timer. + ptimer_reset -- resets the timer's elapsed time to zero. + ptimer_measure -- measure and return the time elapsed since + creation or last reset. + ptimer_read -- reads the last measured elapsed value. + ptimer_destroy -- destroy the timer. + ptimer_granularity -- returns the approximate granularity of the timers. + + Timers measure time in seconds, returning the timings as floating + point numbers, so they can carry as much precision as the + underlying system timer supports. For example, to measure the time + it takes to run a loop, you can use something like: + + ptimer *tmr = ptimer_new (); + while (...) + ... loop ... + double secs = ptimer_measure (); + printf ("The loop took %.2fs\n", secs); */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <unistd.h> +#include <time.h> +#include <sys/time.h> + +/* Cygwin currently (as of 2005-04-08, Cygwin 1.5.14) lacks clock_getres, + but still defines _POSIX_TIMERS! Because of that we simply use the + Windows timers under Cygwin. */ +#ifdef __CYGWIN__ +# include <windows.h> +#endif + +#include "utils.h" +#include "ptimer.h" + +/* Depending on the OS, one and only one of PTIMER_POSIX, + PTIMER_GETTIMEOFDAY, or PTIMER_WINDOWS will be defined. */ + +#undef PTIMER_POSIX +#undef PTIMER_GETTIMEOFDAY +#undef PTIMER_WINDOWS + +#if defined(WINDOWS) || defined(__CYGWIN__) +# define PTIMER_WINDOWS /* use Windows timers */ +#elif _POSIX_TIMERS - 0 > 0 +# define PTIMER_POSIX /* use POSIX timers (clock_gettime) */ +#else +# define PTIMER_GETTIMEOFDAY /* use gettimeofday */ +#endif + +#ifdef PTIMER_POSIX +/* Elapsed time measurement using POSIX timers: system time is held in + struct timespec, time is retrieved using clock_gettime, and + resolution using clock_getres. + + This method is used on Unix systems that implement POSIX + timers. */ + +typedef struct timespec ptimer_system_time; + +#define IMPL_init posix_init +#define IMPL_measure posix_measure +#define IMPL_diff posix_diff +#define IMPL_resolution posix_resolution + +/* clock_id to use for POSIX clocks. This tries to use + CLOCK_MONOTONIC where available, CLOCK_REALTIME otherwise. */ +static int posix_clock_id; + +/* Resolution of the clock, initialized in posix_init. */ +static double posix_clock_resolution; + +/* Decide which clock_id to use. */ + +static void +posix_init (void) +{ + /* List of clocks we want to support: some systems support monotonic + clocks, Solaris has "high resolution" clock (sometimes + unavailable except to superuser), and all should support the + real-time clock. */ +#define NO_SYSCONF_CHECK -1 + static const struct { + int id; + int sysconf_name; + } clocks[] = { +#if defined(_POSIX_MONOTONIC_CLOCK) && _POSIX_MONOTONIC_CLOCK - 0 >= 0 + { CLOCK_MONOTONIC, _SC_MONOTONIC_CLOCK }, +#endif +#ifdef CLOCK_HIGHRES + { CLOCK_HIGHRES, NO_SYSCONF_CHECK }, +#endif + { CLOCK_REALTIME, NO_SYSCONF_CHECK }, + }; + size_t i; + + /* Determine the clock we can use. For a clock to be usable, it + must be confirmed with sysconf (where applicable) and with + clock_getres. If no clock is found, CLOCK_REALTIME is used. */ + + for (i = 0; i < countof (clocks); i++) + { + struct timespec r; + if (clocks[i].sysconf_name != NO_SYSCONF_CHECK) + if (sysconf (clocks[i].sysconf_name) < 0) + continue; /* sysconf claims this clock is unavailable */ + if (clock_getres (clocks[i].id, &r) < 0) + continue; /* clock_getres doesn't work for this clock */ + posix_clock_id = clocks[i].id; + posix_clock_resolution = (double) r.tv_sec + r.tv_nsec / 1e9; + /* Guard against nonsense returned by a broken clock_getres. */ + if (posix_clock_resolution == 0) + posix_clock_resolution = 1e-3; + break; + } + if (i == countof (clocks)) + { + /* If no clock was found, it means that clock_getres failed for + the realtime clock. */ + logprintf (LOG_NOTQUIET, _("Cannot get REALTIME clock frequency: %s\n"), + strerror (errno)); + /* Use CLOCK_REALTIME, but invent a plausible resolution. */ + posix_clock_id = CLOCK_REALTIME; + posix_clock_resolution = 1e-3; + } +} + +static inline void +posix_measure (ptimer_system_time *pst) +{ + clock_gettime (posix_clock_id, pst); +} + +static inline double +posix_diff (ptimer_system_time *pst1, ptimer_system_time *pst2) +{ + return ((pst1->tv_sec - pst2->tv_sec) + + (pst1->tv_nsec - pst2->tv_nsec) / 1e9); +} + +static inline double +posix_resolution (void) +{ + return posix_clock_resolution; +} +#endif /* PTIMER_POSIX */ + +#ifdef PTIMER_GETTIMEOFDAY +/* Elapsed time measurement using gettimeofday: system time is held in + struct timeval, retrieved using gettimeofday, and resolution is + unknown. + + This method is used Unix systems without POSIX timers. */ + +typedef struct timeval ptimer_system_time; + +#define IMPL_measure gettimeofday_measure +#define IMPL_diff gettimeofday_diff +#define IMPL_resolution gettimeofday_resolution + +static inline void +gettimeofday_measure (ptimer_system_time *pst) +{ + gettimeofday (pst, NULL); +} + +static inline double +gettimeofday_diff (ptimer_system_time *pst1, ptimer_system_time *pst2) +{ + return ((pst1->tv_sec - pst2->tv_sec) + + (pst1->tv_usec - pst2->tv_usec) / 1e6); +} + +static inline double +gettimeofday_resolution (void) +{ + /* Granularity of gettimeofday varies wildly between architectures. + However, it appears that on modern machines it tends to be better + than 1ms. Assume 100 usecs. */ + return 0.1; +} +#endif /* PTIMER_GETTIMEOFDAY */ + +#ifdef PTIMER_WINDOWS +/* Elapsed time measurement on Windows: where high-resolution timers + are available, time is stored in a LARGE_INTEGER and retrieved + using QueryPerformanceCounter. Otherwise, it is stored in a DWORD + and retrieved using GetTickCount. + + This method is used on Windows. */ + +typedef union { + DWORD lores; /* In case GetTickCount is used */ + LARGE_INTEGER hires; /* In case high-resolution timer is used */ +} ptimer_system_time; + +#define IMPL_init windows_init +#define IMPL_measure windows_measure +#define IMPL_diff windows_diff +#define IMPL_resolution windows_resolution + +/* Whether high-resolution timers are used. Set by ptimer_initialize_once + the first time ptimer_new is called. */ +static bool windows_hires_timers; + +/* Frequency of high-resolution timers -- number of updates per + second. Calculated the first time ptimer_new is called provided + that high-resolution timers are available. */ +static double windows_hires_freq; + +static void +windows_init (void) +{ + LARGE_INTEGER freq; + freq.QuadPart = 0; + QueryPerformanceFrequency (&freq); + if (freq.QuadPart != 0) + { + windows_hires_timers = true; + windows_hires_freq = (double) freq.QuadPart; + } +} + +static inline void +windows_measure (ptimer_system_time *pst) +{ + if (windows_hires_timers) + QueryPerformanceCounter (&pst->hires); + else + /* Where hires counters are not available, use GetTickCount rather + GetSystemTime, because it is unaffected by clock skew and + simpler to use. Note that overflows don't affect us because we + never use absolute values of the ticker, only the + differences. */ + pst->lores = GetTickCount (); +} + +static inline double +windows_diff (ptimer_system_time *pst1, ptimer_system_time *pst2) +{ + if (windows_hires_timers) + return (pst1->hires.QuadPart - pst2->hires.QuadPart) / windows_hires_freq; + else + return pst1->lores - pst2->lores; +} + +static double +windows_resolution (void) +{ + if (windows_hires_timers) + return 1.0 / windows_hires_freq; + else + return 10; /* according to MSDN */ +} +#endif /* PTIMER_WINDOWS */ + +/* The code below this point is independent of timer implementation. */ + +struct ptimer { + /* The starting point in time which, subtracted from the current + time, yields elapsed time. */ + ptimer_system_time start; + + /* The most recent elapsed time, calculated by ptimer_measure(). */ + double elapsed_last; + + /* Approximately, the time elapsed between the true start of the + measurement and the time represented by START. This is used for + adjustment when clock skew is detected. */ + double elapsed_pre_start; +}; + +/* Allocate a new timer and reset it. Return the new timer. */ + +struct ptimer * +ptimer_new (void) +{ + struct ptimer *pt = xnew0 (struct ptimer); +#ifdef IMPL_init + static bool init_done; + if (!init_done) + { + init_done = true; + IMPL_init (); + } +#endif + ptimer_reset (pt); + return pt; +} + +/* Free the resources associated with the timer. Its further use is + prohibited. */ + +void +ptimer_destroy (struct ptimer *pt) +{ + xfree (pt); +} + +/* Reset timer PT. This establishes the starting point from which + ptimer_measure() will return the elapsed time in seconds. It is + allowed to reset a previously used timer. */ + +void +ptimer_reset (struct ptimer *pt) +{ + /* Set the start time to the current time. */ + IMPL_measure (&pt->start); + pt->elapsed_last = 0; + pt->elapsed_pre_start = 0; +} + +/* Measure the elapsed time since timer creation/reset. This causes + the timer to internally call clock_gettime (or gettimeofday, etc.) + to update its idea of current time. The time is returned, but is + also stored for later access through ptimer_read(). + + This function handles clock skew, i.e. time that moves backwards is + ignored. */ + +double +ptimer_measure (struct ptimer *pt) +{ + ptimer_system_time now; + double elapsed; + + IMPL_measure (&now); + elapsed = pt->elapsed_pre_start + IMPL_diff (&now, &pt->start); + + /* Ideally we'd just return the difference between NOW and + pt->start. However, the system timer can be set back, and we + could return a value smaller than when we were last called, even + a negative value. Both of these would confuse the callers, which + expect us to return monotonically nondecreasing values. + + Therefore: if ELAPSED is smaller than its previous known value, + we reset pt->start to the current time and effectively start + measuring from this point. But since we don't want the elapsed + value to start from zero, we set elapsed_pre_start to the last + elapsed time and increment all future calculations by that + amount. + + This cannot happen with Windows and POSIX monotonic/highres + timers, but the check is not expensive. */ + + if (elapsed < pt->elapsed_last) + { + pt->start = now; + pt->elapsed_pre_start = pt->elapsed_last; + elapsed = pt->elapsed_last; + } + + pt->elapsed_last = elapsed; + return elapsed; +} + +/* Return the most recent elapsed time measured with ptimer_measure. + If ptimer_measure has not yet been called since the timer was + created or reset, this returns 0. */ + +double +ptimer_read (const struct ptimer *pt) +{ + return pt->elapsed_last; +} + +/* Return the assessed resolution of the timer implementation, in + seconds. This is used by code that tries to substitute a better + value for timers that have returned zero. */ + +double +ptimer_resolution (void) +{ + return IMPL_resolution (); +} diff --git a/src/ptimer.h b/src/ptimer.h new file mode 100644 index 0000000..14b2973 --- /dev/null +++ b/src/ptimer.h @@ -0,0 +1,46 @@ +/* Declarations for ptimer.c. + Copyright (C) 2005-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef PTIMER_H +#define PTIMER_H + +struct ptimer; /* forward declaration; all struct + members are private */ + +struct ptimer *ptimer_new (void); +void ptimer_destroy (struct ptimer *); + +void ptimer_reset (struct ptimer *); +double ptimer_measure (struct ptimer *); +double ptimer_read (const struct ptimer *); + +double ptimer_resolution (void); + +#endif /* PTIMER_H */ diff --git a/src/recur.c b/src/recur.c new file mode 100644 index 0000000..8b408b6 --- /dev/null +++ b/src/recur.c @@ -0,0 +1,919 @@ +/* Handling of recursive HTTP retrieving. + Copyright (C) 1996-2012, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> +#include <assert.h> + +#include "url.h" +#include "recur.h" +#include "utils.h" +#include "retr.h" +#include "ftp.h" +#include "host.h" +#include "hash.h" +#include "res.h" +#include "convert.h" +#include "html-url.h" +#include "css-url.h" +#include "spider.h" +#include "exits.h" + +/* Functions for maintaining the URL queue. */ + +struct queue_element { + const char *url; /* the URL to download */ + const char *referer; /* the referring document */ + int depth; /* the depth */ + bool html_allowed; /* whether the document is allowed to + be treated as HTML. */ + struct iri *iri; /* sXXXav */ + bool css_allowed; /* whether the document is allowed to + be treated as CSS. */ + struct queue_element *next; /* next element in queue */ +}; + +struct url_queue { + struct queue_element *head; + struct queue_element *tail; + int count, maxcount; +}; + +/* Create a URL queue. */ + +static struct url_queue * +url_queue_new (void) +{ + struct url_queue *queue = xnew0 (struct url_queue); + return queue; +} + +/* Delete a URL queue. */ + +static void +url_queue_delete (struct url_queue *queue) +{ + xfree (queue); +} + +/* Enqueue a URL in the queue. The queue is FIFO: the items will be + retrieved ("dequeued") from the queue in the order they were placed + into it. */ + +static void +url_enqueue (struct url_queue *queue, struct iri *i, + const char *url, const char *referer, int depth, + bool html_allowed, bool css_allowed) +{ + struct queue_element *qel = xnew (struct queue_element); + qel->iri = i; + qel->url = url; + qel->referer = referer; + qel->depth = depth; + qel->html_allowed = html_allowed; + qel->css_allowed = css_allowed; + qel->next = NULL; + + ++queue->count; + if (queue->count > queue->maxcount) + queue->maxcount = queue->count; + + DEBUGP (("Enqueuing %s at depth %d\n", + quotearg_n_style (0, escape_quoting_style, url), depth)); + DEBUGP (("Queue count %d, maxcount %d.\n", queue->count, queue->maxcount)); + + if (i) + DEBUGP (("[IRI Enqueuing %s with %s\n", quote_n (0, url), + i->uri_encoding ? quote_n (1, i->uri_encoding) : "None")); + + if (queue->tail) + queue->tail->next = qel; + queue->tail = qel; + + if (!queue->head) + queue->head = queue->tail; +} + +/* Take a URL out of the queue. Return true if this operation + succeeded, or false if the queue is empty. */ + +static bool +url_dequeue (struct url_queue *queue, struct iri **i, + const char **url, const char **referer, int *depth, + bool *html_allowed, bool *css_allowed) +{ + struct queue_element *qel = queue->head; + + if (!qel) + return false; + + queue->head = queue->head->next; + if (!queue->head) + queue->tail = NULL; + + *i = qel->iri; + *url = qel->url; + *referer = qel->referer; + *depth = qel->depth; + *html_allowed = qel->html_allowed; + *css_allowed = qel->css_allowed; + + --queue->count; + + DEBUGP (("Dequeuing %s at depth %d\n", + quotearg_n_style (0, escape_quoting_style, qel->url), qel->depth)); + DEBUGP (("Queue count %d, maxcount %d.\n", queue->count, queue->maxcount)); + + xfree (qel); + return true; +} + +static void blacklist_add (struct hash_table *blacklist, const char *url) +{ + char *url_unescaped = xstrdup (url); + + url_unescape (url_unescaped); + string_set_add (blacklist, url_unescaped); + xfree (url_unescaped); +} + +static int blacklist_contains (struct hash_table *blacklist, const char *url) +{ + char *url_unescaped = xstrdup(url); + int ret; + + url_unescape (url_unescaped); + ret = string_set_contains (blacklist, url_unescaped); + xfree (url_unescaped); + + return ret; +} + +typedef enum +{ + WG_RR_SUCCESS, WG_RR_BLACKLIST, WG_RR_NOTHTTPS, WG_RR_NONHTTP, WG_RR_ABSOLUTE, + WG_RR_DOMAIN, WG_RR_PARENT, WG_RR_LIST, WG_RR_REGEX, WG_RR_RULES, + WG_RR_SPANNEDHOST, WG_RR_ROBOTS +} reject_reason; + +static reject_reason download_child (const struct urlpos *, struct url *, int, + struct url *, struct hash_table *, struct iri *); +static reject_reason descend_redirect (const char *, struct url *, int, + struct url *, struct hash_table *, struct iri *); +static void write_reject_log_header (FILE *); +static void write_reject_log_reason (FILE *, reject_reason, + const struct url *, const struct url *); + +/* Retrieve a part of the web beginning with START_URL. This used to + be called "recursive retrieval", because the old function was + recursive and implemented depth-first search. retrieve_tree on the + other hand implements breadth-search traversal of the tree, which + results in much nicer ordering of downloads. + + The algorithm this function uses is simple: + + 1. put START_URL in the queue. + 2. while there are URLs in the queue: + + 3. get next URL from the queue. + 4. download it. + 5. if the URL is HTML and its depth does not exceed maximum depth, + get the list of URLs embedded therein. + 6. for each of those URLs do the following: + + 7. if the URL is not one of those downloaded before, and if it + satisfies the criteria specified by the various command-line + options, add it to the queue. */ + +uerr_t +retrieve_tree (struct url *start_url_parsed, struct iri *pi) +{ + uerr_t status = RETROK; + + /* The queue of URLs we need to load. */ + struct url_queue *queue; + + /* The URLs we do not wish to enqueue, because they are already in + the queue, but haven't been downloaded yet. */ + struct hash_table *blacklist; + + struct iri *i = iri_new (); + + FILE *rejectedlog = NULL; /* Don't write a rejected log. */ + + /* Duplicate pi struct if not NULL */ + if (pi) + { +#define COPYSTR(x) (x) ? xstrdup(x) : NULL; + i->uri_encoding = COPYSTR (pi->uri_encoding); + i->content_encoding = COPYSTR (pi->content_encoding); + i->utf8_encode = pi->utf8_encode; +#undef COPYSTR + } +#ifdef ENABLE_IRI + else + set_uri_encoding (i, opt.locale, true); +#endif + + queue = url_queue_new (); + blacklist = make_string_hash_table (0); + + /* Enqueue the starting URL. Use start_url_parsed->url rather than + just URL so we enqueue the canonical form of the URL. */ + url_enqueue (queue, i, xstrdup (start_url_parsed->url), NULL, 0, true, + false); + blacklist_add (blacklist, start_url_parsed->url); + + if (opt.rejected_log) + { + rejectedlog = fopen (opt.rejected_log, "w"); + write_reject_log_header (rejectedlog); + if (!rejectedlog) + logprintf (LOG_NOTQUIET, "%s: %s\n", opt.rejected_log, strerror (errno)); + } + + while (1) + { + bool descend = false; + char *url, *referer, *file = NULL; + int depth; + bool html_allowed, css_allowed; + bool is_css = false; + bool dash_p_leaf_HTML = false; + + if (opt.quota && total_downloaded_bytes > opt.quota) + break; + if (status == FWRITEERR) + break; + + /* Get the next URL from the queue... */ + + if (!url_dequeue (queue, (struct iri **) &i, + (const char **)&url, (const char **)&referer, + &depth, &html_allowed, &css_allowed)) + break; + + /* ...and download it. Note that this download is in most cases + unconditional, as download_child already makes sure a file + doesn't get enqueued twice -- and yet this check is here, and + not in download_child. This is so that if you run `wget -r + URL1 URL2', and a random URL is encountered once under URL1 + and again under URL2, but at a different (possibly smaller) + depth, we want the URL's children to be taken into account + the second time. */ + if (dl_url_file_map && hash_table_contains (dl_url_file_map, url)) + { + bool is_css_bool; + + file = xstrdup (hash_table_get (dl_url_file_map, url)); + + DEBUGP (("Already downloaded \"%s\", reusing it from \"%s\".\n", + url, file)); + + if ((is_css_bool = (css_allowed + && downloaded_css_set + && string_set_contains (downloaded_css_set, file))) + || (html_allowed + && downloaded_html_set + && string_set_contains (downloaded_html_set, file))) + { + descend = true; + is_css = is_css_bool; + } + } + else + { + int dt = 0, url_err; + char *redirected = NULL; + struct url *url_parsed = url_parse (url, &url_err, i, true); + + if (!url_parsed) + { + char *error = url_error (url, url_err); + logprintf (LOG_NOTQUIET, "%s: %s.\n",url, error); + xfree (error); + inform_exit_status (URLERROR); + } + else + { + + status = retrieve_url (url_parsed, url, &file, &redirected, referer, + &dt, false, i, true); + + if (html_allowed && file && status == RETROK + && (dt & RETROKF) && (dt & TEXTHTML)) + { + descend = true; + is_css = false; + } + + /* a little different, css_allowed can override content type + lots of web servers serve css with an incorrect content type + */ + if (file && status == RETROK + && (dt & RETROKF) && + ((dt & TEXTCSS) || css_allowed)) + { + descend = true; + is_css = true; + } + + if (redirected) + { + /* We have been redirected, possibly to another host, or + different path, or wherever. Check whether we really + want to follow it. */ + if (descend) + { + reject_reason r = descend_redirect (redirected, url_parsed, + depth, start_url_parsed, blacklist, i); + if (r == WG_RR_SUCCESS) + { + /* Make sure that the old pre-redirect form gets + blacklisted. */ + blacklist_add (blacklist, url); + } + else + { + write_reject_log_reason (rejectedlog, r, url_parsed, start_url_parsed); + descend = false; + } + } + + xfree (url); + url = redirected; + } + else + { + xfree (url); + url = xstrdup (url_parsed->url); + } + url_free (url_parsed); + } + } + + if (opt.spider) + { + visited_url (url, referer); + } + + if (descend + && depth >= opt.reclevel && opt.reclevel != INFINITE_RECURSION) + { + if (opt.page_requisites + && (depth == opt.reclevel || depth == opt.reclevel + 1)) + { + /* When -p is specified, we are allowed to exceed the + maximum depth, but only for the "inline" links, + i.e. those that are needed to display the page. + Originally this could exceed the depth at most by + one, but we allow one more level so that the leaf + pages that contain frames can be loaded + correctly. */ + dash_p_leaf_HTML = true; + } + else + { + /* Either -p wasn't specified or it was and we've + already spent the two extra (pseudo-)levels that it + affords us, so we need to bail out. */ + DEBUGP (("Not descending further; at depth %d, max. %d.\n", + depth, opt.reclevel)); + descend = false; + } + } + + /* If the downloaded document was HTML or CSS, parse it and enqueue the + links it contains. */ + + if (descend) + { + bool meta_disallow_follow = false; + struct urlpos *children + = is_css ? get_urls_css_file (file, url) : + get_urls_html (file, url, &meta_disallow_follow, i); + + if (opt.use_robots && meta_disallow_follow) + { + free_urlpos (children); + children = NULL; + } + + if (children) + { + struct urlpos *child = children; + struct url *url_parsed = url_parse (url, NULL, i, true); + struct iri *ci; + char *referer_url = url; + bool strip_auth; + + assert (url_parsed != NULL); + + if (!url_parsed) + continue; + + strip_auth = (url_parsed && url_parsed->user); + + /* Strip auth info if present */ + if (strip_auth) + referer_url = url_string (url_parsed, URL_AUTH_HIDE); + + for (; child; child = child->next) + { + reject_reason r; + + if (child->ignore_when_downloading) + { + DEBUGP (("Not following due to 'ignore' flag: %s\n", child->url->url)); + continue; + } + + if (dash_p_leaf_HTML && !child->link_inline_p) + { + DEBUGP (("Not following due to 'link inline' flag: %s\n", child->url->url)); + continue; + } + + r = download_child (child, url_parsed, depth, + start_url_parsed, blacklist, i); + if (r == WG_RR_SUCCESS) + { + ci = iri_new (); + set_uri_encoding (ci, i->content_encoding, false); + url_enqueue (queue, ci, xstrdup (child->url->url), + xstrdup (referer_url), depth + 1, + child->link_expect_html, + child->link_expect_css); + /* We blacklist the URL we have enqueued, because we + don't want to enqueue (and hence download) the + same URL twice. */ + blacklist_add (blacklist, child->url->url); + } + else + { + write_reject_log_reason (rejectedlog, r, child->url, url_parsed); + } + } + + if (strip_auth) + xfree (referer_url); + url_free (url_parsed); + free_urlpos (children); + } + } + + if (file + && (opt.delete_after + || opt.spider /* opt.recursive is implicitly true */ + || !acceptable (file))) + { + /* Either --delete-after was specified, or we loaded this + (otherwise unneeded because of --spider or rejected by -R) + HTML file just to harvest its hyperlinks -- in either case, + delete the local file. */ + DEBUGP (("Removing file due to %s in recursive_retrieve():\n", + opt.delete_after ? "--delete-after" : + (opt.spider ? "--spider" : + "recursive rejection criteria"))); + logprintf (LOG_VERBOSE, + (opt.delete_after || opt.spider + ? _("Removing %s.\n") + : _("Removing %s since it should be rejected.\n")), + file); + if (unlink (file)) + logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); + logputs (LOG_VERBOSE, "\n"); + register_delete_file (file); + } + + xfree (url); + xfree (referer); + xfree (file); + iri_free (i); + } + + if (rejectedlog) + { + fclose (rejectedlog); + rejectedlog = NULL; + } + + /* If anything is left of the queue due to a premature exit, free it + now. */ + { + char *d1, *d2; + int d3; + bool d4, d5; + struct iri *d6; + while (url_dequeue (queue, (struct iri **)&d6, + (const char **)&d1, (const char **)&d2, &d3, &d4, &d5)) + { + iri_free (d6); + xfree (d1); + xfree (d2); + } + } + url_queue_delete (queue); + + string_set_free (blacklist); + + if (opt.quota && total_downloaded_bytes > opt.quota) + return QUOTEXC; + else if (status == FWRITEERR) + return FWRITEERR; + else + return RETROK; +} + +/* Based on the context provided by retrieve_tree, decide whether a + URL is to be descended to. This is only ever called from + retrieve_tree, but is in a separate function for clarity. + + The most expensive checks (such as those for robots) are memoized + by storing these URLs to BLACKLIST. This may or may not help. It + will help if those URLs are encountered many times. */ + +static reject_reason +download_child (const struct urlpos *upos, struct url *parent, int depth, + struct url *start_url_parsed, struct hash_table *blacklist, + struct iri *iri) +{ + struct url *u = upos->url; + const char *url = u->url; + bool u_scheme_like_http; + reject_reason reason = WG_RR_SUCCESS; + + DEBUGP (("Deciding whether to enqueue \"%s\".\n", url)); + + if (blacklist_contains (blacklist, url)) + { + if (opt.spider) + { + char *referrer = url_string (parent, URL_AUTH_HIDE_PASSWD); + DEBUGP (("download_child: parent->url is: %s\n", quote (parent->url))); + visited_url (url, referrer); + xfree (referrer); + } + DEBUGP (("Already on the black list.\n")); + reason = WG_RR_BLACKLIST; + goto out; + } + + /* Several things to check for: + 1. if scheme is not https and https_only requested + 2. if scheme is not http, and we don't load it + 3. check for relative links (if relative_only is set) + 4. check for domain + 5. check for no-parent + 6. check for excludes && includes + 7. check for suffix + 8. check for same host (if spanhost is unset), with possible + gethostbyname baggage + 9. check for robots.txt + + Addendum: If the URL is FTP, and it is to be loaded, only the + domain and suffix settings are "stronger". + + Note that .html files will get loaded regardless of suffix rules + (but that is remedied later with unlink) unless the depth equals + the maximum depth. + + More time- and memory- consuming tests should be put later on + the list. */ + +#ifdef HAVE_SSL + if (opt.https_only && u->scheme != SCHEME_HTTPS) + { + DEBUGP (("Not following non-HTTPS links.\n")); + reason = WG_RR_NOTHTTPS; + goto out; + } +#endif + + /* Determine whether URL under consideration has a HTTP-like scheme. */ + u_scheme_like_http = schemes_are_similar_p (u->scheme, SCHEME_HTTP); + + /* 1. Schemes other than HTTP are normally not recursed into. */ + if (!u_scheme_like_http && !((u->scheme == SCHEME_FTP +#ifdef HAVE_SSL + || u->scheme == SCHEME_FTPS +#endif + ) && opt.follow_ftp)) + { + DEBUGP (("Not following non-HTTP schemes.\n")); + reason = WG_RR_NONHTTP; + goto out; + } + + /* 2. If it is an absolute link and they are not followed, throw it + out. */ + if (u_scheme_like_http) + if (opt.relative_only && !upos->link_relative_p) + { + DEBUGP (("It doesn't really look like a relative link.\n")); + reason = WG_RR_ABSOLUTE; + goto out; + } + + /* 3. If its domain is not to be accepted/looked-up, chuck it + out. */ + if (!accept_domain (u)) + { + DEBUGP (("The domain was not accepted.\n")); + reason = WG_RR_DOMAIN; + goto out; + } + + /* 4. Check for parent directory. + + If we descended to a different host or changed the scheme, ignore + opt.no_parent. Also ignore it for documents needed to display + the parent page when in -p mode. */ + if (opt.no_parent + && schemes_are_similar_p (u->scheme, start_url_parsed->scheme) + && 0 == strcasecmp (u->host, start_url_parsed->host) + && (u->scheme != start_url_parsed->scheme + || u->port == start_url_parsed->port) + && !(opt.page_requisites && upos->link_inline_p)) + { + if (!subdir_p (start_url_parsed->dir, u->dir)) + { + DEBUGP (("Going to \"%s\" would escape \"%s\" with no_parent on.\n", + u->dir, start_url_parsed->dir)); + reason = WG_RR_PARENT; + goto out; + } + } + + /* 5. If the file does not match the acceptance list, or is on the + rejection list, chuck it out. The same goes for the directory + exclusion and inclusion lists. */ + if (opt.includes || opt.excludes) + { + if (!accdir (u->dir)) + { + DEBUGP (("%s (%s) is excluded/not-included.\n", url, u->dir)); + reason = WG_RR_LIST; + goto out; + } + } + if (!accept_url (url)) + { + DEBUGP (("%s is excluded/not-included through regex.\n", url)); + reason = WG_RR_REGEX; + goto out; + } + + /* 6. Check for acceptance/rejection rules. We ignore these rules + for directories (no file name to match) and for non-leaf HTMLs, + which can lead to other files that do need to be downloaded. (-p + automatically implies non-leaf because with -p we can, if + necessary, overstep the maximum depth to get the page requisites.) */ + if (u->file[0] != '\0' + && !(has_html_suffix_p (u->file) + /* The exception only applies to non-leaf HTMLs (but -p + always implies non-leaf because we can overstep the + maximum depth to get the requisites): */ + && (/* non-leaf */ + opt.reclevel == INFINITE_RECURSION + /* also non-leaf */ + || depth < opt.reclevel - 1 + /* -p, which implies non-leaf (see above) */ + || opt.page_requisites))) + { + if (!acceptable (u->file)) + { + DEBUGP (("%s (%s) does not match acc/rej rules.\n", + url, u->file)); + reason = WG_RR_RULES; + goto out; + } + } + + /* 7. */ + if (schemes_are_similar_p (u->scheme, parent->scheme)) + if (!opt.spanhost && 0 != strcasecmp (parent->host, u->host)) + { + DEBUGP (("This is not the same hostname as the parent's (%s and %s).\n", + u->host, parent->host)); + reason = WG_RR_SPANNEDHOST; + goto out; + } + + /* 8. */ + if (opt.use_robots && u_scheme_like_http) + { + struct robot_specs *specs = res_get_specs (u->host, u->port); + if (!specs) + { + char *rfile; + if (res_retrieve_file (url, &rfile, iri)) + { + specs = res_parse_from_file (rfile); + + /* Delete the robots.txt file if we chose to either delete the + files after downloading or we're just running a spider. */ + if (opt.delete_after || opt.spider) + { + logprintf (LOG_VERBOSE, _("Removing %s.\n"), rfile); + if (unlink (rfile)) + logprintf (LOG_NOTQUIET, "unlink: %s\n", + strerror (errno)); + } + + xfree (rfile); + } + else + { + /* If we cannot get real specs, at least produce + dummy ones so that we can register them and stop + trying to retrieve them. */ + specs = res_parse ("", 0); + } + res_register_specs (u->host, u->port, specs); + } + + /* Now that we have (or don't have) robots.txt specs, we can + check what they say. */ + if (!res_match_path (specs, u->path)) + { + DEBUGP (("Not following %s because robots.txt forbids it.\n", url)); + blacklist_add (blacklist, url); + reason = WG_RR_ROBOTS; + goto out; + } + } + + out: + + if (reason == WG_RR_SUCCESS) + /* The URL has passed all the tests. It can be placed in the + download queue. */ + DEBUGP (("Decided to load it.\n")); + else + DEBUGP (("Decided NOT to load it.\n")); + + return reason; +} + +/* This function determines whether we will consider downloading the + children of a URL whose download resulted in a redirection, + possibly to another host, etc. It is needed very rarely, and thus + it is merely a simple-minded wrapper around download_child. */ + +static reject_reason +descend_redirect (const char *redirected, struct url *orig_parsed, int depth, + struct url *start_url_parsed, struct hash_table *blacklist, + struct iri *iri) +{ + struct url *new_parsed; + struct urlpos *upos; + reject_reason reason; + + assert (orig_parsed != NULL); + + new_parsed = url_parse (redirected, NULL, NULL, false); + assert (new_parsed != NULL); + + upos = xnew0 (struct urlpos); + upos->url = new_parsed; + + reason = download_child (upos, orig_parsed, depth, + start_url_parsed, blacklist, iri); + + if (reason == WG_RR_SUCCESS) + blacklist_add (blacklist, upos->url->url); + else if (reason == WG_RR_LIST || reason == WG_RR_REGEX) + { + DEBUGP (("Ignoring decision for redirects, decided to load it.\n")); + blacklist_add (blacklist, upos->url->url); + reason = WG_RR_SUCCESS; + } + else + DEBUGP (("Redirection \"%s\" failed the test.\n", redirected)); + + url_free (new_parsed); + xfree (upos); + + return reason; +} + + +/* This function writes the rejected log header. */ +static void +write_reject_log_header (FILE *f) +{ + if (!f) + return; + + /* Note: Update this header when columns change in any way. */ + fprintf (f, "REASON\t" + "U_URL\tU_SCHEME\tU_HOST\tU_PORT\tU_PATH\tU_PARAMS\tU_QUERY\tU_FRAGMENT\t" + "P_URL\tP_SCHEME\tP_HOST\tP_PORT\tP_PATH\tP_PARAMS\tP_QUERY\tP_FRAGMENT\n"); +} + +/* This function writes a URL to the reject log. Internal use only. */ +static void +write_reject_log_url (FILE *fp, const struct url *url) +{ + const char *escaped_str; + const char *scheme_str; + + if (!fp) + return; + + escaped_str = url_escape (url->url); + + switch (url->scheme) + { + case SCHEME_HTTP: scheme_str = "SCHEME_HTTP"; break; +#ifdef HAVE_SSL + case SCHEME_HTTPS: scheme_str = "SCHEME_HTTPS"; break; + case SCHEME_FTPS: scheme_str = "SCHEME_FTPS"; break; +#endif + case SCHEME_FTP: scheme_str = "SCHEME_FTP"; break; + default: scheme_str = "SCHEME_INVALID"; break; + } + + fprintf (fp, "%s\t%s\t%s\t%i\t%s\t%s\t%s\t%s", + escaped_str, + scheme_str, + url->host, + url->port, + url->path, + url->params ? url->params : "", + url->query ? url->query : "", + url->fragment ? url->fragment : ""); + + xfree (escaped_str); +} + +/* This function writes out information on why a URL was rejected and its + context from download_child such as the URL being rejected and it's + parent's URL. The format it uses is comma separated values but with tabs. */ +static void +write_reject_log_reason (FILE *fp, reject_reason reason, + const struct url *url, const struct url *parent) +{ + const char *reason_str; + + if (!fp) + return; + + switch (reason) + { + case WG_RR_SUCCESS: reason_str = "SUCCESS"; break; + case WG_RR_BLACKLIST: reason_str = "BLACKLIST"; break; + case WG_RR_NOTHTTPS: reason_str = "NOTHTTPS"; break; + case WG_RR_NONHTTP: reason_str = "NONHTTP"; break; + case WG_RR_ABSOLUTE: reason_str = "ABSOLUTE"; break; + case WG_RR_DOMAIN: reason_str = "DOMAIN"; break; + case WG_RR_PARENT: reason_str = "PARENT"; break; + case WG_RR_LIST: reason_str = "LIST"; break; + case WG_RR_REGEX: reason_str = "REGEX"; break; + case WG_RR_RULES: reason_str = "RULES"; break; + case WG_RR_SPANNEDHOST: reason_str = "SPANNEDHOST"; break; + case WG_RR_ROBOTS: reason_str = "ROBOTS"; break; + default: reason_str = "UNKNOWN"; break; + } + + fprintf (fp, "%s\t", reason_str); + write_reject_log_url (fp, url); + fprintf (fp, "\t"); + write_reject_log_url (fp, parent); + fprintf (fp, "\n"); +} + +/* vim:set sts=2 sw=2 cino+={s: */ diff --git a/src/recur.h b/src/recur.h new file mode 100644 index 0000000..ea89d99 --- /dev/null +++ b/src/recur.h @@ -0,0 +1,49 @@ +/* Declarations for recur.c. + Copyright (C) 1996-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef RECUR_H +#define RECUR_H + +#include "url.h" + +/* For most options, 0 means no limits, but with -p in the picture, + that causes a problem on the maximum recursion depth variable. To + retain backwards compatibility we allow users to consider "0" to be + synonymous with "inf" for -l, but internally infinite recursion is + specified by -1 and 0 means to only retrieve the requisites of a + single document. */ +#define INFINITE_RECURSION -1 + +struct urlpos; + +void recursive_cleanup (void); +uerr_t retrieve_tree (struct url *, struct iri *); + +#endif /* RECUR_H */ diff --git a/src/res.c b/src/res.c new file mode 100644 index 0000000..0ed0df9 --- /dev/null +++ b/src/res.c @@ -0,0 +1,650 @@ +/* Support for Robot Exclusion Standard (RES). + Copyright (C) 2001, 2006-2011, 2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of Wget. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +/* This file implements the Robot Exclusion Standard (RES). + + RES is a simple protocol that enables site admins to signalize to + the web crawlers that certain parts of the site should not be + accessed. All the admin needs to do is create a "robots.txt" file + in the web server root, and use simple commands to allow or + disallow access to certain parts of the site. + + The first specification was written by Martijn Koster in 1994, and + is still available at <http://www.robotstxt.org/orig.html>. + In 1996, Martijn wrote an Internet Draft specifying an improved RES + specification; however, that work was apparently abandoned since + the draft has expired in 1997 and hasn't been replaced since. The + draft is available at + <http://www.robotstxt.org/norobots-rfc.txt>. + + This file implements RES as specified by the draft. Note that this + only handles the "robots.txt" support. The META tag that controls + whether the links should be followed is handled in `html-url.c'. + + Known deviations: + + * The end-of-line comment recognition is more in the spirit of the + Bourne Shell (as specified by RES-1994). That means that + "foo#bar" is taken literally, whereas "foo #bar" is interpreted + as "foo". The Draft apparently specifies that both should be + interpreted as "foo". + + * We don't recognize sole CR as the line ending. + + * We don't implement expiry mechanism for /robots.txt specs. I + consider it non-necessary for a relatively short-lived + application such as Wget. Besides, it is highly questionable + whether anyone deploys the recommended expiry scheme for + robots.txt. + + Entry points are functions res_parse, res_parse_from_file, + res_match_path, res_register_specs, res_get_specs, and + res_retrieve_file. */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <assert.h> + +#include "utils.h" +#include "hash.h" +#include "url.h" +#include "retr.h" +#include "res.h" +#include "c-strcase.h" + +#ifdef TESTING +#include "../tests/unit-tests.h" +#endif + +struct path_info { + char *path; + bool allowedp; + bool user_agent_exact_p; +}; + +struct robot_specs { + int count; + int size; + struct path_info *paths; +}; + +/* Parsing the robot spec. */ + +/* Check whether AGENT (a string of length LENGTH) equals "wget" or + "*". If it is either of them, *matches is set to one. If it is + "wget", *exact_match is set to one. */ + +static void +match_user_agent (const char *agent, int length, + bool *matches, bool *exact_match) +{ + if (length == 1 && *agent == '*') + { + *matches = true; + *exact_match = false; + } + else if (BOUNDED_EQUAL_NO_CASE (agent, agent + length, "wget")) + { + *matches = true; + *exact_match = true; + } + else + { + *matches = false; + *exact_match = false; + } +} + +/* Add a path specification between PATH_B and PATH_E as one of the + paths in SPECS. */ + +static void +add_path (struct robot_specs *specs, const char *path_b, const char *path_e, + bool allowedp, bool exactp) +{ + struct path_info pp; + if (path_b < path_e && *path_b == '/') + /* Our path representation doesn't use a leading slash, so remove + one from theirs. */ + ++path_b; + pp.path = strdupdelim (path_b, path_e); + pp.allowedp = allowedp; + pp.user_agent_exact_p = exactp; + ++specs->count; + if (specs->count > specs->size) + { + if (specs->size == 0) + specs->size = 1; + else + specs->size <<= 1; + specs->paths = xrealloc (specs->paths, + specs->size * sizeof (struct path_info)); + } + specs->paths[specs->count - 1] = pp; +} + +/* Recreate SPECS->paths with only those paths that have + user_agent_exact_p set to true. */ + +static void +prune_non_exact (struct robot_specs *specs) +{ + struct path_info *newpaths; + int i, j, cnt; + cnt = 0; + for (i = 0; i < specs->count; i++) + if (specs->paths[i].user_agent_exact_p) + ++cnt; + newpaths = xnew_array (struct path_info, cnt); + for (i = 0, j = 0; i < specs->count; i++) + if (specs->paths[i].user_agent_exact_p) + newpaths[j++] = specs->paths[i]; + else + xfree (specs->paths[i].path); + assert (j == cnt); + xfree (specs->paths); + specs->paths = newpaths; + specs->count = cnt; + specs->size = cnt; +} + +#define EOL(p) ((p) >= lineend) + +#define SKIP_SPACE(p) do { \ + while (!EOL (p) && c_isspace (*p)) \ + ++p; \ +} while (0) + +#define FIELD_IS(string_literal) \ + BOUNDED_EQUAL_NO_CASE (field_b, field_e, string_literal) + +/* Parse textual RES specs beginning with SOURCE of length LENGTH. + Return a specs objects ready to be fed to res_match_path. + + The parsing itself is trivial, but creating a correct SPECS object + is trickier than it seems, because RES is surprisingly byzantine if + you attempt to implement it correctly. + + A "record" is a block of one or more `User-Agent' lines followed by + one or more `Allow' or `Disallow' lines. Record is accepted by + Wget if one of the `User-Agent' lines was "wget", or if the user + agent line was "*". + + After all the lines have been read, we examine whether an exact + ("wget") user-agent field was specified. If so, we delete all the + lines read under "User-Agent: *" blocks because we have our own + Wget-specific blocks. This enables the admin to say: + + User-Agent: * + Disallow: / + + User-Agent: google + User-Agent: wget + Disallow: /cgi-bin + + This means that to Wget and to Google, /cgi-bin is disallowed, + whereas for all other crawlers, everything is disallowed. + res_parse is implemented so that the order of records doesn't + matter. In the case above, the "User-Agent: *" could have come + after the other one. */ + +struct robot_specs * +res_parse (const char *source, int length) +{ + int line_count = 1; + + const char *p = source; + const char *end = source + length; + + /* true if last applicable user-agent field matches Wget. */ + bool user_agent_applies = false; + + /* true if last applicable user-agent field *exactly* matches + Wget. */ + bool user_agent_exact = false; + + /* whether we ever encountered exact user agent. */ + bool found_exact = false; + + /* count of allow/disallow lines in the current "record", i.e. after + the last `user-agent' instructions. */ + int record_count = 0; + + struct robot_specs *specs = xnew0 (struct robot_specs); + + while (1) + { + const char *lineend, *lineend_real; + const char *field_b, *field_e; + const char *value_b, *value_e; + + if (p == end) + break; + lineend_real = memchr (p, '\n', end - p); + if (lineend_real) + ++lineend_real; + else + lineend_real = end; + lineend = lineend_real; + + /* Before doing anything else, check whether the line is empty + or comment-only. */ + SKIP_SPACE (p); + if (EOL (p) || *p == '#') + goto next; + + /* Make sure the end-of-line comments are respected by setting + lineend to a location preceding the first comment. Real line + ending remains in lineend_real. */ + for (lineend = p; lineend < lineend_real; lineend++) + if ((lineend == p || c_isspace (*(lineend - 1))) + && *lineend == '#') + break; + + /* Ignore trailing whitespace in the same way. */ + while (lineend > p && c_isspace (*(lineend - 1))) + --lineend; + + assert (!EOL (p)); + + field_b = p; + while (!EOL (p) && (c_isalnum (*p) || *p == '-')) + ++p; + field_e = p; + + SKIP_SPACE (p); + if (field_b == field_e || EOL (p) || *p != ':') + { + DEBUGP (("Ignoring malformed line %d\n", line_count)); + goto next; + } + ++p; /* skip ':' */ + SKIP_SPACE (p); + + value_b = p; + while (!EOL (p)) + ++p; + value_e = p; + + /* Finally, we have a syntactically valid line. */ + if (FIELD_IS ("user-agent")) + { + /* We have to support several cases: + + --previous records-- + + User-Agent: foo + User-Agent: Wget + User-Agent: bar + ... matching record ... + + User-Agent: baz + User-Agent: qux + ... non-matching record ... + + User-Agent: * + ... matching record, but will be pruned later ... + + We have to respect `User-Agent' at the beginning of each + new record simply because we don't know if we're going to + encounter "Wget" among the agents or not. Hence, + match_user_agent is called when record_count != 0. + + But if record_count is 0, we have to keep calling it + until it matches, and if that happens, we must not call + it any more, until the next record. Hence the other part + of the condition. */ + if (record_count != 0 || user_agent_applies == false) + match_user_agent (value_b, value_e - value_b, + &user_agent_applies, &user_agent_exact); + if (user_agent_exact) + found_exact = true; + record_count = 0; + } + else if (FIELD_IS ("allow")) + { + if (user_agent_applies) + { + add_path (specs, value_b, value_e, true, user_agent_exact); + } + ++record_count; + } + else if (FIELD_IS ("disallow")) + { + if (user_agent_applies) + { + bool allowed = false; + if (value_b == value_e) + /* Empty "disallow" line means everything is *allowed*! */ + allowed = true; + add_path (specs, value_b, value_e, allowed, user_agent_exact); + } + ++record_count; + } + else + { + DEBUGP (("Ignoring unknown field at line %d\n", line_count)); + goto next; + } + + next: + p = lineend_real; + ++line_count; + } + + if (found_exact) + { + /* We've encountered an exactly matching user-agent. Throw out + all the stuff with user-agent: *. */ + prune_non_exact (specs); + } + else if (specs->size > specs->count) + { + /* add_path normally over-allocates specs->paths. Reallocate it + to the correct size in order to conserve some memory. */ + specs->paths = xrealloc (specs->paths, + specs->count * sizeof (struct path_info)); + specs->size = specs->count; + } + + return specs; +} + +/* The same like res_parse, but first map the FILENAME into memory, + and then parse it. */ + +struct robot_specs * +res_parse_from_file (const char *filename) +{ + struct robot_specs *specs; + struct file_memory *fm = wget_read_file (filename); + if (!fm) + { + logprintf (LOG_NOTQUIET, _("Cannot open %s: %s\n"), + filename, strerror (errno)); + return NULL; + } + specs = res_parse (fm->content, fm->length); + wget_read_file_free (fm); + return specs; +} + +static void +free_specs (struct robot_specs *specs) +{ + int i; + for (i = 0; i < specs->count; i++) + xfree (specs->paths[i].path); + xfree (specs->paths); + xfree (specs); +} + +/* Matching of a path according to the specs. */ + +/* If C is '%' and (ptr[1], ptr[2]) form a hexadecimal number, and if + that number is not a numerical representation of '/', decode C and + advance the pointer. */ + +#define DECODE_MAYBE(c, ptr) do { \ + if (c == '%' && c_isxdigit (ptr[1]) && c_isxdigit (ptr[2])) \ + { \ + unsigned char decoded = X2DIGITS_TO_NUM (ptr[1], ptr[2]); \ + if (decoded != '/') \ + { \ + c = decoded; \ + ptr += 2; \ + } \ + } \ +} while (0) + +/* The inner matching engine: return true if RECORD_PATH matches + URL_PATH. The rules for matching are described at + <http://www.robotstxt.org/norobots-rfc.txt>, section 3.2.2. */ + +static bool +matches (const char *record_path, const char *url_path) +{ + const char *rp = record_path; + const char *up = url_path; + + for (; ; ++rp, ++up) + { + char rc = *rp; + char uc = *up; + if (!rc) + return true; + if (!uc) + return false; + DECODE_MAYBE(rc, rp); + DECODE_MAYBE(uc, up); + if (rc != uc) + return false; + } +} + +/* Iterate through all paths in SPECS. For the first one that + matches, return its allow/reject status. If none matches, + retrieval is by default allowed. */ + +bool +res_match_path (const struct robot_specs *specs, const char *path) +{ + int i; + if (!specs) + return true; + for (i = 0; i < specs->count; i++) + if (matches (specs->paths[i].path, path)) + { + bool allowedp = specs->paths[i].allowedp; + DEBUGP (("%s path %s because of rule %s.\n", + allowedp ? "Allowing" : "Rejecting", + path, quote (specs->paths[i].path))); + return allowedp; + } + return true; +} + +/* Registering the specs. */ + +static struct hash_table *registered_specs; + +/* Register RES specs that below to server on HOST:PORT. They will + later be retrievable using res_get_specs. */ + +void +res_register_specs (const char *host, int port, struct robot_specs *specs) +{ + struct robot_specs *old; + char buf[256], *hp, *hp_old; + + if (((unsigned) snprintf (buf, sizeof (buf), "%s:%d", host, port)) >= sizeof (buf)) + hp = aprintf("%s:%d", host, port); + else + hp = buf; + + if (!registered_specs) + registered_specs = make_nocase_string_hash_table (0); + + if (hash_table_get_pair (registered_specs, hp, &hp_old, &old)) + { + if (hp != buf) + xfree (hp); + if (old) + free_specs (old); + hash_table_put (registered_specs, hp_old, specs); + } + else + { + hash_table_put (registered_specs, hp == buf ? xstrdup (hp) : hp, specs); + } +} + +/* Get the specs that belong to HOST:PORT. */ + +struct robot_specs * +res_get_specs (const char *host, int port) +{ + char buf[256], *hp; + + if (!registered_specs) + return NULL; + + if (((unsigned) snprintf (buf, sizeof (buf), "%s:%d", host, port)) >= sizeof (buf)) + hp = aprintf("%s:%d", host, port); + else + hp = buf; + + return hash_table_get (registered_specs, hp); +} + +/* Loading the robots file. */ + +#define RES_SPECS_LOCATION "/robots.txt" + +/* Retrieve the robots.txt from the server root of the server that + serves URL. The file will be named according to the currently + active rules, and the file name will be returned in *file. + + Return true if robots were retrieved OK, false otherwise. */ + +bool +res_retrieve_file (const char *url, char **file, struct iri *iri) +{ + struct iri *i = iri_new (); + uerr_t err; + char *robots_url = uri_merge (url, RES_SPECS_LOCATION); + int saved_ts_val = opt.timestamping; + int saved_sp_val = opt.spider, url_err; + struct url * url_parsed; + + /* Copy server URI encoding for a possible IDNA transformation, no need to + encode the full URI in UTF-8 because "robots.txt" is plain ASCII */ + set_uri_encoding (i, iri->uri_encoding, false); + i->utf8_encode = false; + + logputs (LOG_VERBOSE, _("Loading robots.txt; please ignore errors.\n")); + *file = NULL; + opt.timestamping = false; + opt.spider = false; + + url_parsed = url_parse (robots_url, &url_err, i, true); + if (!url_parsed) + { + char *error = url_error (robots_url, url_err); + logprintf (LOG_NOTQUIET, "%s: %s.\n", robots_url, error); + xfree (error); + err = URLERROR; + } + else + { + err = retrieve_url (url_parsed, robots_url, file, NULL, NULL, NULL, + false, i, false); + url_free(url_parsed); + } + + opt.timestamping = saved_ts_val; + opt.spider = saved_sp_val; + xfree (robots_url); + iri_free (i); + + if (err != RETROK && *file != NULL) + { + /* If the file is not retrieved correctly, but retrieve_url + allocated the file name, deallocate is here so that the + caller doesn't have to worry about it. */ + xfree (*file); + } + return err == RETROK; +} + +bool +is_robots_txt_url (const char *url) +{ + char *robots_url = uri_merge (url, RES_SPECS_LOCATION); + bool ret = are_urls_equal (url, robots_url); + + xfree (robots_url); + + return ret; +} + +#if defined DEBUG_MALLOC || defined TESTING +void +res_cleanup (void) +{ + if (registered_specs) + { + hash_table_iterator iter; + for (hash_table_iterate (registered_specs, &iter); + hash_table_iter_next (&iter); + ) + { + xfree (iter.key); + free_specs (iter.value); + } + hash_table_destroy (registered_specs); + registered_specs = NULL; + } +} +#endif + +#ifdef TESTING + +const char * +test_is_robots_txt_url(void) +{ + unsigned i; + static const struct { + const char *url; + bool expected_result; + } test_array[] = { + { "http://www.yoyodyne.com/robots.txt", true }, + { "http://www.yoyodyne.com/somepath/", false }, + { "http://www.yoyodyne.com/somepath/robots.txt", false }, + }; + + for (i = 0; i < countof(test_array); ++i) + { + mu_assert ("test_is_robots_txt_url: wrong result", + is_robots_txt_url (test_array[i].url) == test_array[i].expected_result); + } + + return NULL; +} + +#endif /* TESTING */ + +/* + * vim: et ts=2 sw=2 + */ diff --git a/src/res.h b/src/res.h new file mode 100644 index 0000000..9235e39 --- /dev/null +++ b/src/res.h @@ -0,0 +1,50 @@ +/* Declarations for res.c. + Copyright (C) 2001, 2007-2011, 2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of Wget. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef RES_H +#define RES_H + +struct robot_specs; + +struct robot_specs *res_parse (const char *, int); +struct robot_specs *res_parse_from_file (const char *); + +bool res_match_path (const struct robot_specs *, const char *); + +void res_register_specs (const char *, int, struct robot_specs *); +struct robot_specs *res_get_specs (const char *, int); + +bool res_retrieve_file (const char *, char **, struct iri *); + +bool is_robots_txt_url (const char *); + +void res_cleanup (void); + +#endif /* RES_H */ diff --git a/src/retr.c b/src/retr.c new file mode 100644 index 0000000..9a17be3 --- /dev/null +++ b/src/retr.c @@ -0,0 +1,1561 @@ +/* File retrieval. + Copyright (C) 1996-2011, 2014-2015, 2018-2020 Free Software + Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <errno.h> +#include <string.h> +#include <assert.h> +#ifdef VMS +# include <unixio.h> /* For delete(). */ +#endif + +#ifdef HAVE_LIBZ +# include <zlib.h> +#endif + +#include "exits.h" +#include "utils.h" +#include "retr.h" +#include "progress.h" +#include "url.h" +#include "recur.h" +#include "ftp.h" +#include "http.h" +#include "host.h" +#include "connect.h" +#include "hash.h" +#include "convert.h" +#include "ptimer.h" +#include "html-url.h" +#include "iri.h" +#include "hsts.h" + +/* Total size of downloaded files. Used to enforce quota. */ +SUM_SIZE_INT total_downloaded_bytes; + +/* Total download time in seconds. */ +double total_download_time; + +/* If non-NULL, the stream to which output should be written. This + stream is initialized when `-O' is used. */ +FILE *output_stream; + +/* Whether output_document is a regular file we can manipulate, + i.e. not `-' or a device file. */ +bool output_stream_regular; + +static struct { + wgint chunk_bytes; + double chunk_start; + double sleep_adjust; +} limit_data; + +static void +limit_bandwidth_reset (void) +{ + xzero (limit_data); +} + +#ifdef HAVE_LIBZ +static voidpf +zalloc (voidpf opaque, unsigned int items, unsigned int size) +{ + (void) opaque; + return (voidpf) xcalloc (items, size); +} + +static void +zfree (voidpf opaque, voidpf address) +{ + (void) opaque; + xfree (address); +} +#endif + +/* Limit the bandwidth by pausing the download for an amount of time. + BYTES is the number of bytes received from the network, and TIMER + is the timer that started at the beginning of download. */ + +static void +limit_bandwidth (wgint bytes, struct ptimer *timer) +{ + double delta_t = ptimer_read (timer) - limit_data.chunk_start; + double expected; + + limit_data.chunk_bytes += bytes; + + /* Calculate the amount of time we expect downloading the chunk + should take. If in reality it took less time, sleep to + compensate for the difference. */ + expected = (double) limit_data.chunk_bytes / opt.limit_rate; + + if (expected > delta_t) + { + double slp = expected - delta_t + limit_data.sleep_adjust; + double t0, t1; + if (slp < 0.2) + { + DEBUGP (("deferring a %.2f ms sleep (%s/%.2f).\n", + slp * 1000, number_to_static_string (limit_data.chunk_bytes), + delta_t)); + return; + } + DEBUGP (("\nsleeping %.2f ms for %s bytes, adjust %.2f ms\n", + slp * 1000, number_to_static_string (limit_data.chunk_bytes), + limit_data.sleep_adjust)); + + t0 = ptimer_read (timer); + xsleep (slp); + t1 = ptimer_measure (timer); + + /* Due to scheduling, we probably slept slightly longer (or + shorter) than desired. Calculate the difference between the + desired and the actual sleep, and adjust the next sleep by + that amount. */ + limit_data.sleep_adjust = slp - (t1 - t0); + /* If sleep_adjust is very large, it's likely due to suspension + and not clock inaccuracy. Don't enforce those. */ + if (limit_data.sleep_adjust > 0.5) + limit_data.sleep_adjust = 0.5; + else if (limit_data.sleep_adjust < -0.5) + limit_data.sleep_adjust = -0.5; + } + + limit_data.chunk_bytes = 0; + limit_data.chunk_start = ptimer_read (timer); +} + +/* Write data in BUF to OUT. However, if *SKIP is non-zero, skip that + amount of data and decrease SKIP. Increment *TOTAL by the amount + of data written. If OUT2 is not NULL, also write BUF to OUT2. + In case of error writing to OUT, -2 is returned. In case of error + writing to OUT2, -3 is returned. Return 1 if the whole BUF was + skipped. */ + +static int +write_data (FILE *out, FILE *out2, const char *buf, int bufsize, + wgint *skip, wgint *written) +{ + if (out == NULL && out2 == NULL) + return 1; + + if (skip) + { + if (*skip > bufsize) + { + *skip -= bufsize; + return 1; + } + if (*skip) + { + buf += *skip; + bufsize -= *skip; + *skip = 0; + if (bufsize == 0) + return 1; + } + } + + if (out) + fwrite (buf, 1, bufsize, out); + if (out2) + fwrite (buf, 1, bufsize, out2); + + if (written) + *written += bufsize; + + /* Immediately flush the downloaded data. This should not hinder + performance: fast downloads will arrive in large 16K chunks + (which stdio would write out immediately anyway), and slow + downloads wouldn't be limited by disk speed. */ + + /* 2005-04-20 SMS. + Perhaps it shouldn't hinder performance, but it sure does, at least + on VMS (more than 2X). Rather than speculate on what it should or + shouldn't do, it might make more sense to test it. Even better, it + might be nice to explain what possible benefit it could offer, as + it appears to be a clear invitation to poor performance with no + actual justification. (Also, why 16K? Anyone test other values?) + */ +#ifndef __VMS + if (out) + fflush (out); + if (out2) + fflush (out2); +#endif /* ndef __VMS */ + + if (out && ferror (out)) + return -2; + else if (out2 && ferror (out2)) + return -3; + + return 0; +} + +/* Read the contents of file descriptor FD until it the connection + terminates or a read error occurs. The data is read in portions of + up to 16K and written to OUT as it arrives. If opt.verbose is set, + the progress is shown. + + TOREAD is the amount of data expected to arrive, normally only used + by the progress gauge. + + STARTPOS is the position from which the download starts, used by + the progress gauge. If QTYREAD is non-NULL, the value it points to + is incremented by the amount of data read from the network. If + QTYWRITTEN is non-NULL, the value it points to is incremented by + the amount of data written to disk. The time it took to download + the data is stored to ELAPSED. + + If OUT2 is non-NULL, the contents is also written to OUT2. + OUT2 will get an exact copy of the response: if this is a chunked + response, everything -- including the chunk headers -- is written + to OUT2. (OUT will only get the unchunked response.) + + The function exits and returns the amount of data read. In case of + error while reading data, -1 is returned. In case of error while + writing data to OUT, -2 is returned. In case of error while writing + data to OUT2, -3 is returned. */ + +int +fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread, wgint startpos, + + wgint *qtyread, wgint *qtywritten, double *elapsed, int flags, + FILE *out2) +{ + int ret = 0; +#undef max +#define max(a,b) ((a) > (b) ? (a) : (b)) + int dlbufsize = max (BUFSIZ, 8 * 1024); + char *dlbuf = xmalloc (dlbufsize); + + struct ptimer *timer = NULL; + double last_successful_read_tm = 0; + + /* The progress gauge, set according to the user preferences. */ + void *progress = NULL; + + /* Non-zero if the progress gauge is interactive, i.e. if it can + continually update the display. When true, smaller timeout + values are used so that the gauge can update the display when + data arrives slowly. */ + bool progress_interactive = false; + + bool exact = !!(flags & rb_read_exactly); + + /* Used only by HTTP/HTTPS chunked transfer encoding. */ + bool chunked = flags & rb_chunked_transfer_encoding; + wgint skip = 0; + + /* How much data we've read/written. */ + wgint sum_read = 0; + wgint sum_written = 0; + wgint remaining_chunk_size = 0; + +#ifdef HAVE_LIBZ + /* try to minimize the number of calls to inflate() and write_data() per + call to fd_read() */ + unsigned int gzbufsize = dlbufsize * 4; + char *gzbuf = NULL; + z_stream gzstream; + + if (flags & rb_compressed_gzip) + { + gzbuf = xmalloc (gzbufsize); + if (gzbuf != NULL) + { + gzstream.zalloc = zalloc; + gzstream.zfree = zfree; + gzstream.opaque = Z_NULL; + gzstream.next_in = Z_NULL; + gzstream.avail_in = 0; + + #define GZIP_DETECT 32 /* gzip format detection */ + #define GZIP_WINDOW 15 /* logarithmic window size (default: 15) */ + ret = inflateInit2 (&gzstream, GZIP_DETECT | GZIP_WINDOW); + if (ret != Z_OK) + { + xfree (gzbuf); + errno = (ret == Z_MEM_ERROR) ? ENOMEM : EINVAL; + ret = -1; + goto out; + } + } + else + { + errno = ENOMEM; + ret = -1; + goto out; + } + } +#endif + + if (flags & rb_skip_startpos) + skip = startpos; + + if (opt.show_progress) + { + const char *filename_progress; + /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL + argument to progress_create because the indicator doesn't + (yet) know about "skipping" data. */ + wgint start = skip ? 0 : startpos; + if (opt.dir_prefix) + filename_progress = downloaded_filename + strlen (opt.dir_prefix) + 1; + else + filename_progress = downloaded_filename; + progress = progress_create (filename_progress, start, start + toread); + progress_interactive = progress_interactive_p (progress); + } + + if (opt.limit_rate) + limit_bandwidth_reset (); + + /* A timer is needed for tracking progress, for throttling, and for + tracking elapsed time. If either of these are requested, start + the timer. */ + if (progress || opt.limit_rate || elapsed) + { + timer = ptimer_new (); + last_successful_read_tm = 0; + } + + /* Use a smaller buffer for low requested bandwidths. For example, + with --limit-rate=2k, it doesn't make sense to slurp in 16K of + data and then sleep for 8s. With buffer size equal to the limit, + we never have to sleep for more than one second. */ + if (opt.limit_rate && opt.limit_rate < dlbufsize) + dlbufsize = opt.limit_rate; + + /* Read from FD while there is data to read. Normally toread==0 + means that it is unknown how much data is to arrive. However, if + EXACT is set, then toread==0 means what it says: that no data + should be read. */ + while (!exact || (sum_read < toread)) + { + int rdsize; + double tmout = opt.read_timeout; + + if (chunked) + { + if (remaining_chunk_size == 0) + { + char *line = fd_read_line (fd); + char *endl; + if (line == NULL) + { + ret = -1; + break; + } + else if (out2 != NULL) + fwrite (line, 1, strlen (line), out2); + + remaining_chunk_size = strtol (line, &endl, 16); + xfree (line); + + if (remaining_chunk_size < 0) + { + ret = -1; + break; + } + + if (remaining_chunk_size == 0) + { + ret = 0; + line = fd_read_line (fd); + if (line == NULL) + ret = -1; + else + { + if (out2 != NULL) + fwrite (line, 1, strlen (line), out2); + xfree (line); + } + break; + } + } + + rdsize = MIN (remaining_chunk_size, dlbufsize); + } + else + rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize; + + if (progress_interactive) + { + /* For interactive progress gauges, always specify a ~1s + timeout, so that the gauge can be updated regularly even + when the data arrives very slowly or stalls. */ + tmout = 0.95; + /* avoid wrong 'interactive timeout' */ + errno = 0; + if (opt.read_timeout) + { + double waittm; + waittm = ptimer_read (timer) - last_successful_read_tm; + if (waittm + tmout > opt.read_timeout) + { + /* Don't let total idle time exceed read timeout. */ + tmout = opt.read_timeout - waittm; + /* if 0 fd_read can be 'blocked read' */ + if (tmout <= 0) + { + /* We've already exceeded the timeout. */ + ret = -1, errno = ETIMEDOUT; + break; + } + } + } + } + ret = fd_read (fd, dlbuf, rdsize, tmout); + + if (progress_interactive && ret < 0 && errno == ETIMEDOUT) + ret = 0; /* interactive timeout, handled above */ + else if (ret <= 0) + break; /* EOF or read error */ + + if (progress || opt.limit_rate || elapsed) + { + ptimer_measure (timer); + if (ret > 0) + last_successful_read_tm = ptimer_read (timer); + } + + if (ret > 0) + { + int write_res; + + sum_read += ret; + +#ifdef HAVE_LIBZ + if (gzbuf != NULL) + { + int err; + int towrite; + + /* Write original data to WARC file */ + write_res = write_data (NULL, out2, dlbuf, ret, NULL, NULL); + if (write_res < 0) + { + ret = write_res; + goto out; + } + + gzstream.avail_in = ret; + gzstream.next_in = (unsigned char *) dlbuf; + + do + { + gzstream.avail_out = gzbufsize; + gzstream.next_out = (unsigned char *) gzbuf; + + err = inflate (&gzstream, Z_NO_FLUSH); + + switch (err) + { + case Z_MEM_ERROR: + errno = ENOMEM; + ret = -1; + goto out; + case Z_NEED_DICT: + case Z_DATA_ERROR: + errno = EINVAL; + ret = -1; + goto out; + case Z_STREAM_END: + if (exact && sum_read != toread) + { + DEBUGP(("zlib stream ended unexpectedly after " + "%ld/%ld bytes\n", sum_read, toread)); + } + } + + towrite = gzbufsize - gzstream.avail_out; + write_res = write_data (out, NULL, gzbuf, towrite, &skip, + &sum_written); + if (write_res < 0) + { + ret = write_res; + goto out; + } + } + while (gzstream.avail_out == 0); + } + else +#endif + { + write_res = write_data (out, out2, dlbuf, ret, &skip, + &sum_written); + if (write_res < 0) + { + ret = write_res; + goto out; + } + } + + if (chunked) + { + remaining_chunk_size -= ret; + if (remaining_chunk_size == 0) + { + char *line = fd_read_line (fd); + if (line == NULL) + { + ret = -1; + break; + } + else + { + if (out2 != NULL) + fwrite (line, 1, strlen (line), out2); + xfree (line); + } + } + } + } + + if (opt.limit_rate) + limit_bandwidth (ret, timer); + + if (progress) + progress_update (progress, ret, ptimer_read (timer)); +#ifdef WINDOWS + if (toread > 0 && opt.show_progress) + ws_percenttitle (100.0 * + (startpos + sum_read) / (startpos + toread)); +#endif + } + if (ret < -1) + ret = -1; + + out: + if (progress) + progress_finish (progress, ptimer_read (timer)); + + if (timer) + { + if (elapsed) + *elapsed = ptimer_read (timer); + ptimer_destroy (timer); + } + +#ifdef HAVE_LIBZ + if (gzbuf != NULL) + { + int err = inflateEnd (&gzstream); + if (ret >= 0) + { + /* with compression enabled, ret must be 0 if successful */ + if (err == Z_OK) + ret = 0; + else + { + errno = EINVAL; + ret = -1; + } + } + xfree (gzbuf); + + if (gzstream.total_in != (uLong) sum_read) + { + DEBUGP(("zlib read size differs from raw read size (%lu/%ld)\n", + gzstream.total_in, sum_read)); + } + } +#endif + + if (qtyread) + *qtyread += sum_read; + if (qtywritten) + *qtywritten += sum_written; + + xfree (dlbuf); + + return ret; +} + +/* Read a hunk of data from FD, up until a terminator. The hunk is + limited by whatever the TERMINATOR callback chooses as its + terminator. For example, if terminator stops at newline, the hunk + will consist of a line of data; if terminator stops at two + newlines, it can be used to read the head of an HTTP response. + Upon determining the boundary, the function returns the data (up to + the terminator) in malloc-allocated storage. + + In case of read error, NULL is returned. In case of EOF and no + data read, NULL is returned and errno set to 0. In case of having + read some data, but encountering EOF before seeing the terminator, + the data that has been read is returned, but it will (obviously) + not contain the terminator. + + The TERMINATOR function is called with three arguments: the + beginning of the data read so far, the beginning of the current + block of peeked-at data, and the length of the current block. + Depending on its needs, the function is free to choose whether to + analyze all data or just the newly arrived data. If TERMINATOR + returns NULL, it means that the terminator has not been seen. + Otherwise it should return a pointer to the charactre immediately + following the terminator. + + The idea is to be able to read a line of input, or otherwise a hunk + of text, such as the head of an HTTP request, without crossing the + boundary, so that the next call to fd_read etc. reads the data + after the hunk. To achieve that, this function does the following: + + 1. Peek at incoming data. + + 2. Determine whether the peeked data, along with the previously + read data, includes the terminator. + + 2a. If yes, read the data until the end of the terminator, and + exit. + + 2b. If no, read the peeked data and goto 1. + + The function is careful to assume as little as possible about the + implementation of peeking. For example, every peek is followed by + a read. If the read returns a different amount of data, the + process is retried until all data arrives safely. + + SIZEHINT is the buffer size sufficient to hold all the data in the + typical case (it is used as the initial buffer size). MAXSIZE is + the maximum amount of memory this function is allowed to allocate, + or 0 if no upper limit is to be enforced. + + This function should be used as a building block for other + functions -- see fd_read_line as a simple example. */ + +char * +fd_read_hunk (int fd, hunk_terminator_t terminator, long sizehint, long maxsize) +{ + long bufsize = sizehint; + char *hunk = xmalloc (bufsize); + int tail = 0; /* tail position in HUNK */ + + assert (!maxsize || maxsize >= bufsize); + + while (1) + { + const char *end; + int pklen, rdlen, remain; + + /* First, peek at the available data. */ + + pklen = fd_peek (fd, hunk + tail, bufsize - 1 - tail, -1); + if (pklen < 0) + { + xfree (hunk); + return NULL; + } + end = terminator (hunk, hunk + tail, pklen); + if (end) + { + /* The data contains the terminator: we'll drain the data up + to the end of the terminator. */ + remain = end - (hunk + tail); + assert (remain >= 0); + if (remain == 0) + { + /* No more data needs to be read. */ + hunk[tail] = '\0'; + return hunk; + } + if (bufsize - 1 < tail + remain) + { + bufsize = tail + remain + 1; + hunk = xrealloc (hunk, bufsize); + } + } + else + /* No terminator: simply read the data we know is (or should + be) available. */ + remain = pklen; + + /* Now, read the data. Note that we make no assumptions about + how much data we'll get. (Some TCP stacks are notorious for + read returning less data than the previous MSG_PEEK.) */ + + rdlen = fd_read (fd, hunk + tail, remain, 0); + if (rdlen < 0) + { + xfree (hunk); + return NULL; + } + tail += rdlen; + hunk[tail] = '\0'; + + if (rdlen == 0) + { + if (tail == 0) + { + /* EOF without anything having been read */ + xfree (hunk); + errno = 0; + return NULL; + } + else + /* EOF seen: return the data we've read. */ + return hunk; + } + if (end && rdlen == remain) + /* The terminator was seen and the remaining data drained -- + we got what we came for. */ + return hunk; + + /* Keep looping until all the data arrives. */ + + if (tail == bufsize - 1) + { + /* Double the buffer size, but refuse to allocate more than + MAXSIZE bytes. */ + if (maxsize && bufsize >= maxsize) + { + xfree (hunk); + errno = ENOMEM; + return NULL; + } + bufsize <<= 1; + if (maxsize && bufsize > maxsize) + bufsize = maxsize; + hunk = xrealloc (hunk, bufsize); + } + } +} + +static const char * +line_terminator (const char *start _GL_UNUSED, const char *peeked, int peeklen) +{ + const char *p = memchr (peeked, '\n', peeklen); + if (p) + /* p+1 because the line must include '\n' */ + return p + 1; + return NULL; +} + +/* The maximum size of the single line we agree to accept. This is + not meant to impose an arbitrary limit, but to protect the user + from Wget slurping up available memory upon encountering malicious + or buggy server output. Define it to 0 to remove the limit. */ +#define FD_READ_LINE_MAX 4096 + +/* Read one line from FD and return it. The line is allocated using + malloc, but is never larger than FD_READ_LINE_MAX. + + If an error occurs, or if no data can be read, NULL is returned. + In the former case errno indicates the error condition, and in the + latter case, errno is NULL. */ + +char * +fd_read_line (int fd) +{ + return fd_read_hunk (fd, line_terminator, 128, FD_READ_LINE_MAX); +} + +/* Return a printed representation of the download rate, along with + the units appropriate for the download speed. */ + +const char * +retr_rate (wgint bytes, double secs) +{ + static char res[20]; + static const char *rate_names[] = {"B/s", "KB/s", "MB/s", "GB/s" }; + static const char *rate_names_bits[] = {"b/s", "Kb/s", "Mb/s", "Gb/s" }; + int units; + + double dlrate = calc_rate (bytes, secs, &units); + /* Use more digits for smaller numbers (regardless of unit used), + e.g. "1022", "247", "12.5", "2.38". */ + snprintf (res, sizeof(res), "%.*f %s", + dlrate >= 99.95 ? 0 : dlrate >= 9.995 ? 1 : 2, + dlrate, !opt.report_bps ? rate_names[units]: rate_names_bits[units]); + + return res; +} + +/* Calculate the download rate and trim it as appropriate for the + speed. Appropriate means that if rate is greater than 1K/s, + kilobytes are used, and if rate is greater than 1MB/s, megabytes + are used. + + UNITS is zero for B/s, one for KB/s, two for MB/s, and three for + GB/s. */ + +double +calc_rate (wgint bytes, double secs, int *units) +{ + double dlrate; + double bibyte; + + if (!opt.report_bps) + bibyte = 1024.0; + else + bibyte = 1000.0; + + if (secs == 0) + /* If elapsed time is exactly zero, it means we're under the + resolution of the timer. This can easily happen on systems + that use time() for the timer. Since the interval lies between + 0 and the timer's resolution, assume half the resolution. */ + secs = ptimer_resolution () / 2.0; + + dlrate = secs ? convert_to_bits (bytes) / secs : 0; + if (dlrate < bibyte) + *units = 0; + else if (dlrate < (bibyte * bibyte)) + *units = 1, dlrate /= bibyte; + else if (dlrate < (bibyte * bibyte * bibyte)) + *units = 2, dlrate /= (bibyte * bibyte); + else if (dlrate < (bibyte * bibyte * bibyte * bibyte)) + *units = 3, dlrate /= (bibyte * bibyte * bibyte); + else { + *units = 4, dlrate /= (bibyte * bibyte * bibyte * bibyte); + if (dlrate > 99.99) + dlrate = 99.99; // upper limit 99.99TB/s + } + + return dlrate; +} + + +#define SUSPEND_METHOD do { \ + method_suspended = true; \ + saved_body_data = opt.body_data; \ + saved_body_file_name = opt.body_file; \ + saved_method = opt.method; \ + opt.body_data = NULL; \ + opt.body_file = NULL; \ + opt.method = NULL; \ +} while (0) + +#define RESTORE_METHOD do { \ + if (method_suspended) \ + { \ + opt.body_data = saved_body_data; \ + opt.body_file = saved_body_file_name; \ + opt.method = saved_method; \ + method_suspended = false; \ + } \ +} while (0) + +static char *getproxy (struct url *); + +/* Retrieve the given URL. Decides which loop to call -- HTTP, FTP, + FTP, proxy, etc. */ + +/* #### This function should be rewritten so it doesn't return from + multiple points. */ + +uerr_t +retrieve_url (struct url * orig_parsed, const char *origurl, char **file, + char **newloc, const char *refurl, int *dt, bool recursive, + struct iri *iri, bool register_status) +{ + uerr_t result; + char *url; + bool location_changed; + bool iri_fallbacked = 0; + int dummy; + char *mynewloc, *proxy; + struct url *u = orig_parsed, *proxy_url; + int up_error_code; /* url parse error code */ + char *local_file = NULL; + int redirection_count = 0; + + bool method_suspended = false; + char *saved_body_data = NULL; + char *saved_method = NULL; + char *saved_body_file_name = NULL; + + /* If dt is NULL, use local storage. */ + if (!dt) + { + dt = &dummy; + dummy = 0; + } + url = xstrdup (origurl); + if (newloc) + *newloc = NULL; + if (file) + *file = NULL; + + if (!refurl) + refurl = opt.referer; + + redirected: + /* (also for IRI fallbacking) */ + + result = NOCONERROR; + mynewloc = NULL; + xfree(local_file); + proxy_url = NULL; + + proxy = getproxy (u); + if (proxy) + { + struct iri *pi = iri_new (); + set_uri_encoding (pi, opt.locale, true); + pi->utf8_encode = false; + + /* Parse the proxy URL. */ + proxy_url = url_parse (proxy, &up_error_code, pi, true); + if (!proxy_url) + { + char *error = url_error (proxy, up_error_code); + logprintf (LOG_NOTQUIET, _("Error parsing proxy URL %s: %s.\n"), + proxy, error); + xfree (url); + xfree (error); + xfree (proxy); + iri_free (pi); + RESTORE_METHOD; + result = PROXERR; + if (orig_parsed != u) + url_free (u); + goto bail; + } + if (proxy_url->scheme != SCHEME_HTTP && proxy_url->scheme != u->scheme) + { + logprintf (LOG_NOTQUIET, _("Error in proxy URL %s: Must be HTTP.\n"), proxy); + url_free (proxy_url); + xfree (url); + xfree (proxy); + iri_free (pi); + RESTORE_METHOD; + result = PROXERR; + if (orig_parsed != u) + url_free (u); + goto bail; + } + iri_free(pi); + xfree (proxy); + } + + if (u->scheme == SCHEME_HTTP +#ifdef HAVE_SSL + || u->scheme == SCHEME_HTTPS +#endif + || (proxy_url && proxy_url->scheme == SCHEME_HTTP)) + { +#ifdef HAVE_HSTS +#ifdef TESTING + /* we don't link against main.o when we're testing */ + hsts_store_t hsts_store = NULL; +#else + extern hsts_store_t hsts_store; +#endif + + if (opt.hsts && hsts_store) + { + if (hsts_match (hsts_store, u)) + logprintf (LOG_VERBOSE, "URL transformed to HTTPS due to an HSTS policy\n"); + } +#endif + result = http_loop (u, orig_parsed, &mynewloc, &local_file, refurl, dt, + proxy_url, iri); + } + else if (u->scheme == SCHEME_FTP +#ifdef HAVE_SSL + || u->scheme == SCHEME_FTPS +#endif + ) + { + /* If this is a redirection, temporarily turn off opt.ftp_glob + and opt.recursive, both being undesirable when following + redirects. */ + bool oldrec = recursive, glob = opt.ftp_glob; + if (redirection_count) + oldrec = glob = false; + + result = ftp_loop (u, orig_parsed, &local_file, dt, proxy_url, + recursive, glob); + recursive = oldrec; + + /* There is a possibility of having HTTP being redirected to + FTP. In these cases we must decide whether the text is HTML + according to the suffix. The HTML suffixes are `.html', + `.htm' and a few others, case-insensitive. */ + if (redirection_count && local_file && (u->scheme == SCHEME_FTP +#ifdef HAVE_SSL + || u->scheme == SCHEME_FTPS +#endif + )) + { + if (has_html_suffix_p (local_file)) + *dt |= TEXTHTML; + } + } + + if (proxy_url) + { + url_free (proxy_url); + proxy_url = NULL; + } + + location_changed = (result == NEWLOCATION || result == NEWLOCATION_KEEP_POST); + if (location_changed) + { + char *construced_newloc; + struct url *newloc_parsed; + + assert (mynewloc != NULL); + + xfree (local_file); + + /* The HTTP specs only allow absolute URLs to appear in + redirects, but a ton of boneheaded webservers and CGIs out + there break the rules and use relative URLs, and popular + browsers are lenient about this, so wget should be too. */ + construced_newloc = uri_merge (url, mynewloc ? mynewloc : ""); + xfree (mynewloc); + mynewloc = construced_newloc; + +#ifdef ENABLE_IRI + /* Reset UTF-8 encoding state, set the URI encoding and reset + the content encoding. */ + iri->utf8_encode = opt.enable_iri; + if (opt.encoding_remote) + set_uri_encoding (iri, opt.encoding_remote, true); + set_content_encoding (iri, NULL); + xfree (iri->orig_url); +#endif + + /* Now, see if this new location makes sense. */ + newloc_parsed = url_parse (mynewloc, &up_error_code, iri, true); + if (!newloc_parsed) + { + char *error = url_error (mynewloc, up_error_code); + logprintf (LOG_NOTQUIET, "%s: %s.\n", escnonprint_uri (mynewloc), + error); + if (orig_parsed != u) + { + url_free (u); + } + xfree (url); + xfree (mynewloc); + xfree (error); + RESTORE_METHOD; + goto bail; + } + + /* Now mynewloc will become newloc_parsed->url, because if the + Location contained relative paths like .././something, we + don't want that propagating as url. */ + xfree (mynewloc); + mynewloc = xstrdup (newloc_parsed->url); + + /* Check for max. number of redirections. */ + if (++redirection_count > opt.max_redirect) + { + logprintf (LOG_NOTQUIET, _("%d redirections exceeded.\n"), + opt.max_redirect); + url_free (newloc_parsed); + if (orig_parsed != u) + { + url_free (u); + } + xfree (url); + xfree (mynewloc); + RESTORE_METHOD; + result = WRONGCODE; + goto bail; + } + + xfree (url); + url = mynewloc; + if (orig_parsed != u) + { + url_free (u); + } + u = newloc_parsed; + + /* If we're being redirected from POST, and we received a + redirect code different than 307, we don't want to POST + again. Many requests answer POST with a redirection to an + index page; that redirection is clearly a GET. We "suspend" + POST data for the duration of the redirections, and restore + it when we're done. + + RFC2616 HTTP/1.1 introduces code 307 Temporary Redirect + specifically to preserve the method of the request. + */ + if (result != NEWLOCATION_KEEP_POST && !method_suspended) + SUSPEND_METHOD; + + goto redirected; + } + else + { + xfree(mynewloc); + } + + /* Try to not encode in UTF-8 if fetching failed */ + if (!(*dt & RETROKF) && iri->utf8_encode) + { + iri->utf8_encode = false; + if (orig_parsed != u) + { + url_free (u); + } + u = url_parse (origurl, NULL, iri, true); + if (u) + { + if (strcmp(u->url, orig_parsed->url)) + { + DEBUGP (("[IRI fallbacking to non-utf8 for %s\n", quote (url))); + xfree (url); + url = xstrdup (u->url); + iri_fallbacked = 1; + goto redirected; + } + else + DEBUGP (("[Needn't fallback to non-utf8 for %s\n", quote (url))); + } + else + DEBUGP (("[Couldn't fallback to non-utf8 for %s\n", quote (url))); + } + + if (local_file && u && (*dt & RETROKF || opt.content_on_error)) + { + register_download (u->url, local_file); + + if (!opt.spider && redirection_count && 0 != strcmp (origurl, u->url)) + register_redirection (origurl, u->url); + + if (*dt & TEXTHTML) + register_html (local_file); + + if (*dt & TEXTCSS) + register_css (local_file); + } + + if (file) + *file = local_file ? local_file : NULL; + else + xfree (local_file); + + if (orig_parsed != u) + url_free (u); + + if (redirection_count || iri_fallbacked) + { + if (newloc) + *newloc = url; + else + xfree (url); + } + else + { + if (newloc) + *newloc = NULL; + xfree (url); + } + + RESTORE_METHOD; + +bail: + if (register_status) + inform_exit_status (result); + + return result; +} + +/* Find the URLs in the file and call retrieve_url() for each of them. + If HTML is true, treat the file as HTML, and construct the URLs + accordingly. + + If opt.recursive is set, call retrieve_tree() for each file. */ + +uerr_t +retrieve_from_file (const char *file, bool html, int *count) +{ + uerr_t status; + struct urlpos *url_list, *cur_url; + struct iri *iri = iri_new(); + + char *input_file, *url_file = NULL; + const char *url = file; + + status = RETROK; /* Suppose everything is OK. */ + *count = 0; /* Reset the URL count. */ + + /* sXXXav : Assume filename and links in the file are in the locale */ + set_uri_encoding (iri, opt.locale, true); + set_content_encoding (iri, opt.locale); + + if (url_valid_scheme (url)) + { + int dt,url_err; + struct url *url_parsed = url_parse (url, &url_err, iri, true); + if (!url_parsed) + { + char *error = url_error (url, url_err); + logprintf (LOG_NOTQUIET, "%s: %s.\n", url, error); + xfree (error); + iri_free (iri); + return URLERROR; + } + + if (!opt.base_href) + opt.base_href = xstrdup (url); + + status = retrieve_url (url_parsed, url, &url_file, NULL, NULL, &dt, + false, iri, true); + url_free (url_parsed); + + if (!url_file || (status != RETROK)) + return status; + + if (dt & TEXTHTML) + html = true; + +#ifdef ENABLE_IRI + /* If we have a found a content encoding, use it. + * ( == is okay, because we're checking for identical object) */ + if (iri->content_encoding != opt.locale) + set_uri_encoding (iri, iri->content_encoding, false); +#endif + + /* Reset UTF-8 encode status */ + iri->utf8_encode = opt.enable_iri; + xfree (iri->orig_url); + + input_file = url_file; + } + else + input_file = (char *) file; + + url_list = (html ? get_urls_html (input_file, NULL, NULL, iri) + : get_urls_file (input_file)); + + xfree (url_file); + + for (cur_url = url_list; cur_url; cur_url = cur_url->next, ++*count) + { + char *filename = NULL, *new_file = NULL, *proxy; + int dt = 0; + struct iri *tmpiri = iri_dup (iri); + struct url *parsed_url = NULL; + + if (cur_url->ignore_when_downloading) + continue; + + if (opt.quota && total_downloaded_bytes > opt.quota) + { + status = QUOTEXC; + break; + } + + parsed_url = url_parse (cur_url->url->url, NULL, tmpiri, true); + + proxy = getproxy (cur_url->url); + if ((opt.recursive || opt.page_requisites) + && ((cur_url->url->scheme != SCHEME_FTP +#ifdef HAVE_SSL + && cur_url->url->scheme != SCHEME_FTPS +#endif + ) || proxy)) + { + int old_follow_ftp = opt.follow_ftp; + + /* Turn opt.follow_ftp on in case of recursive FTP retrieval */ + if (cur_url->url->scheme == SCHEME_FTP +#ifdef HAVE_SSL + || cur_url->url->scheme == SCHEME_FTPS +#endif + ) + opt.follow_ftp = 1; + + status = retrieve_tree (parsed_url ? parsed_url : cur_url->url, + tmpiri); + + opt.follow_ftp = old_follow_ftp; + } + else + status = retrieve_url (parsed_url ? parsed_url : cur_url->url, + cur_url->url->url, &filename, + &new_file, NULL, &dt, opt.recursive, tmpiri, + true); + xfree (proxy); + + if (parsed_url) + url_free (parsed_url); + + if (filename && opt.delete_after && file_exists_p (filename, NULL)) + { + DEBUGP (("\ +Removing file due to --delete-after in retrieve_from_file():\n")); + logprintf (LOG_VERBOSE, _("Removing %s.\n"), filename); + if (unlink (filename)) + logprintf (LOG_NOTQUIET, "Failed to unlink %s: (%d) %s\n", filename, errno, strerror (errno)); + dt &= ~RETROKF; + } + + xfree (new_file); + xfree (filename); + iri_free (tmpiri); + } + + /* Free the linked list of URL-s. */ + free_urlpos (url_list); + + iri_free (iri); + + return status; +} + +/* Print `giving up', or `retrying', depending on the impending + action. N1 and N2 are the attempt number and the attempt limit. */ +void +printwhat (int n1, int n2) +{ + logputs (LOG_VERBOSE, (n1 == n2) ? _("Giving up.\n\n") : _("Retrying.\n\n")); +} + +/* If opt.wait or opt.waitretry are specified, and if certain + conditions are met, sleep the appropriate number of seconds. See + the documentation of --wait and --waitretry for more information. + + COUNT is the count of current retrieval, beginning with 1. */ + +void +sleep_between_retrievals (int count) +{ + static bool first_retrieval = true; + + if (first_retrieval) + { + /* Don't sleep before the very first retrieval. */ + first_retrieval = false; + return; + } + + if (opt.waitretry && count > 1) + { + /* If opt.waitretry is specified and this is a retry, wait for + COUNT-1 number of seconds, or for opt.waitretry seconds. */ + if (count <= opt.waitretry) + xsleep (count - 1); + else + xsleep (opt.waitretry); + } + else if (opt.wait) + { + if (!opt.random_wait || count > 1) + /* If random-wait is not specified, or if we are sleeping + between retries of the same download, sleep the fixed + interval. */ + xsleep (opt.wait); + else + { + /* Sleep a random amount of time averaging in opt.wait + seconds. The sleeping amount ranges from 0.5*opt.wait to + 1.5*opt.wait. */ + double waitsecs = (0.5 + random_float ()) * opt.wait; + DEBUGP (("sleep_between_retrievals: avg=%f,sleep=%f\n", + opt.wait, waitsecs)); + xsleep (waitsecs); + } + } +} + +/* Free the linked list of urlpos. */ +void +free_urlpos (struct urlpos *l) +{ + while (l) + { + struct urlpos *next = l->next; + if (l->url) + url_free (l->url); + xfree (l->local_name); + xfree (l); + l = next; + } +} + +/* Rotate FNAME opt.backups times */ +void +rotate_backups(const char *fname) +{ +#ifdef __VMS +# define SEP "_" +# define AVS ";*" /* All-version suffix. */ +# define AVSL (sizeof (AVS) - 1) +#else +# define SEP "." +# define AVSL 0 +#endif + + /* avoid alloca() here */ + char from[1024], to[1024]; + struct stat sb; + bool overflow; + int i; + + if (stat (fname, &sb) == 0) + if (S_ISREG (sb.st_mode) == 0) + return; + + for (i = opt.backups; i > 1; i--) + { +#ifdef VMS + /* Delete (all versions of) any existing max-suffix file, to avoid + * creating multiple versions of it. (On VMS, rename() will + * create a new version of an existing destination file, not + * destroy/overwrite it.) + */ + if (i == opt.backups) + { + if (((unsigned) snprintf (to, sizeof (to), "%s%s%d%s", fname, SEP, i, AVS)) >= sizeof (to)) + logprintf (LOG_NOTQUIET, "Failed to delete %s: File name truncation\n", to); + else + delete (to); + } +#endif + if ((overflow = ((unsigned) snprintf (to, sizeof (to), "%s%s%d", fname, SEP, i)) >= sizeof (to))) + errno = ENAMETOOLONG; + else if ((overflow = ((unsigned) snprintf (from, sizeof (from), "%s%s%d", fname, SEP, i - 1)) >= sizeof (from))) + errno = ENAMETOOLONG; + if (overflow || rename (from, to)) + logprintf (LOG_NOTQUIET, "Failed to rename %s to %s: (%d) %s\n", + from, to, errno, strerror (errno)); + } + + if ((overflow = ((unsigned) snprintf (to, sizeof (to), "%s%s%d", fname, SEP, 1)) >= sizeof (to))) + errno = ENAMETOOLONG; + if (overflow || rename(fname, to)) + logprintf (LOG_NOTQUIET, "Failed to rename %s to %s: (%d) %s\n", + fname, to, errno, strerror (errno)); +} + +static bool no_proxy_match (const char *, const char **); + +/* Return the URL of the proxy appropriate for url U. */ + +static char * +getproxy (struct url *u) +{ + char *proxy = NULL; + char *rewritten_url; + + if (!opt.use_proxy) + return NULL; + if (no_proxy_match (u->host, (const char **)opt.no_proxy)) + return NULL; + + switch (u->scheme) + { + case SCHEME_HTTP: + proxy = opt.http_proxy ? opt.http_proxy : getenv ("http_proxy"); + break; +#ifdef HAVE_SSL + case SCHEME_HTTPS: + proxy = opt.https_proxy ? opt.https_proxy : getenv ("https_proxy"); + break; + case SCHEME_FTPS: + proxy = opt.ftp_proxy ? opt.ftp_proxy : getenv ("ftps_proxy"); + break; +#endif + case SCHEME_FTP: + proxy = opt.ftp_proxy ? opt.ftp_proxy : getenv ("ftp_proxy"); + break; + case SCHEME_INVALID: + break; + } + if (!proxy || !*proxy) + return NULL; + + /* Handle shorthands. `rewritten_storage' is a kludge to allow + getproxy() to return static storage. */ + rewritten_url = rewrite_shorthand_url (proxy); + if (rewritten_url) + return rewritten_url; + + return strdup(proxy); +} + +/* Returns true if URL would be downloaded through a proxy. */ + +bool +url_uses_proxy (struct url * u) +{ + bool ret; + char *proxy; + + if (!u) + return false; + proxy = getproxy (u); + ret = proxy != NULL; + xfree (proxy); + return ret; +} + +/* Should a host be accessed through proxy, concerning no_proxy? */ +static bool +no_proxy_match (const char *host, const char **no_proxy) +{ + if (!no_proxy) + return false; + else + return sufmatch (no_proxy, host); +} + +/* Set the file parameter to point to the local file string. */ +void +set_local_file (const char **file, const char *default_file) +{ + if (opt.output_document) + { + if (output_stream_regular) + *file = opt.output_document; + } + else + *file = default_file; +} + +/* Return true for an input file's own URL, false otherwise. */ +bool +input_file_url (const char *input_file) +{ + static bool first = true; + + if (input_file + && url_has_scheme (input_file) + && first) + { + first = false; + return true; + } + else + return false; +} diff --git a/src/retr.h b/src/retr.h new file mode 100644 index 0000000..9da2a27 --- /dev/null +++ b/src/retr.h @@ -0,0 +1,81 @@ +/* Declarations for retr.c. + Copyright (C) 1996-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef RETR_H +#define RETR_H + +#include "url.h" + +extern int numurls; + +/* These global vars should be made static to retr.c and exported via + functions! */ +extern SUM_SIZE_INT total_downloaded_bytes; +extern double total_download_time; +extern FILE *output_stream; +extern bool output_stream_regular; + +/* Flags for fd_read_body. */ +enum { + rb_read_exactly = 1, + rb_skip_startpos = 2, + + /* Used by HTTP/HTTPS*/ + rb_chunked_transfer_encoding = 4, + + rb_compressed_gzip = 8 +}; + +int fd_read_body (const char *, int, FILE *, wgint, wgint, wgint *, wgint *, double *, int, FILE *); + +typedef const char *(*hunk_terminator_t) (const char *, const char *, int); + +char *fd_read_hunk (int, hunk_terminator_t, long, long); +char *fd_read_line (int); + +uerr_t retrieve_url (struct url *, const char *, char **, char **, + const char *, int *, bool, struct iri *, bool); +uerr_t retrieve_from_file (const char *, bool, int *); + +const char *retr_rate (wgint, double); +double calc_rate (wgint, double, int *); +void printwhat (int, int); + +void sleep_between_retrievals (int); + +void rotate_backups (const char *); + +bool url_uses_proxy (struct url *); + +void set_local_file (const char **, const char *); + +bool input_file_url (const char *); + +#endif /* RETR_H */ diff --git a/src/spider.c b/src/spider.c new file mode 100644 index 0000000..8727185 --- /dev/null +++ b/src/spider.c @@ -0,0 +1,101 @@ +/* Keep track of visited URLs in spider mode. + Copyright (C) 2006-2011, 2015, 2019-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <errno.h> +#include <assert.h> + +#include "spider.h" +#include "url.h" +#include "utils.h" +#include "hash.h" +#include "res.h" + + +static struct hash_table *nonexisting_urls_set; + +/* Cleanup the data structures associated with this file. */ + +#if defined DEBUG_MALLOC || defined TESTING +void +spider_cleanup (void) +{ + if (nonexisting_urls_set) + string_set_free (nonexisting_urls_set); +} +#endif + +/* Remembers broken links. */ +void +nonexisting_url (const char *url) +{ + /* Ignore robots.txt URLs */ + if (is_robots_txt_url (url)) + return; + if (!nonexisting_urls_set) + nonexisting_urls_set = make_string_hash_table (0); + string_set_add (nonexisting_urls_set, url); +} + +void +print_broken_links (void) +{ + hash_table_iterator iter; + int num_elems; + + if (!nonexisting_urls_set) + { + logprintf (LOG_NOTQUIET, _("Found no broken links.\n\n")); + return; + } + + num_elems = hash_table_count (nonexisting_urls_set); + assert (num_elems > 0); + + logprintf (LOG_NOTQUIET, ngettext("Found %d broken link.\n\n", + "Found %d broken links.\n\n", num_elems), + num_elems); + + for (hash_table_iterate (nonexisting_urls_set, &iter); + hash_table_iter_next (&iter); ) + { + /* Struct url_list *list; */ + const char *url = (const char *) iter.key; + + logprintf (LOG_NOTQUIET, _("%s\n"), url); + } + logputs (LOG_NOTQUIET, "\n"); +} + +/* + * vim: et ts=2 sw=2 + */ diff --git a/src/spider.h b/src/spider.h new file mode 100644 index 0000000..48e0ae8 --- /dev/null +++ b/src/spider.h @@ -0,0 +1,39 @@ +/* Declarations for spider.c + Copyright (C) 2006-2011, 2015, 2019-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef SPIDER_H +#define SPIDER_H + +#define visited_url(a,b) +void nonexisting_url (const char *); +void print_broken_links (void); +void spider_cleanup (void); + +#endif /* SPIDER_H */ diff --git a/src/ssl.h b/src/ssl.h new file mode 100644 index 0000000..e348252 --- /dev/null +++ b/src/ssl.h @@ -0,0 +1,40 @@ +/* SSL support. + Copyright (C) 2000-2012, 2015, 2018-2020 Free Software Foundation, + Inc. + Originally contributed by Christian Fraenkel. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef GEN_SSLFUNC_H +#define GEN_SSLFUNC_H + +bool ssl_init (void); +void ssl_cleanup (void); +bool ssl_connect_wget (int, const char *, int *); +bool ssl_check_certificate (int, const char *); + +#endif /* GEN_SSLFUNC_H */ diff --git a/src/sysdep.h b/src/sysdep.h new file mode 100644 index 0000000..f623e8d --- /dev/null +++ b/src/sysdep.h @@ -0,0 +1,59 @@ +/* Dirty system-dependent hacks. + Copyright (C) 1996-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +/* This file is included by wget.h. Random .c files need not include + it. */ + +#ifndef SYSDEP_H +#define SYSDEP_H + +/* Provided by gnulib on systems that don't have it: */ + +#include <sys/types.h> +#include <sys/stat.h> +#include <stdint.h> +#include <inttypes.h> + +#ifdef WINDOWS +/* Windows doesn't have some functions normally found on Unix-like + systems, such as strcasecmp, strptime, etc. Include mswindows.h so + we get the declarations for their replacements in mswindows.c, as + well as to pick up Windows-specific includes and constants. To be + able to test for such features, the file must be included as early + as possible. */ +# include "mswindows.h" +#endif + +#include <stdbool.h> +#include <limits.h> +#include <fnmatch.h> +#include "intprops.h" + +#endif /* SYSDEP_H */ diff --git a/src/url.c b/src/url.c new file mode 100644 index 0000000..950002b --- /dev/null +++ b/src/url.c @@ -0,0 +1,2540 @@ +/* URL handling. + Copyright (C) 1996-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> +#include <assert.h> + +#include "utils.h" +#include "url.h" +#include "host.h" /* for is_valid_ipv6_address */ +#include "c-strcase.h" + +#ifdef HAVE_ICONV +# include <iconv.h> +#endif +#include <langinfo.h> + +#ifdef __VMS +#include "vms.h" +#endif /* def __VMS */ + +#ifdef TESTING +#include "../tests/unit-tests.h" +#endif + +enum { + scm_disabled = 1, /* for https when OpenSSL fails to init. */ + scm_has_params = 2, /* whether scheme has ;params */ + scm_has_query = 4, /* whether scheme has ?query */ + scm_has_fragment = 8 /* whether scheme has #fragment */ +}; + +struct scheme_data +{ + /* Short name of the scheme, such as "http" or "ftp". */ + const char *name; + /* Leading string that identifies the scheme, such as "https://". */ + const char *leading_string; + /* Default port of the scheme when none is specified. */ + int default_port; + /* Various flags. */ + int flags; +}; + +/* Supported schemes: */ +static struct scheme_data supported_schemes[] = +{ + { "http", "http://", DEFAULT_HTTP_PORT, scm_has_query|scm_has_fragment }, +#ifdef HAVE_SSL + { "https", "https://", DEFAULT_HTTPS_PORT, scm_has_query|scm_has_fragment }, +#endif + { "ftp", "ftp://", DEFAULT_FTP_PORT, scm_has_params|scm_has_fragment }, +#ifdef HAVE_SSL + /* + * Explicit FTPS uses the same port as FTP. + * Implicit FTPS has its own port (990), but it is disabled by default. + */ + { "ftps", "ftps://", DEFAULT_FTP_PORT, scm_has_params|scm_has_fragment }, +#endif + + /* SCHEME_INVALID */ + { NULL, NULL, -1, 0 } +}; + +/* Forward declarations: */ + +static bool path_simplify (enum url_scheme, char *); + +/* Support for escaping and unescaping of URL strings. */ + +/* Table of "reserved" and "unsafe" characters. Those terms are + rfc1738-speak, as such largely obsoleted by rfc2396 and later + specs, but the general idea remains. + + A reserved character is the one that you can't decode without + changing the meaning of the URL. For example, you can't decode + "/foo/%2f/bar" into "/foo///bar" because the number and contents of + path components is different. Non-reserved characters can be + changed, so "/foo/%78/bar" is safe to change to "/foo/x/bar". The + unsafe characters are loosely based on rfc1738, plus "$" and ",", + as recommended by rfc2396, and minus "~", which is very frequently + used (and sometimes unrecognized as %7E by broken servers). + + An unsafe character is the one that should be encoded when URLs are + placed in foreign environments. E.g. space and newline are unsafe + in HTTP contexts because HTTP uses them as separator and line + terminator, so they must be encoded to %20 and %0A respectively. + "*" is unsafe in shell context, etc. + + We determine whether a character is unsafe through static table + lookup. This code assumes ASCII character set and 8-bit chars. */ + +enum { + /* rfc1738 reserved chars + "$" and ",". */ + urlchr_reserved = 1, + + /* rfc1738 unsafe chars, plus non-printables. */ + urlchr_unsafe = 2 +}; + +#define urlchr_test(c, mask) (urlchr_table[(unsigned char)(c)] & (mask)) +#define URL_RESERVED_CHAR(c) urlchr_test(c, urlchr_reserved) +#define URL_UNSAFE_CHAR(c) urlchr_test(c, urlchr_unsafe) + +/* Shorthands for the table: */ +#define R urlchr_reserved +#define U urlchr_unsafe +#define RU R|U + +static const unsigned char urlchr_table[256] = +{ + U, U, U, U, U, U, U, U, /* NUL SOH STX ETX EOT ENQ ACK BEL */ + U, U, U, U, U, U, U, U, /* BS HT LF VT FF CR SO SI */ + U, U, U, U, U, U, U, U, /* DLE DC1 DC2 DC3 DC4 NAK SYN ETB */ + U, U, U, U, U, U, U, U, /* CAN EM SUB ESC FS GS RS US */ + U, 0, U, RU, R, U, R, 0, /* SP ! " # $ % & ' */ + 0, 0, 0, R, R, 0, 0, R, /* ( ) * + , - . / */ + 0, 0, 0, 0, 0, 0, 0, 0, /* 0 1 2 3 4 5 6 7 */ + 0, 0, RU, R, U, R, U, R, /* 8 9 : ; < = > ? */ + RU, 0, 0, 0, 0, 0, 0, 0, /* @ A B C D E F G */ + 0, 0, 0, 0, 0, 0, 0, 0, /* H I J K L M N O */ + 0, 0, 0, 0, 0, 0, 0, 0, /* P Q R S T U V W */ + 0, 0, 0, RU, U, RU, U, 0, /* X Y Z [ \ ] ^ _ */ + U, 0, 0, 0, 0, 0, 0, 0, /* ` a b c d e f g */ + 0, 0, 0, 0, 0, 0, 0, 0, /* h i j k l m n o */ + 0, 0, 0, 0, 0, 0, 0, 0, /* p q r s t u v w */ + 0, 0, 0, U, U, U, 0, U, /* x y z { | } ~ DEL */ + + U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, + U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, + U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, + U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, + + U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, + U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, + U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, + U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, U, +}; +#undef R +#undef U +#undef RU + +static void +url_unescape_1 (char *s, unsigned char mask) +{ + unsigned char *t = (unsigned char *) s; /* t - tortoise */ + unsigned char *h = (unsigned char *) s; /* h - hare */ + + for (; *h; h++, t++) + { + if (*h != '%') + { + copychar: + *t = *h; + } + else + { + unsigned char c; + /* Do nothing if '%' is not followed by two hex digits. */ + if (!h[1] || !h[2] || !(c_isxdigit (h[1]) && c_isxdigit (h[2]))) + goto copychar; + c = X2DIGITS_TO_NUM (h[1], h[2]); + if (urlchr_test(c, mask)) + goto copychar; + /* Don't unescape %00 because there is no way to insert it + into a C string without effectively truncating it. */ + if (c == '\0') + goto copychar; + *t = c; + h += 2; + } + } + *t = '\0'; +} + +/* URL-unescape the string S. + + This is done by transforming the sequences "%HH" to the character + represented by the hexadecimal digits HH. If % is not followed by + two hexadecimal digits, it is inserted literally. + + The transformation is done in place. If you need the original + string intact, make a copy before calling this function. */ +void +url_unescape (char *s) +{ + url_unescape_1 (s, 0); +} + +/* URL-unescape the string S. + + This functions behaves identically as url_unescape(), but does not + convert characters from "reserved". In other words, it only converts + "unsafe" characters. */ +void +url_unescape_except_reserved (char *s) +{ + url_unescape_1 (s, urlchr_reserved); +} + +/* The core of url_escape_* functions. Escapes the characters that + match the provided mask in urlchr_table. + + If ALLOW_PASSTHROUGH is true, a string with no unsafe chars will be + returned unchanged. If ALLOW_PASSTHROUGH is false, a freshly + allocated string will be returned in all cases. */ + +static char * +url_escape_1 (const char *s, unsigned char mask, bool allow_passthrough) +{ + const char *p1; + char *p2, *newstr; + int newlen; + int addition = 0; + + for (p1 = s; *p1; p1++) + if (urlchr_test (*p1, mask)) + addition += 2; /* Two more characters (hex digits) */ + + if (!addition) + return allow_passthrough ? (char *)s : xstrdup (s); + + newlen = (p1 - s) + addition; + newstr = xmalloc (newlen + 1); + + p1 = s; + p2 = newstr; + while (*p1) + { + /* Quote the characters that match the test mask. */ + if (urlchr_test (*p1, mask)) + { + unsigned char c = *p1++; + *p2++ = '%'; + *p2++ = XNUM_TO_DIGIT (c >> 4); + *p2++ = XNUM_TO_DIGIT (c & 0xf); + } + else + *p2++ = *p1++; + } + assert (p2 - newstr == newlen); + *p2 = '\0'; + + return newstr; +} + +/* URL-escape the unsafe characters (see urlchr_table) in a given + string, returning a freshly allocated string. */ + +char * +url_escape (const char *s) +{ + return url_escape_1 (s, urlchr_unsafe, false); +} + +/* URL-escape the unsafe and reserved characters (see urlchr_table) in + a given string, returning a freshly allocated string. */ + +char * +url_escape_unsafe_and_reserved (const char *s) +{ + return url_escape_1 (s, urlchr_unsafe|urlchr_reserved, false); +} + +/* URL-escape the unsafe characters (see urlchr_table) in a given + string. If no characters are unsafe, S is returned. */ + +static char * +url_escape_allow_passthrough (const char *s) +{ + return url_escape_1 (s, urlchr_unsafe, true); +} + +/* Decide whether the char at position P needs to be encoded. (It is + not enough to pass a single char *P because the function may need + to inspect the surrounding context.) + + Return true if the char should be escaped as %XX, false otherwise. */ + +static inline bool +char_needs_escaping (const char *p) +{ + if (*p == '%') + { + if (c_isxdigit (*(p + 1)) && c_isxdigit (*(p + 2))) + return false; + else + /* Garbled %.. sequence: encode `%'. */ + return true; + } + else if (URL_UNSAFE_CHAR (*p) && !URL_RESERVED_CHAR (*p)) + return true; + else + return false; +} + +/* Translate a %-escaped (but possibly non-conformant) input string S + into a %-escaped (and conformant) output string. If no characters + are encoded or decoded, return the same string S; otherwise, return + a freshly allocated string with the new contents. + + After a URL has been run through this function, the protocols that + use `%' as the quote character can use the resulting string as-is, + while those that don't can use url_unescape to get to the intended + data. This function is stable: once the input is transformed, + further transformations of the result yield the same output. + + Let's discuss why this function is needed. + + Imagine Wget is asked to retrieve `http://abc.xyz/abc def'. Since + a raw space character would mess up the HTTP request, it needs to + be quoted, like this: + + GET /abc%20def HTTP/1.0 + + It would appear that the unsafe chars need to be quoted, for + example with url_escape. But what if we're requested to download + `abc%20def'? url_escape transforms "%" to "%25", which would leave + us with `abc%2520def'. This is incorrect -- since %-escapes are + part of URL syntax, "%20" is the correct way to denote a literal + space on the Wget command line. This leads to the conclusion that + in that case Wget should not call url_escape, but leave the `%20' + as is. This is clearly contradictory, but it only gets worse. + + What if the requested URI is `abc%20 def'? If we call url_escape, + we end up with `/abc%2520%20def', which is almost certainly not + intended. If we don't call url_escape, we are left with the + embedded space and cannot complete the request. What the user + meant was for Wget to request `/abc%20%20def', and this is where + reencode_escapes kicks in. + + Wget used to solve this by first decoding %-quotes, and then + encoding all the "unsafe" characters found in the resulting string. + This was wrong because it didn't preserve certain URL special + (reserved) characters. For instance, URI containing "a%2B+b" (0x2b + == '+') would get translated to "a%2B%2Bb" or "a++b" depending on + whether we considered `+' reserved (it is). One of these results + is inevitable because by the second step we would lose information + on whether the `+' was originally encoded or not. Both results + were wrong because in CGI parameters + means space, while %2B means + literal plus. reencode_escapes correctly translates the above to + "a%2B+b", i.e. returns the original string. + + This function uses a modified version of the algorithm originally + proposed by Anon Sricharoenchai: + + * Encode all "unsafe" characters, except those that are also + "reserved", to %XX. See urlchr_table for which characters are + unsafe and reserved. + + * Encode the "%" characters not followed by two hex digits to + "%25". + + * Pass through all other characters and %XX escapes as-is. (Up to + Wget 1.10 this decoded %XX escapes corresponding to "safe" + characters, but that was obtrusive and broke some servers.) + + Anon's test case: + + "http://abc.xyz/%20%3F%%36%31%25aa% a?a=%61+a%2Ba&b=b%26c%3Dc" + -> + "http://abc.xyz/%20%3F%25%36%31%25aa%25%20a?a=%61+a%2Ba&b=b%26c%3Dc" + + Simpler test cases: + + "foo bar" -> "foo%20bar" + "foo%20bar" -> "foo%20bar" + "foo %20bar" -> "foo%20%20bar" + "foo%%20bar" -> "foo%25%20bar" (0x25 == '%') + "foo%25%20bar" -> "foo%25%20bar" + "foo%2%20bar" -> "foo%252%20bar" + "foo+bar" -> "foo+bar" (plus is reserved!) + "foo%2b+bar" -> "foo%2b+bar" */ + +static char * +reencode_escapes (const char *s) +{ + const char *p1; + char *newstr, *p2; + int oldlen, newlen; + + int encode_count = 0; + + /* First pass: inspect the string to see if there's anything to do, + and to calculate the new length. */ + for (p1 = s; *p1; p1++) + if (char_needs_escaping (p1)) + ++encode_count; + + if (!encode_count) + /* The string is good as it is. */ + return (char *) s; /* C const model sucks. */ + + oldlen = p1 - s; + /* Each encoding adds two characters (hex digits). */ + newlen = oldlen + 2 * encode_count; + newstr = xmalloc (newlen + 1); + + /* Second pass: copy the string to the destination address, encoding + chars when needed. */ + p1 = s; + p2 = newstr; + + while (*p1) + if (char_needs_escaping (p1)) + { + unsigned char c = *p1++; + *p2++ = '%'; + *p2++ = XNUM_TO_DIGIT (c >> 4); + *p2++ = XNUM_TO_DIGIT (c & 0xf); + } + else + *p2++ = *p1++; + + *p2 = '\0'; + assert (p2 - newstr == newlen); + return newstr; +} + +/* Returns the scheme type if the scheme is supported, or + SCHEME_INVALID if not. */ + +enum url_scheme +url_scheme (const char *url) +{ + int i; + + for (i = 0; supported_schemes[i].leading_string; i++) + if (0 == c_strncasecmp (url, supported_schemes[i].leading_string, + strlen (supported_schemes[i].leading_string))) + { + if (!(supported_schemes[i].flags & scm_disabled)) + return (enum url_scheme) i; + else + return SCHEME_INVALID; + } + + return SCHEME_INVALID; +} + +#define SCHEME_CHAR(ch) (c_isalnum (ch) || (ch) == '-' || (ch) == '+') + +/* Return 1 if the URL begins with any "scheme", 0 otherwise. As + currently implemented, it returns true if URL begins with + [-+a-zA-Z0-9]+: . */ + +bool +url_has_scheme (const char *url) +{ + const char *p = url; + + /* The first char must be a scheme char. */ + if (!*p || !SCHEME_CHAR (*p)) + return false; + ++p; + /* Followed by 0 or more scheme chars. */ + while (*p && SCHEME_CHAR (*p)) + ++p; + /* Terminated by ':'. */ + return *p == ':'; +} + +bool +url_valid_scheme (const char *url) +{ + enum url_scheme scheme = url_scheme (url); + return scheme != SCHEME_INVALID; +} + +int +scheme_default_port (enum url_scheme scheme) +{ + return supported_schemes[scheme].default_port; +} + +void +scheme_disable (enum url_scheme scheme) +{ + supported_schemes[scheme].flags |= scm_disabled; +} + +const char * +scheme_leading_string (enum url_scheme scheme) +{ + return supported_schemes[scheme].leading_string; +} + +/* Skip the username and password, if present in the URL. The + function should *not* be called with the complete URL, but with the + portion after the scheme. + + If no username and password are found, return URL. */ + +static const char * +url_skip_credentials (const char *url) +{ + /* Look for '@' that comes before terminators, such as '/', '?', + '#', or ';'. */ + const char *p = (const char *)strpbrk (url, "@/?#;"); + if (!p || *p != '@') + return url; + return p + 1; +} + +/* Parse credentials contained in [BEG, END). The region is expected + to have come from a URL and is unescaped. */ + +static bool +parse_credentials (const char *beg, const char *end, char **user, char **passwd) +{ + char *colon; + const char *userend; + + if (beg == end) + return false; /* empty user name */ + + colon = memchr (beg, ':', end - beg); + if (colon == beg) + return false; /* again empty user name */ + + if (colon) + { + *passwd = strdupdelim (colon + 1, end); + userend = colon; + url_unescape (*passwd); + } + else + { + *passwd = NULL; + userend = end; + } + *user = strdupdelim (beg, userend); + url_unescape (*user); + return true; +} + +/* Used by main.c: detect URLs written using the "shorthand" URL forms + originally popularized by Netscape and NcFTP. HTTP shorthands look + like this: + + www.foo.com[:port]/dir/file -> http://www.foo.com[:port]/dir/file + www.foo.com[:port] -> http://www.foo.com[:port] + + FTP shorthands look like this: + + foo.bar.com:dir/file -> ftp://foo.bar.com/dir/file + foo.bar.com:/absdir/file -> ftp://foo.bar.com//absdir/file + + If the URL needs not or cannot be rewritten, return NULL. */ + +char * +rewrite_shorthand_url (const char *url) +{ + const char *p; + char *ret; + + if (url_scheme (url) != SCHEME_INVALID) + return NULL; + + /* Look for a ':' or '/'. The former signifies NcFTP syntax, the + latter Netscape. */ + p = strpbrk (url, ":/"); + if (p == url) + return NULL; + + /* If we're looking at "://", it means the URL uses a scheme we + don't support, which may include "https" when compiled without + SSL support. Don't bogusly rewrite such URLs. */ + if (p && p[0] == ':' && p[1] == '/' && p[2] == '/') + return NULL; + + if (p && *p == ':') + { + /* Colon indicates ftp, as in foo.bar.com:path. Check for + special case of http port number ("localhost:10000"). */ + int digits = strspn (p + 1, "0123456789"); + if (digits && (p[1 + digits] == '/' || p[1 + digits] == '\0')) + goto http; + + /* Turn "foo.bar.com:path" to "ftp://foo.bar.com/path". */ + if ((ret = aprintf ("ftp://%s", url)) != NULL) + ret[6 + (p - url)] = '/'; + } + else + { + http: + /* Just prepend "http://" to URL. */ + ret = aprintf ("http://%s", url); + } + return ret; +} + +static void split_path (const char *, char **, char **); + +/* Like strpbrk, with the exception that it returns the pointer to the + terminating zero (end-of-string aka "eos") if no matching character + is found. */ + +static inline char * +strpbrk_or_eos (const char *s, const char *accept) +{ + char *p = strpbrk (s, accept); + if (!p) + p = strchr (s, '\0'); + return p; +} + +/* Turn STR into lowercase; return true if a character was actually + changed. */ + +static bool +lowercase_str (char *str) +{ + bool changed = false; + for (; *str; str++) + if (c_isupper (*str)) + { + changed = true; + *str = c_tolower (*str); + } + return changed; +} + +static const char * +init_seps (enum url_scheme scheme) +{ + static char seps[8] = ":/"; + char *p = seps + 2; + int flags = supported_schemes[scheme].flags; + + if (flags & scm_has_params) + *p++ = ';'; + if (flags & scm_has_query) + *p++ = '?'; + if (flags & scm_has_fragment) + *p++ = '#'; + *p = '\0'; + return seps; +} + +static const char *parse_errors[] = { +#define PE_NO_ERROR 0 + N_("No error"), +#define PE_UNSUPPORTED_SCHEME 1 + N_("Unsupported scheme %s"), /* support for format token only here */ +#define PE_MISSING_SCHEME 2 + N_("Scheme missing"), +#define PE_INVALID_HOST_NAME 3 + N_("Invalid host name"), +#define PE_BAD_PORT_NUMBER 4 + N_("Bad port number"), +#define PE_INVALID_USER_NAME 5 + N_("Invalid user name"), +#define PE_UNTERMINATED_IPV6_ADDRESS 6 + N_("Unterminated IPv6 numeric address"), +#define PE_IPV6_NOT_SUPPORTED 7 + N_("IPv6 addresses not supported"), +#define PE_INVALID_IPV6_ADDRESS 8 + N_("Invalid IPv6 numeric address") +}; + +/* Parse a URL. + + Return a new struct url if successful, NULL on error. In case of + error, and if ERROR is not NULL, also set *ERROR to the appropriate + error code. */ +struct url * +url_parse (const char *url, int *error, struct iri *iri, bool percent_encode) +{ + struct url *u; + const char *p; + bool path_modified, host_modified; + + enum url_scheme scheme; + const char *seps; + + const char *uname_b, *uname_e; + const char *host_b, *host_e; + const char *path_b, *path_e; + const char *params_b, *params_e; + const char *query_b, *query_e; + const char *fragment_b, *fragment_e; + + int port; + char *user = NULL, *passwd = NULL; + + const char *url_encoded = NULL; + + int error_code; + + scheme = url_scheme (url); + if (scheme == SCHEME_INVALID) + { + if (url_has_scheme (url)) + error_code = PE_UNSUPPORTED_SCHEME; + else + error_code = PE_MISSING_SCHEME; + goto error; + } + + url_encoded = url; + + if (iri && iri->utf8_encode) + { + char *new_url = NULL; + + iri->utf8_encode = remote_to_utf8 (iri, iri->orig_url ? iri->orig_url : url, &new_url); + if (!iri->utf8_encode) + new_url = NULL; + else + { + xfree (iri->orig_url); + iri->orig_url = xstrdup (url); + url_encoded = reencode_escapes (new_url); + if (url_encoded != new_url) + xfree (new_url); + percent_encode = false; + } + } + + if (percent_encode) + url_encoded = reencode_escapes (url); + + p = url_encoded; + p += strlen (supported_schemes[scheme].leading_string); + uname_b = p; + p = url_skip_credentials (p); + uname_e = p; + + /* scheme://user:pass@host[:port]... */ + /* ^ */ + + /* We attempt to break down the URL into the components path, + params, query, and fragment. They are ordered like this: + + scheme://host[:port][/path][;params][?query][#fragment] */ + + path_b = path_e = NULL; + params_b = params_e = NULL; + query_b = query_e = NULL; + fragment_b = fragment_e = NULL; + + /* Initialize separators for optional parts of URL, depending on the + scheme. For example, FTP has params, and HTTP and HTTPS have + query string and fragment. */ + seps = init_seps (scheme); + + host_b = p; + + if (*p == '[') + { + /* Handle IPv6 address inside square brackets. Ideally we'd + just look for the terminating ']', but rfc2732 mandates + rejecting invalid IPv6 addresses. */ + + /* The address begins after '['. */ + host_b = p + 1; + host_e = strchr (host_b, ']'); + + if (!host_e) + { + error_code = PE_UNTERMINATED_IPV6_ADDRESS; + goto error; + } + +#ifdef ENABLE_IPV6 + /* Check if the IPv6 address is valid. */ + if (!is_valid_ipv6_address(host_b, host_e)) + { + error_code = PE_INVALID_IPV6_ADDRESS; + goto error; + } + + /* Continue parsing after the closing ']'. */ + p = host_e + 1; +#else + error_code = PE_IPV6_NOT_SUPPORTED; + goto error; +#endif + + /* The closing bracket must be followed by a separator or by the + null char. */ + /* http://[::1]... */ + /* ^ */ + if (!strchr (seps, *p)) + { + /* Trailing garbage after []-delimited IPv6 address. */ + error_code = PE_INVALID_HOST_NAME; + goto error; + } + } + else + { + p = strpbrk_or_eos (p, seps); + host_e = p; + } + ++seps; /* advance to '/' */ + + if (host_b == host_e) + { + error_code = PE_INVALID_HOST_NAME; + goto error; + } + + port = scheme_default_port (scheme); + if (*p == ':') + { + const char *port_b, *port_e, *pp; + + /* scheme://host:port/tralala */ + /* ^ */ + ++p; + port_b = p; + p = strpbrk_or_eos (p, seps); + port_e = p; + + /* Allow empty port, as per rfc2396. */ + if (port_b != port_e) + for (port = 0, pp = port_b; pp < port_e; pp++) + { + if (!c_isdigit (*pp)) + { + /* http://host:12randomgarbage/blah */ + /* ^ */ + error_code = PE_BAD_PORT_NUMBER; + goto error; + } + port = 10 * port + (*pp - '0'); + /* Check for too large port numbers here, before we have + a chance to overflow on bogus port values. */ + if (port > 0xffff) + { + error_code = PE_BAD_PORT_NUMBER; + goto error; + } + } + } + /* Advance to the first separator *after* '/' (either ';' or '?', + depending on the scheme). */ + ++seps; + + /* Get the optional parts of URL, each part being delimited by + current location and the position of the next separator. */ +#define GET_URL_PART(sepchar, var) do { \ + if (*p == sepchar) \ + var##_b = ++p, var##_e = p = strpbrk_or_eos (p, seps); \ + ++seps; \ +} while (0) + + GET_URL_PART ('/', path); + if (supported_schemes[scheme].flags & scm_has_params) + GET_URL_PART (';', params); + if (supported_schemes[scheme].flags & scm_has_query) + GET_URL_PART ('?', query); + if (supported_schemes[scheme].flags & scm_has_fragment) + GET_URL_PART ('#', fragment); + +#undef GET_URL_PART + assert (*p == 0); + + if (uname_b != uname_e) + { + /* http://user:pass@host */ + /* ^ ^ */ + /* uname_b uname_e */ + if (!parse_credentials (uname_b, uname_e - 1, &user, &passwd)) + { + error_code = PE_INVALID_USER_NAME; + goto error; + } + } + + u = xnew0 (struct url); + u->scheme = scheme; + u->host = strdupdelim (host_b, host_e); + u->port = port; + u->user = user; + u->passwd = passwd; + + u->path = strdupdelim (path_b, path_e); + path_modified = path_simplify (scheme, u->path); + split_path (u->path, &u->dir, &u->file); + + host_modified = lowercase_str (u->host); + + /* Decode %HH sequences in host name. This is important not so much + to support %HH sequences in host names (which other browser + don't), but to support binary characters (which will have been + converted to %HH by reencode_escapes). */ + if (strchr (u->host, '%')) + { + url_unescape (u->host); + host_modified = true; + + /* check for invalid control characters in host name */ + for (p = u->host; *p; p++) + { + if (c_iscntrl(*p)) + { + url_free(u); + error_code = PE_INVALID_HOST_NAME; + goto error; + } + } + + /* Apply IDNA regardless of iri->utf8_encode status */ + if (opt.enable_iri && iri) + { + char *new = idn_encode (iri, u->host); + if (new) + { + xfree (u->host); + u->host = new; + host_modified = true; + } + } + } + + if (params_b) + u->params = strdupdelim (params_b, params_e); + if (query_b) + u->query = strdupdelim (query_b, query_e); + if (fragment_b) + u->fragment = strdupdelim (fragment_b, fragment_e); + + if (opt.enable_iri || path_modified || u->fragment || host_modified || path_b == path_e) + { + /* If we suspect that a transformation has rendered what + url_string might return different from URL_ENCODED, rebuild + u->url using url_string. */ + u->url = url_string (u, URL_AUTH_SHOW); + + if (url_encoded != url) + xfree (url_encoded); + } + else + { + if (url_encoded == url) + u->url = xstrdup (url); + else + u->url = (char *) url_encoded; + } + + return u; + + error: + /* Cleanup in case of error: */ + if (url_encoded && url_encoded != url) + xfree (url_encoded); + + /* Transmit the error code to the caller, if the caller wants to + know. */ + if (error) + *error = error_code; + return NULL; +} + +/* Return the error message string from ERROR_CODE, which should have + been retrieved from url_parse. The error message is translated. */ + +char * +url_error (const char *url, int error_code) +{ + assert (error_code >= 0 && ((size_t) error_code) < countof (parse_errors)); + + if (error_code == PE_UNSUPPORTED_SCHEME) + { + char *error, *p; + char *scheme = xstrdup (url); + assert (url_has_scheme (url)); + + if ((p = strchr (scheme, ':'))) + *p = '\0'; + if (!c_strcasecmp (scheme, "https")) + error = aprintf (_("HTTPS support not compiled in")); + else + error = aprintf (_(parse_errors[error_code]), quote (scheme)); + xfree (scheme); + + return error; + } + else + return xstrdup (_(parse_errors[error_code])); +} + +/* Split PATH into DIR and FILE. PATH comes from the URL and is + expected to be URL-escaped. + + The path is split into directory (the part up to the last slash) + and file (the part after the last slash), which are subsequently + unescaped. Examples: + + PATH DIR FILE + "foo/bar/baz" "foo/bar" "baz" + "foo/bar/" "foo/bar" "" + "foo" "" "foo" + "foo/bar/baz%2fqux" "foo/bar" "baz/qux" (!) + + DIR and FILE are freshly allocated. */ + +static void +split_path (const char *path, char **dir, char **file) +{ + char *last_slash = strrchr (path, '/'); + if (!last_slash) + { + *dir = xstrdup (""); + *file = xstrdup (path); + } + else + { + *dir = strdupdelim (path, last_slash); + *file = xstrdup (last_slash + 1); + } + url_unescape (*dir); + url_unescape (*file); +} + +/* Note: URL's "full path" is the path with the query string and + params appended. The "fragment" (#foo) is intentionally ignored, + but that might be changed. For example, if the original URL was + "http://host:port/foo/bar/baz;bullshit?querystring#uselessfragment", + the full path will be "/foo/bar/baz;bullshit?querystring". */ + +/* Return the length of the full path, without the terminating + zero. */ + +static int +full_path_length (const struct url *url) +{ + int len = 0; + +#define FROB(el) if (url->el) len += 1 + strlen (url->el) + + FROB (path); + FROB (params); + FROB (query); + +#undef FROB + + return len; +} + +/* Write out the full path. */ + +static void +full_path_write (const struct url *url, char *where) +{ +#define FROB(el, chr) do { \ + char *f_el = url->el; \ + if (f_el) { \ + int l = strlen (f_el); \ + *where++ = chr; \ + memcpy (where, f_el, l); \ + where += l; \ + } \ +} while (0) + + FROB (path, '/'); + FROB (params, ';'); + FROB (query, '?'); + +#undef FROB +} + +/* Public function for getting the "full path". E.g. if u->path is + "foo/bar" and u->query is "param=value", full_path will be + "/foo/bar?param=value". */ + +char * +url_full_path (const struct url *url) +{ + int length = full_path_length (url); + char *full_path = xmalloc (length + 1); + + full_path_write (url, full_path); + full_path[length] = '\0'; + + return full_path; +} + +/* Unescape CHR in an otherwise escaped STR. Used to selectively + escaping of certain characters, such as "/" and ":". Returns a + count of unescaped chars. */ + +static void +unescape_single_char (char *str, char chr) +{ + const char c1 = XNUM_TO_DIGIT (chr >> 4); + const char c2 = XNUM_TO_DIGIT (chr & 0xf); + char *h = str; /* hare */ + char *t = str; /* tortoise */ + for (; *h; h++, t++) + { + if (h[0] == '%' && h[1] == c1 && h[2] == c2) + { + *t = chr; + h += 2; + } + else + *t = *h; + } + *t = '\0'; +} + +/* Escape unsafe and reserved characters, except for the slash + characters. */ + +static char * +url_escape_dir (const char *dir) +{ + char *newdir = url_escape_1 (dir, urlchr_unsafe | urlchr_reserved, 1); + if (newdir == dir) + return (char *)dir; + + unescape_single_char (newdir, '/'); + return newdir; +} + +/* Sync u->path and u->url with u->dir and u->file. Called after + u->file or u->dir have been changed, typically by the FTP code. */ + +static void +sync_path (struct url *u) +{ + char *newpath, *efile, *edir; + + xfree (u->path); + + /* u->dir and u->file are not escaped. URL-escape them before + reassembling them into u->path. That way, if they contain + separators like '?' or even if u->file contains slashes, the + path will be correctly assembled. (u->file can contain slashes + if the URL specifies it with %2f, or if an FTP server returns + it.) */ + edir = url_escape_dir (u->dir); + efile = url_escape_1 (u->file, urlchr_unsafe | urlchr_reserved, 1); + + if (!*edir) + newpath = xstrdup (efile); + else + { + int dirlen = strlen (edir); + int filelen = strlen (efile); + + /* Copy "DIR/FILE" to newpath. */ + char *p = newpath = xmalloc (dirlen + 1 + filelen + 1); + memcpy (p, edir, dirlen); + p += dirlen; + *p++ = '/'; + memcpy (p, efile, filelen); + p += filelen; + *p = '\0'; + } + + u->path = newpath; + + if (edir != u->dir) + xfree (edir); + if (efile != u->file) + xfree (efile); + + /* Regenerate u->url as well. */ + xfree (u->url); + u->url = url_string (u, URL_AUTH_SHOW); +} + +/* Mutators. Code in ftp.c insists on changing u->dir and u->file. + This way we can sync u->path and u->url when they get changed. */ + +void +url_set_dir (struct url *url, const char *newdir) +{ + xfree (url->dir); + url->dir = xstrdup (newdir); + sync_path (url); +} + +void +url_set_file (struct url *url, const char *newfile) +{ + xfree (url->file); + url->file = xstrdup (newfile); + sync_path (url); +} + +void +url_free (struct url *url) +{ + if (url) + { + xfree (url->host); + + xfree (url->path); + xfree (url->url); + + xfree (url->params); + xfree (url->query); + xfree (url->fragment); + xfree (url->user); + xfree (url->passwd); + + xfree (url->dir); + xfree (url->file); + + xfree (url); + } +} + +/* Create all the necessary directories for PATH (a file). Calls + make_directory internally. */ +int +mkalldirs (const char *path) +{ + const char *p; + char *t; + struct stat st; + int res; + + p = strrchr(path, '/'); + p = p == NULL ? path : p; + + /* Don't create if it's just a file. */ + if ((p == path) && (*p != '/')) + return 0; + t = strdupdelim (path, p); + + /* Check whether the directory exists. */ + if ((stat (t, &st) == 0)) + { + if (S_ISDIR (st.st_mode)) + { + xfree (t); + return 0; + } + else + { + /* If the dir exists as a file name, remove it first. This + is *only* for Wget to work with buggy old CERN http + servers. Here is the scenario: When Wget tries to + retrieve a directory without a slash, e.g. + http://foo/bar (bar being a directory), CERN server will + not redirect it too http://foo/bar/ -- it will generate a + directory listing containing links to bar/file1, + bar/file2, etc. Wget will lose because it saves this + HTML listing to a file `bar', so it cannot create the + directory. To work around this, if the file of the same + name exists, we just remove it and create the directory + anyway. */ + DEBUGP (("Removing %s because of directory danger!\n", t)); + if (unlink (t)) + logprintf (LOG_NOTQUIET, "Failed to unlink %s (%d): %s\n", + t, errno, strerror(errno)); + } + } + res = make_directory (t); + if (res != 0) + logprintf (LOG_NOTQUIET, "%s: %s\n", t, strerror (errno)); + xfree (t); + return res; +} + +/* Functions for constructing the file name out of URL components. */ + +/* A growable string structure, used by url_file_name and friends. + This should perhaps be moved to utils.c. + + The idea is to have a convenient and efficient way to construct a + string by having various functions append data to it. Instead of + passing the obligatory BASEVAR, SIZEVAR and TAILPOS to all the + functions in questions, we pass the pointer to this struct. + + Functions that write to the members in this struct must make sure + that base remains null terminated by calling append_null(). + */ + +struct growable { + char *base; + int size; /* memory allocated */ + int tail; /* string length */ +}; + +/* Ensure that the string can accept APPEND_COUNT more characters past + the current TAIL position. If necessary, this will grow the string + and update its allocated size. If the string is already large + enough to take TAIL+APPEND_COUNT characters, this does nothing. */ +#define GROW(g, append_size) do { \ + struct growable *G_ = g; \ + DO_REALLOC (G_->base, G_->size, G_->tail + append_size, char); \ +} while (0) + +/* Return the tail position of the string. */ +#define TAIL(r) ((r)->base + (r)->tail) + +/* Move the tail position by APPEND_COUNT characters. */ +#define TAIL_INCR(r, append_count) ((r)->tail += append_count) + + +/* Append NULL to DEST. */ +static void +append_null (struct growable *dest) +{ + GROW (dest, 1); + *TAIL (dest) = 0; +} + +/* Append CH to DEST. */ +static void +append_char (char ch, struct growable *dest) +{ + if (ch) + { + GROW (dest, 1); + *TAIL (dest) = ch; + TAIL_INCR (dest, 1); + } + + append_null (dest); +} + +/* Append the string STR to DEST. */ +static void +append_string (const char *str, struct growable *dest) +{ + int l = strlen (str); + + if (l) + { + GROW (dest, l); + memcpy (TAIL (dest), str, l); + TAIL_INCR (dest, l); + } + + append_null (dest); +} + + +enum { + filechr_not_unix = 1, /* unusable on Unix, / and \0 */ + filechr_not_vms = 2, /* unusable on VMS (ODS5), 0x00-0x1F * ? */ + filechr_not_windows = 4, /* unusable on Windows, one of \|/<>?:*" */ + filechr_control = 8 /* a control character, e.g. 0-31 */ +}; + +#define FILE_CHAR_TEST(c, mask) \ + ((opt.restrict_files_nonascii && !c_isascii ((unsigned char)(c))) || \ + (filechr_table[(unsigned char)(c)] & (mask))) + +/* Shorthands for the table: */ +#define U filechr_not_unix +#define V filechr_not_vms +#define W filechr_not_windows +#define C filechr_control + +#define UVWC U|V|W|C +#define UW U|W +#define VC V|C +#define VW V|W + +/* Table of characters unsafe under various conditions (see above). + + Arguably we could also claim `%' to be unsafe, since we use it as + the escape character. If we ever want to be able to reliably + translate file name back to URL, this would become important + crucial. Right now, it's better to be minimal in escaping. */ + +static const unsigned char filechr_table[256] = +{ +UVWC, VC, VC, VC, VC, VC, VC, VC, /* NUL SOH STX ETX EOT ENQ ACK BEL */ + VC, VC, VC, VC, VC, VC, VC, VC, /* BS HT LF VT FF CR SO SI */ + VC, VC, VC, VC, VC, VC, VC, VC, /* DLE DC1 DC2 DC3 DC4 NAK SYN ETB */ + VC, VC, VC, VC, VC, VC, VC, VC, /* CAN EM SUB ESC FS GS RS US */ + 0, 0, W, 0, 0, 0, 0, 0, /* SP ! " # $ % & ' */ + 0, 0, VW, 0, 0, 0, 0, UW, /* ( ) * + , - . / */ + 0, 0, 0, 0, 0, 0, 0, 0, /* 0 1 2 3 4 5 6 7 */ + 0, 0, W, 0, W, 0, W, VW, /* 8 9 : ; < = > ? */ + 0, 0, 0, 0, 0, 0, 0, 0, /* @ A B C D E F G */ + 0, 0, 0, 0, 0, 0, 0, 0, /* H I J K L M N O */ + 0, 0, 0, 0, 0, 0, 0, 0, /* P Q R S T U V W */ + 0, 0, 0, 0, W, 0, 0, 0, /* X Y Z [ \ ] ^ _ */ + 0, 0, 0, 0, 0, 0, 0, 0, /* ` a b c d e f g */ + 0, 0, 0, 0, 0, 0, 0, 0, /* h i j k l m n o */ + 0, 0, 0, 0, 0, 0, 0, 0, /* p q r s t u v w */ + 0, 0, 0, 0, W, 0, 0, C, /* x y z { | } ~ DEL */ + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 128-143 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 144-159 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#undef U +#undef V +#undef W +#undef C +#undef UW +#undef UVWC +#undef VC +#undef VW + +/* FN_PORT_SEP is the separator between host and port in file names + for non-standard port numbers. On Unix this is normally ':', as in + "www.xemacs.org:4001/index.html". Under Windows, we set it to + + because Windows can't handle ':' in file names. */ +#define FN_PORT_SEP (opt.restrict_files_os != restrict_windows ? ':' : '+') + +/* FN_QUERY_SEP is the separator between the file name and the URL + query, normally '?'. Because VMS and Windows cannot handle '?' in a + file name, we use '@' instead there. */ +#define FN_QUERY_SEP \ + (((opt.restrict_files_os != restrict_vms) && \ + (opt.restrict_files_os != restrict_windows)) ? '?' : '@') +#define FN_QUERY_SEP_STR \ + (((opt.restrict_files_os != restrict_vms) && \ + (opt.restrict_files_os != restrict_windows)) ? "?" : "@") + +/* Quote path element, characters in [b, e), as file name, and append + the quoted string to DEST. Each character is quoted as per + file_unsafe_char and the corresponding table. + + If ESCAPED is true, the path element is considered to be + URL-escaped and will be unescaped prior to inspection. */ + +static void +append_uri_pathel (const char *b, const char *e, bool escaped, + struct growable *dest) +{ + const char *p; + char buf[1024]; + char *unescaped = NULL; + int quoted, outlen; + int mask; + + if (!dest) + return; + + if (opt.restrict_files_os == restrict_unix) + mask = filechr_not_unix; + else if (opt.restrict_files_os == restrict_vms) + mask = filechr_not_vms; + else + mask = filechr_not_windows; + + if (opt.restrict_files_ctrl) + mask |= filechr_control; + + /* Copy [b, e) to PATHEL and URL-unescape it. */ + if (escaped) + { + size_t len = e - b; + if (len < sizeof (buf)) + unescaped = buf; + else + unescaped = xmalloc(len + 1); + + memcpy(unescaped, b, len); + unescaped[len] = 0; + + url_unescape (unescaped); + b = unescaped; + e = unescaped + strlen (unescaped); + } + + /* Defang ".." when found as component of path. Remember that path + comes from the URL and might contain malicious input. */ + if (e - b == 2 && b[0] == '.' && b[1] == '.') + { + b = "%2E%2E"; + e = b + 6; + } + + /* Walk the PATHEL string and check how many characters we'll need + to quote. */ + quoted = 0; + for (p = b; p < e; p++) + if (FILE_CHAR_TEST (*p, mask)) + ++quoted; + + /* Calculate the length of the output string. e-b is the input + string length. Each quoted char introduces two additional + characters in the string, hence 2*quoted. */ + outlen = (e - b) + (2 * quoted); + GROW (dest, outlen); + + if (!quoted) + { + /* If there's nothing to quote, we can simply append the string + without processing it again. */ + memcpy (TAIL (dest), b, outlen); + } + else + { + char *q = TAIL (dest); + for (p = b; p < e; p++) + { + if (!FILE_CHAR_TEST (*p, mask)) + *q++ = *p; + else + { + unsigned char ch = *p; + *q++ = '%'; + *q++ = XNUM_TO_DIGIT (ch >> 4); + *q++ = XNUM_TO_DIGIT (ch & 0xf); + } + } + assert (q - TAIL (dest) == outlen); + } + + /* Perform inline case transformation if required. */ + if (opt.restrict_files_case == restrict_lowercase + || opt.restrict_files_case == restrict_uppercase) + { + char *q; + for (q = TAIL (dest); q < TAIL (dest) + outlen; ++q) + { + if (opt.restrict_files_case == restrict_lowercase) + *q = c_tolower (*q); + else + *q = c_toupper (*q); + } + } + + TAIL_INCR (dest, outlen); + append_null (dest); + + if (unescaped && unescaped != buf) + free (unescaped); +} + +#ifdef HAVE_ICONV +static char * +convert_fname (char *fname) +{ + char *converted_fname; + const char *from_encoding = opt.encoding_remote; + const char *to_encoding = opt.locale; + iconv_t cd; + size_t len, done, inlen, outlen; + char *s; + const char *orig_fname; + + /* Defaults for remote and local encodings. */ + if (!from_encoding) + from_encoding = "UTF-8"; + if (!to_encoding) + to_encoding = nl_langinfo (CODESET); + + cd = iconv_open (to_encoding, from_encoding); + if (cd == (iconv_t) (-1)) + { + logprintf (LOG_VERBOSE, _ ("Conversion from %s to %s isn't supported\n"), + quote (from_encoding), quote (to_encoding)); + return fname; + } + + orig_fname = fname; + inlen = strlen (fname); + len = outlen = inlen * 2; + converted_fname = s = xmalloc (outlen + 1); + done = 0; + + for (;;) + { + errno = 0; + if (iconv (cd, (ICONV_CONST char **) &fname, &inlen, &s, &outlen) == 0 + && iconv (cd, NULL, NULL, &s, &outlen) == 0) + { + *(converted_fname + len - outlen - done) = '\0'; + iconv_close (cd); + DEBUGP (("Converted file name '%s' (%s) -> '%s' (%s)\n", + orig_fname, from_encoding, converted_fname, to_encoding)); + xfree (orig_fname); + return converted_fname; + } + + /* Incomplete or invalid multibyte sequence */ + if (errno == EINVAL || errno == EILSEQ || errno == 0) + { + if (errno) + logprintf (LOG_VERBOSE, + _ ("Incomplete or invalid multibyte sequence encountered\n")); + else + logprintf (LOG_VERBOSE, + _ ("Unconvertable multibyte sequence encountered\n")); + xfree (converted_fname); + converted_fname = (char *) orig_fname; + break; + } + else if (errno == E2BIG) /* Output buffer full */ + { + done = len; + len = outlen = done + inlen * 2; + converted_fname = xrealloc (converted_fname, outlen + 1); + s = converted_fname + done; + } + else /* Weird, we got an unspecified error */ + { + logprintf (LOG_VERBOSE, _ ("Unhandled errno %d\n"), errno); + xfree (converted_fname); + converted_fname = (char *) orig_fname; + break; + } + } + DEBUGP (("Failed to convert file name '%s' (%s) -> '?' (%s)\n", + orig_fname, from_encoding, to_encoding)); + + iconv_close (cd); + + return converted_fname; +} +#else +static char * +convert_fname (char *fname) +{ + return fname; +} +#endif + +/* Append to DEST the directory structure that corresponds the + directory part of URL's path. For example, if the URL is + http://server/dir1/dir2/file, this appends "/dir1/dir2". + + Each path element ("dir1" and "dir2" in the above example) is + examined, url-unescaped, and re-escaped as file name element. + + Additionally, it cuts as many directories from the path as + specified by opt.cut_dirs. For example, if opt.cut_dirs is 1, it + will produce "bar" for the above example. For 2 or more, it will + produce "". + + Each component of the path is quoted for use as file name. */ + +static void +append_dir_structure (const struct url *u, struct growable *dest) +{ + char *pathel, *next; + int cut = opt.cut_dirs; + + /* Go through the path components, de-URL-quote them, and quote them + (if necessary) as file names. */ + + pathel = u->path; + for (; (next = strchr (pathel, '/')) != NULL; pathel = next + 1) + { + if (cut-- > 0) + continue; + if (pathel == next) + /* Ignore empty pathels. */ + continue; + + if (dest->tail) + append_char ('/', dest); + append_uri_pathel (pathel, next, true, dest); + } +} + +/* Return a unique file name that matches the given URL as well as + possible. Does not create directories on the file system. */ + +char * +url_file_name (const struct url *u, char *replaced_filename) +{ + struct growable fnres; /* stands for "file name result" */ + struct growable temp_fnres; + + const char *u_file; + char *fname, *unique, *fname_len_check; + const char *index_filename = "index.html"; /* The default index file is index.html */ + size_t max_length; + + fnres.base = NULL; + fnres.size = 0; + fnres.tail = 0; + + temp_fnres.base = NULL; + temp_fnres.size = 0; + temp_fnres.tail = 0; + + /* If an alternative index file was defined, change index_filename */ + if (opt.default_page) + index_filename = opt.default_page; + + + /* Start with the directory prefix, if specified. */ + if (opt.dir_prefix) + append_string (opt.dir_prefix, &fnres); + + /* If "dirstruct" is turned on (typically the case with -r), add + the host and port (unless those have been turned off) and + directory structure. */ + /* All safe remote chars are unescaped and stored in temp_fnres, + then converted to local and appended to fnres. + Internationalized URL/IDN will produce punycode to lookup IP from DNS: + https://en.wikipedia.org/wiki/URL + https://en.wikipedia.org/wiki/Internationalized_domain_name + Non-ASCII code chars in the path: + https://en.wikipedia.org/wiki/List_of_Unicode_characters + https://en.wikipedia.org/wiki/List_of_writing_systems */ + if (opt.dirstruct) + { + if (opt.protocol_directories) + { + if (temp_fnres.tail) + append_char ('/', &temp_fnres); + append_string (supported_schemes[u->scheme].name, &temp_fnres); + } + if (opt.add_hostdir) + { + if (temp_fnres.tail) + append_char ('/', &temp_fnres); + if (0 != strcmp (u->host, "..")) + append_string (u->host, &temp_fnres); + else + /* Host name can come from the network; malicious DNS may + allow ".." to be resolved, causing us to write to + "../<file>". Defang such host names. */ + append_string ("%2E%2E", &temp_fnres); + if (u->port != scheme_default_port (u->scheme)) + { + char portstr[24]; + number_to_string (portstr, u->port); + append_char (FN_PORT_SEP, &temp_fnres); + append_string (portstr, &temp_fnres); + } + } + + append_dir_structure (u, &temp_fnres); + } + + if (!replaced_filename) + { + /* Create the filename. */ + u_file = *u->file ? u->file : index_filename; + + /* Append "?query" to the file name, even if empty, + * and create fname_len_check. */ + if (u->query) + fname_len_check = concat_strings (u_file, FN_QUERY_SEP_STR, u->query, NULL); + else + fname_len_check = strdupdelim (u_file, u_file + strlen (u_file)); + } + else + { + u_file = replaced_filename; + fname_len_check = strdupdelim (u_file, u_file + strlen (u_file)); + } + + if (temp_fnres.tail) + append_char ('/', &temp_fnres); + + append_uri_pathel (fname_len_check, + fname_len_check + strlen (fname_len_check), true, &temp_fnres); + + /* Zero-terminate the temporary file name. */ + append_char ('\0', &temp_fnres); + + /* convert all remote chars before length check and appending to local path */ + fname = convert_fname (temp_fnres.base); + temp_fnres.base = NULL; + temp_fnres.size = 0; + temp_fnres.tail = 0; + append_string (fname, &temp_fnres); + xfree (fname); + + /* Check that the length of the file name is acceptable. */ +#ifdef WINDOWS + if (MAX_PATH > (fnres.tail + CHOMP_BUFFER + 2)) + { + max_length = MAX_PATH - (fnres.tail + CHOMP_BUFFER + 2); + /* FIXME: In Windows a filename is usually limited to 255 characters. + To really be accurate you could call GetVolumeInformation() to get + lpMaximumComponentLength + */ + if (max_length > 255) + { + max_length = 255; + } + } + else + { + max_length = 0; + } +#else + max_length = get_max_length (fnres.base, fnres.tail, _PC_NAME_MAX) - CHOMP_BUFFER; +#endif + if (max_length > 0 && strlen (temp_fnres.base) > max_length) + { + logprintf (LOG_NOTQUIET, "The name is too long, %lu chars total.\n", + (unsigned long) strlen (temp_fnres.base)); + logprintf (LOG_NOTQUIET, "Trying to shorten...\n"); + + /* Shorten the file name. */ + temp_fnres.base[max_length] = '\0'; + + logprintf (LOG_NOTQUIET, "New name is %s.\n", temp_fnres.base); + } + + xfree (fname_len_check); + + /* The filename has already been 'cleaned' by append_uri_pathel() above. So, + * just append it. */ + if (fnres.tail) + append_char ('/', &fnres); + append_string (temp_fnres.base, &fnres); + + fname = fnres.base; + + /* Make a final check that the path length is acceptable? */ + /* TODO: check fnres.base for path length problem */ + + xfree (temp_fnres.base); + + /* Check the cases in which the unique extensions are not used: + 1) Clobbering is turned off (-nc). + 2) Retrieval with regetting. + 3) Timestamping is used. + 4) Hierarchy is built. + 5) Backups are specified. + + The exception is the case when file does exist and is a + directory (see `mkalldirs' for explanation). */ + + if (ALLOW_CLOBBER + && !(file_exists_p (fname, NULL) && !file_non_directory_p (fname))) + { + unique = fname; + } + else + { + unique = unique_name_passthrough (fname); + if (unique != fname) + xfree (fname); + } + +/* On VMS, alter the name as required. */ +#ifdef __VMS + { + char *unique2; + + unique2 = ods_conform( unique); + if (unique2 != unique) + { + xfree (unique); + unique = unique2; + } + } +#endif /* def __VMS */ + + return unique; +} + +/* Resolve "." and ".." elements of PATH by destructively modifying + PATH and return true if PATH has been modified, false otherwise. + + The algorithm is in spirit similar to the one described in rfc1808, + although implemented differently, in one pass. To recap, path + elements containing only "." are removed, and ".." is taken to mean + "back up one element". Single leading and trailing slashes are + preserved. + + For example, "a/b/c/./../d/.." will yield "a/b/". More exhaustive + test examples are provided below. If you change anything in this + function, run test_path_simplify to make sure you haven't broken a + test case. */ + +static bool +path_simplify (enum url_scheme scheme, char *path) +{ + char *h = path; /* hare */ + char *t = path; /* tortoise */ + char *beg = path; + char *end = strchr (path, '\0'); + + while (h < end) + { + /* Hare should be at the beginning of a path element. */ + + if (h[0] == '.' && (h[1] == '/' || h[1] == '\0')) + { + /* Ignore "./". */ + h += 2; + } + else if (h[0] == '.' && h[1] == '.' && (h[2] == '/' || h[2] == '\0')) + { + /* Handle "../" by retreating the tortoise by one path + element -- but not past beginning. */ + if (t > beg) + { + /* Move backwards until T hits the beginning of the + previous path element or the beginning of path. */ + for (--t; t > beg && t[-1] != '/'; t--) + ; + } + else if (scheme == SCHEME_FTP +#ifdef HAVE_SSL + || scheme == SCHEME_FTPS +#endif + ) + { + /* If we're at the beginning, copy the "../" literally + and move the beginning so a later ".." doesn't remove + it. This violates RFC 3986; but we do it for FTP + anyway because there is otherwise no way to get at a + parent directory, when the FTP server drops us in a + non-root directory (which is not uncommon). */ + beg = t + 3; + goto regular; + } + h += 3; + } + else + { + regular: + /* A regular path element. If H hasn't advanced past T, + simply skip to the next path element. Otherwise, copy + the path element until the next slash. */ + if (t == h) + { + /* Skip the path element, including the slash. */ + while (h < end && *h != '/') + t++, h++; + if (h < end) + t++, h++; + } + else + { + /* Copy the path element, including the final slash. */ + while (h < end && *h != '/') + *t++ = *h++; + if (h < end) + *t++ = *h++; + } + } + } + + if (t != h) + *t = '\0'; + + return t != h; +} + +/* Return the length of URL's path. Path is considered to be + terminated by one or more of the ?query or ;params or #fragment, + depending on the scheme. */ + +static const char * +path_end (const char *url) +{ + enum url_scheme scheme = url_scheme (url); + const char *seps; + if (scheme == SCHEME_INVALID) + scheme = SCHEME_HTTP; /* use http semantics for rel links */ + /* +2 to ignore the first two separators ':' and '/' */ + seps = init_seps (scheme) + 2; + return strpbrk_or_eos (url, seps); +} + +/* Find the last occurrence of character C in the range [b, e), or + NULL, if none are present. */ +#define find_last_char(b, e, c) memrchr ((b), (c), (e) - (b)) + +/* Merge BASE with LINK and return the resulting URI. + + Either of the URIs may be absolute or relative, complete with the + host name, or path only. This tries to reasonably handle all + foreseeable cases. It only employs minimal URL parsing, without + knowledge of the specifics of schemes. + + I briefly considered making this function call path_simplify after + the merging process, as rfc1738 seems to suggest. This is a bad + idea for several reasons: 1) it complexifies the code, and 2) + url_parse has to simplify path anyway, so it's wasteful to boot. */ + +char * +uri_merge (const char *base, const char *link) +{ + int linklength; + const char *end; + char *merge; + + if (url_has_scheme (link)) + return xstrdup (link); + + /* We may not examine BASE past END. */ + end = path_end (base); + linklength = strlen (link); + + if (!*link) + { + /* Empty LINK points back to BASE, query string and all. */ + return xstrdup (base); + } + else if (*link == '?') + { + /* LINK points to the same location, but changes the query + string. Examples: */ + /* uri_merge("path", "?new") -> "path?new" */ + /* uri_merge("path?foo", "?new") -> "path?new" */ + /* uri_merge("path?foo#bar", "?new") -> "path?new" */ + /* uri_merge("path#foo", "?new") -> "path?new" */ + int baselength = end - base; + merge = xmalloc (baselength + linklength + 1); + memcpy (merge, base, baselength); + memcpy (merge + baselength, link, linklength); + merge[baselength + linklength] = '\0'; + } + else if (*link == '#') + { + /* uri_merge("path", "#new") -> "path#new" */ + /* uri_merge("path#foo", "#new") -> "path#new" */ + /* uri_merge("path?foo", "#new") -> "path?foo#new" */ + /* uri_merge("path?foo#bar", "#new") -> "path?foo#new" */ + int baselength; + const char *end1 = strchr (base, '#'); + if (!end1) + end1 = base + strlen (base); + baselength = end1 - base; + merge = xmalloc (baselength + linklength + 1); + memcpy (merge, base, baselength); + memcpy (merge + baselength, link, linklength); + merge[baselength + linklength] = '\0'; + } + else if (*link == '/' && *(link + 1) == '/') + { + /* LINK begins with "//" and so is a net path: we need to + replace everything after (and including) the double slash + with LINK. */ + + /* uri_merge("foo", "//new/bar") -> "//new/bar" */ + /* uri_merge("//old/foo", "//new/bar") -> "//new/bar" */ + /* uri_merge("http://old/foo", "//new/bar") -> "http://new/bar" */ + + int span; + const char *slash; + const char *start_insert; + + /* Look for first slash. */ + slash = memchr (base, '/', end - base); + /* If found slash and it is a double slash, then replace + from this point, else default to replacing from the + beginning. */ + if (slash && *(slash + 1) == '/') + start_insert = slash; + else + start_insert = base; + + span = start_insert - base; + merge = xmalloc (span + linklength + 1); + if (span) + memcpy (merge, base, span); + memcpy (merge + span, link, linklength); + merge[span + linklength] = '\0'; + } + else if (*link == '/') + { + /* LINK is an absolute path: we need to replace everything + after (and including) the FIRST slash with LINK. + + So, if BASE is "http://host/whatever/foo/bar", and LINK is + "/qux/xyzzy", our result should be + "http://host/qux/xyzzy". */ + int span; + const char *slash; + const char *start_insert = NULL; /* for gcc to shut up. */ + const char *pos = base; + bool seen_slash_slash = false; + /* We're looking for the first slash, but want to ignore + double slash. */ + again: + slash = memchr (pos, '/', end - pos); + if (slash && !seen_slash_slash) + if (*(slash + 1) == '/') + { + pos = slash + 2; + seen_slash_slash = true; + goto again; + } + + /* At this point, SLASH is the location of the first / after + "//", or the first slash altogether. START_INSERT is the + pointer to the location where LINK will be inserted. When + examining the last two examples, keep in mind that LINK + begins with '/'. */ + + if (!slash && !seen_slash_slash) + /* example: "foo" */ + /* ^ */ + start_insert = base; + else if (!slash && seen_slash_slash) + /* example: "http://foo" */ + /* ^ */ + start_insert = end; + else if (slash && !seen_slash_slash) + /* example: "foo/bar" */ + /* ^ */ + start_insert = base; + else if (slash && seen_slash_slash) + /* example: "http://something/" */ + /* ^ */ + start_insert = slash; + + span = start_insert - base; + merge = xmalloc (span + linklength + 1); + if (span) + memcpy (merge, base, span); + memcpy (merge + span, link, linklength); + merge[span + linklength] = '\0'; + } + else + { + /* LINK is a relative URL: we need to replace everything + after last slash (possibly empty) with LINK. + + So, if BASE is "whatever/foo/bar", and LINK is "qux/xyzzy", + our result should be "whatever/foo/qux/xyzzy". */ + bool need_explicit_slash = false; + int span; + const char *start_insert; + const char *last_slash = find_last_char (base, end, '/'); + if (!last_slash) + { + /* No slash found at all. Replace what we have with LINK. */ + start_insert = base; + } + else if (last_slash && last_slash >= base + 2 + && last_slash[-2] == ':' && last_slash[-1] == '/') + { + /* example: http://host" */ + /* ^ */ + start_insert = end + 1; + need_explicit_slash = true; + } + else + { + /* example: "whatever/foo/bar" */ + /* ^ */ + start_insert = last_slash + 1; + } + + span = start_insert - base; + merge = xmalloc (span + linklength + 1); + if (span) + memcpy (merge, base, span); + if (need_explicit_slash) + merge[span - 1] = '/'; + memcpy (merge + span, link, linklength); + merge[span + linklength] = '\0'; + } + + return merge; +} + +#define APPEND(p, s) do { \ + int len = strlen (s); \ + memcpy (p, s, len); \ + p += len; \ +} while (0) + +/* Use this instead of password when the actual password is supposed + to be hidden. We intentionally use a generic string without giving + away the number of characters in the password, like previous + versions did. */ +#define HIDDEN_PASSWORD "*password*" + +/* Recreate the URL string from the data in URL. + + If HIDE is true (as it is when we're calling this on a URL we plan + to print, but not when calling it to canonicalize a URL for use + within the program), password will be hidden. Unsafe characters in + the URL will be quoted. */ + +char * +url_string (const struct url *url, enum url_auth_mode auth_mode) +{ + int size; + char *result, *p; + char *quoted_host, *quoted_user = NULL, *quoted_passwd = NULL; + + int scheme_port = supported_schemes[url->scheme].default_port; + const char *scheme_str = supported_schemes[url->scheme].leading_string; + int fplen = full_path_length (url); + + bool brackets_around_host; + + assert (scheme_str != NULL); + + /* Make sure the user name and password are quoted. */ + if (url->user) + { + if (auth_mode != URL_AUTH_HIDE) + { + quoted_user = url_escape_allow_passthrough (url->user); + if (url->passwd) + { + if (auth_mode == URL_AUTH_HIDE_PASSWD) + quoted_passwd = (char *) HIDDEN_PASSWORD; + else + quoted_passwd = url_escape_allow_passthrough (url->passwd); + } + } + } + + /* In the unlikely event that the host name contains non-printable + characters, quote it for displaying to the user. */ + quoted_host = url_escape_allow_passthrough (url->host); + + /* Undo the quoting of colons that URL escaping performs. IPv6 + addresses may legally contain colons, and in that case must be + placed in square brackets. */ + if (quoted_host != url->host) + unescape_single_char (quoted_host, ':'); + brackets_around_host = strchr (quoted_host, ':') != NULL; + + size = (strlen (scheme_str) + + strlen (quoted_host) + + (brackets_around_host ? 2 : 0) + + fplen + + 1); + if (url->port != scheme_port) + size += 1 + numdigit (url->port); + if (quoted_user) + { + size += 1 + strlen (quoted_user); + if (quoted_passwd) + size += 1 + strlen (quoted_passwd); + } + + p = result = xmalloc (size); + + APPEND (p, scheme_str); + if (quoted_user) + { + APPEND (p, quoted_user); + if (quoted_passwd) + { + *p++ = ':'; + APPEND (p, quoted_passwd); + } + *p++ = '@'; + } + + if (brackets_around_host) + *p++ = '['; + APPEND (p, quoted_host); + if (brackets_around_host) + *p++ = ']'; + if (url->port != scheme_port) + { + *p++ = ':'; + p = number_to_string (p, url->port); + } + + full_path_write (url, p); + p += fplen; + *p++ = '\0'; + + assert (p - result == size); + + if (quoted_user && quoted_user != url->user) + xfree (quoted_user); + if (quoted_passwd && auth_mode == URL_AUTH_SHOW + && quoted_passwd != url->passwd) + xfree (quoted_passwd); + if (quoted_host != url->host) + xfree (quoted_host); + + return result; +} + +/* Return true if scheme a is similar to scheme b. + + Schemes are similar if they are equal. If SSL is supported, schemes + are also similar if one is http (SCHEME_HTTP) and the other is https + (SCHEME_HTTPS). */ +bool +schemes_are_similar_p (enum url_scheme a, enum url_scheme b) +{ + if (a == b) + return true; +#ifdef HAVE_SSL + if ((a == SCHEME_HTTP && b == SCHEME_HTTPS) + || (a == SCHEME_HTTPS && b == SCHEME_HTTP)) + return true; +#endif + return false; +} + +static int +getchar_from_escaped_string (const char *str, char *c) +{ + const char *p = str; + + assert (str && *str); + assert (c); + + if (p[0] == '%') + { + if (!c_isxdigit(p[1]) || !c_isxdigit(p[2])) + { + *c = '%'; + return 1; + } + else + { + if (p[2] == 0) + return 0; /* error: invalid string */ + + *c = X2DIGITS_TO_NUM (p[1], p[2]); + if (URL_RESERVED_CHAR(*c)) + { + *c = '%'; + return 1; + } + else + return 3; + } + } + else + { + *c = p[0]; + } + + return 1; +} + +bool +are_urls_equal (const char *u1, const char *u2) +{ + const char *p, *q; + int pp, qq; + char ch1, ch2; + assert(u1 && u2); + + p = u1; + q = u2; + + while (*p && *q + && (pp = getchar_from_escaped_string (p, &ch1)) + && (qq = getchar_from_escaped_string (q, &ch2)) + && (c_tolower(ch1) == c_tolower(ch2))) + { + p += pp; + q += qq; + } + + return (*p == 0 && *q == 0 ? true : false); +} + +#ifdef TESTING +/* Debugging and testing support for path_simplify. */ + +#if 0 +/* Debug: run path_simplify on PATH and return the result in a new + string. Useful for calling from the debugger. */ +static char * +ps (char *path) +{ + char *copy = xstrdup (path); + path_simplify (copy); + return copy; +} +#endif + +static const char * +run_test (const char *test, const char *expected_result, enum url_scheme scheme, + bool expected_change) +{ + char *test_copy = xstrdup (test); + bool modified = path_simplify (scheme, test_copy); + + if (0 != strcmp (test_copy, expected_result)) + { + printf ("Failed path_simplify(\"%s\"): expected \"%s\", got \"%s\".\n", + test, expected_result, test_copy); + mu_assert ("", 0); + } + if (modified != expected_change) + { + if (expected_change) + printf ("Expected modification with path_simplify(\"%s\").\n", + test); + else + printf ("Expected no modification with path_simplify(\"%s\").\n", + test); + } + xfree (test_copy); + mu_assert ("", modified == expected_change); + return NULL; +} + +const char * +test_path_simplify (void) +{ + static const struct { + const char *test, *result; + enum url_scheme scheme; + bool should_modify; + } tests[] = { + { "", "", SCHEME_HTTP, false }, + { ".", "", SCHEME_HTTP, true }, + { "./", "", SCHEME_HTTP, true }, + { "..", "", SCHEME_HTTP, true }, + { "../", "", SCHEME_HTTP, true }, + { "..", "..", SCHEME_FTP, false }, + { "../", "../", SCHEME_FTP, false }, + { "foo", "foo", SCHEME_HTTP, false }, + { "foo/bar", "foo/bar", SCHEME_HTTP, false }, + { "foo///bar", "foo///bar", SCHEME_HTTP, false }, + { "foo/.", "foo/", SCHEME_HTTP, true }, + { "foo/./", "foo/", SCHEME_HTTP, true }, + { "foo./", "foo./", SCHEME_HTTP, false }, + { "foo/../bar", "bar", SCHEME_HTTP, true }, + { "foo/../bar/", "bar/", SCHEME_HTTP, true }, + { "foo/bar/..", "foo/", SCHEME_HTTP, true }, + { "foo/bar/../x", "foo/x", SCHEME_HTTP, true }, + { "foo/bar/../x/", "foo/x/", SCHEME_HTTP, true }, + { "foo/..", "", SCHEME_HTTP, true }, + { "foo/../..", "", SCHEME_HTTP, true }, + { "foo/../../..", "", SCHEME_HTTP, true }, + { "foo/../../bar/../../baz", "baz", SCHEME_HTTP, true }, + { "foo/../..", "..", SCHEME_FTP, true }, + { "foo/../../..", "../..", SCHEME_FTP, true }, + { "foo/../../bar/../../baz", "../../baz", SCHEME_FTP, true }, + { "a/b/../../c", "c", SCHEME_HTTP, true }, + { "./a/../b", "b", SCHEME_HTTP, true } + }; + unsigned i; + + for (i = 0; i < countof (tests); i++) + { + const char *message; + const char *test = tests[i].test; + const char *expected_result = tests[i].result; + enum url_scheme scheme = tests[i].scheme; + bool expected_change = tests[i].should_modify; + + message = run_test (test, expected_result, scheme, expected_change); + if (message) return message; + } + return NULL; +} + +const char * +test_append_uri_pathel(void) +{ + unsigned i; + static const struct { + const char *original_url; + const char *input; + bool escaped; + const char *expected_result; + } test_array[] = { + { "http://www.yoyodyne.com/path/", "somepage.html", false, "http://www.yoyodyne.com/path/somepage.html" }, + }; + + for (i = 0; i < countof(test_array); ++i) + { + struct growable dest; + const char *p = test_array[i].input; + + memset (&dest, 0, sizeof (dest)); + + append_string (test_array[i].original_url, &dest); + append_uri_pathel (p, p + strlen(p), test_array[i].escaped, &dest); + + mu_assert ("test_append_uri_pathel: wrong result", + strcmp (dest.base, test_array[i].expected_result) == 0); + xfree (dest.base); + } + + return NULL; +} + +const char * +test_are_urls_equal(void) +{ + unsigned i; + static const struct { + const char *url1; + const char *url2; + bool expected_result; + } test_array[] = { + { "http://www.adomain.com/apath/", "http://www.adomain.com/apath/", true }, + { "http://www.adomain.com/apath/", "http://www.adomain.com/anotherpath/", false }, + { "http://www.adomain.com/apath/", "http://www.anotherdomain.com/path/", false }, + { "http://www.adomain.com/~path/", "http://www.adomain.com/%7epath/", true }, + { "http://www.adomain.com/longer-path/", "http://www.adomain.com/path/", false }, + { "http://www.adomain.com/path%2f", "http://www.adomain.com/path/", false }, + }; + + for (i = 0; i < countof(test_array); ++i) + { + mu_assert ("test_are_urls_equal: wrong result", + are_urls_equal (test_array[i].url1, test_array[i].url2) == test_array[i].expected_result); + } + + return NULL; +} + +#endif /* TESTING */ + +/* + * vim: et ts=2 sw=2 + */ diff --git a/src/url.h b/src/url.h new file mode 100644 index 0000000..9313c2f --- /dev/null +++ b/src/url.h @@ -0,0 +1,136 @@ +/* Declarations for url.c. + Copyright (C) 1996-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef URL_H +#define URL_H + +/* Default port definitions */ +#define DEFAULT_HTTP_PORT 80 +#define DEFAULT_FTP_PORT 21 +#define DEFAULT_HTTPS_PORT 443 +#define DEFAULT_FTPS_IMPLICIT_PORT 990 + +/* This represents how many characters less than the OS max name length a file + * should be. More precisely, a file name should be at most + * (NAME_MAX - CHOMP_BUFFER) characters in length. This number was arrived at + * by adding the lengths of all possible strings that could be appended to a + * file name later in the code (e.g. ".orig", ".html", etc.). This is + * hopefully plenty of extra characters, but I am not guaranteeing that a file + * name will be of the proper length by the time the code wants to open a + * file descriptor. */ +#define CHOMP_BUFFER 19 + +/* The flags that allow clobbering the file (opening with "wb"). + Defined here to avoid repetition later. #### This will require + rework. */ +#define ALLOW_CLOBBER (opt.noclobber || opt.always_rest || opt.timestamping \ + || opt.dirstruct || opt.output_document || opt.backups > 0) + +/* Specifies how, or whether, user auth information should be included + * in URLs regenerated from URL parse structures. */ +enum url_auth_mode { + URL_AUTH_SHOW, + URL_AUTH_HIDE_PASSWD, + URL_AUTH_HIDE +}; + +/* Note: the ordering here is related to the order of elements in + `supported_schemes' in url.c. */ + +enum url_scheme { + SCHEME_HTTP, +#ifdef HAVE_SSL + SCHEME_HTTPS, +#endif + SCHEME_FTP, +#ifdef HAVE_SSL + SCHEME_FTPS, +#endif + SCHEME_INVALID +}; + +/* Structure containing info on a URL. */ +struct url +{ + char *url; /* Original URL */ + enum url_scheme scheme; /* URL scheme */ + + char *host; /* Extracted hostname */ + int port; /* Port number */ + + /* URL components (URL-quoted). */ + char *path; + char *params; + char *query; + char *fragment; + + /* Extracted path info (unquoted). */ + char *dir; + char *file; + + /* Username and password (unquoted). */ + char *user; + char *passwd; +}; + +/* Function declarations */ + +char *url_escape (const char *); +char *url_escape_unsafe_and_reserved (const char *); +void url_unescape (char *); +void url_unescape_except_reserved (char *); + +struct url *url_parse (const char *, int *, struct iri *iri, bool percent_encode); +char *url_error (const char *, int); +char *url_full_path (const struct url *); +void url_set_dir (struct url *, const char *); +void url_set_file (struct url *, const char *); +void url_free (struct url *); + +enum url_scheme url_scheme (const char *); +bool url_has_scheme (const char *); +bool url_valid_scheme (const char *); +int scheme_default_port (enum url_scheme); +void scheme_disable (enum url_scheme); +const char *scheme_leading_string (enum url_scheme); + +char *url_string (const struct url *, enum url_auth_mode); +char *url_file_name (const struct url *, char *); + +char *uri_merge (const char *, const char *); + +int mkalldirs (const char *); + +char *rewrite_shorthand_url (const char *); +bool schemes_are_similar_p (enum url_scheme a, enum url_scheme b); + +bool are_urls_equal (const char *u1, const char *u2); + +#endif /* URL_H */ diff --git a/src/utils.c b/src/utils.c new file mode 100644 index 0000000..89a8687 --- /dev/null +++ b/src/utils.c @@ -0,0 +1,2993 @@ +/* Various utility functions. + Copyright (C) 1996-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" + +#include "sha256.h" +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> +#ifdef HAVE_PROCESS_H +# include <process.h> /* getpid() */ +#endif +#include <errno.h> +#include <fcntl.h> +#include <assert.h> +#include <stdarg.h> +#include <locale.h> +#include <errno.h> + +#if HAVE_UTIME +# include <sys/types.h> +# include <utime.h> +# ifdef HAVE_SYS_UTIME_H +# include <sys/utime.h> +# endif +#endif + +#include <sys/time.h> + +#include <sys/stat.h> + +/* For TIOCGWINSZ and friends: */ +#ifndef WINDOWS +# include <sys/ioctl.h> +# include <termios.h> +#endif + +/* Needed for Unix version of run_with_timeout. */ +#include <signal.h> +#include <setjmp.h> + +#include <regex.h> +#ifdef HAVE_LIBPCRE2 +# define PCRE2_CODE_UNIT_WIDTH 8 +# include <pcre2.h> +#elif defined HAVE_LIBPCRE +# include <pcre.h> +#endif + +#ifndef HAVE_SIGSETJMP +/* If sigsetjmp is a macro, configure won't pick it up. */ +# ifdef sigsetjmp +# define HAVE_SIGSETJMP +# endif +#endif + +#if defined HAVE_SIGSETJMP || defined HAVE_SIGBLOCK +# define USE_SIGNAL_TIMEOUT +#endif + +/* Some systems (Linux libc5, "NCR MP-RAS 3.0", and others) don't + provide MAP_FAILED, a symbolic constant for the value returned by + mmap() when it doesn't work. Usually, this constant should be -1. + This only makes sense for files that use mmap() and include + sys/mman.h *before* sysdep.h, but doesn't hurt others. */ +#ifdef HAVE_MMAP +# include <sys/mman.h> +# ifndef MAP_FAILED +# define MAP_FAILED ((void *) -1) +# endif +#endif + +#include "utils.h" +#include "hash.h" + +#ifdef __VMS +#include "vms.h" +#endif /* def __VMS */ + +#ifdef TESTING +#include "../tests/unit-tests.h" +#endif + +#include "exits.h" +#include "c-strcase.h" + +_Noreturn static void +memfatal (const char *context, long attempted_size) +{ + /* Make sure we don't try to store part of the log line, and thus + call malloc. */ + log_set_save_context (false); + + /* We have different log outputs in different situations: + 1) output without bytes information + 2) output with bytes information */ + if (attempted_size == UNKNOWN_ATTEMPTED_SIZE) + { + logprintf (LOG_ALWAYS, + _("%s: %s: Failed to allocate enough memory; memory exhausted.\n"), + exec_name, context); + } + else + { + logprintf (LOG_ALWAYS, + _("%s: %s: Failed to allocate %ld bytes; memory exhausted.\n"), + exec_name, context, attempted_size); + } + + exit (WGET_EXIT_GENERIC_ERROR); +} + +/* Character property table for (re-)escaping VMS ODS5 extended file + names. Note that this table ignores Unicode. + + ODS2 valid characters: 0-9 A-Z a-z $ - _ ~ + + ODS5 Invalid characters: + C0 control codes (0x00 to 0x1F inclusive) + Asterisk (*) + Question mark (?) + + ODS5 Invalid characters only in VMS V7.2 (which no one runs, right?): + Double quotation marks (") + Backslash (\) + Colon (:) + Left angle bracket (<) + Right angle bracket (>) + Slash (/) + Vertical bar (|) + + Characters escaped by "^": + SP ! " # % & ' ( ) + , . : ; = + @ [ \ ] ^ ` { | } ~ + + Either "^_" or "^ " is accepted as a space. Period (.) is a special + case. Note that un-escaped < and > can also confuse a directory + spec. + + Characters put out as ^xx: + 7F (DEL) + 80-9F (C1 control characters) + A0 (nonbreaking space) + FF (Latin small letter y diaeresis) + + Other cases: + Unicode: "^Uxxxx", where "xxxx" is four hex digits. + + Property table values: + Normal escape: 1 + Space: 2 + Dot: 4 + Hex-hex escape: 8 + ODS2 normal: 16 + ODS2 lower case: 32 + Hex digit: 64 +*/ + +unsigned char char_prop[ 256] = { + +/* NUL SOH STX ETX EOT ENQ ACK BEL BS HT LF VT FF CR SO SI */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + +/* DLE DC1 DC2 DC3 DC4 NAK SYN ETB CAN EM SUB ESC FS GS RS US */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + +/* SP ! " # $ % & ' ( ) * + , - . / */ + 2, 1, 1, 1, 16, 1, 1, 1, 1, 1, 0, 1, 1, 16, 4, 0, + +/* 0 1 2 3 4 5 6 7 8 9 : ; < = > ? */ + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 1, 1, 1, 1, 1, 1, + +/* @ A B C D E F G H I J K L M N O */ + 1, 80, 80, 80, 80, 80, 80, 16, 16, 16, 16, 16, 16, 16, 16, 16, + +/* P Q R S T U V W X Y Z [ \ ] ^ _ */ + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 1, 1, 1, 1, 16, + +/* ` a b c d e f g h i j k l m n o */ + 1, 96, 96, 96, 96, 96, 96, 32, 32, 32, 32, 32, 32, 32, 32, 32, + +/* p q r s t u v w x y z { | } ~ DEL */ + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 1, 1, 1, 17, 8, + + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8 +}; + +/* Utility function: like xstrdup(), but also lowercases S. */ + +char * +xstrdup_lower (const char *s) +{ + char *copy = xstrdup (s); + char *p = copy; + for (; *p; p++) + *p = c_tolower (*p); + return copy; +} + +/* Copy the string formed by two pointers (one on the beginning, other + on the char after the last char) to a new, malloc-ed location. + 0-terminate it. + If both pointers are NULL, the function returns an empty string. */ +char * +strdupdelim (const char *beg, const char *end) +{ + if (beg && beg <= end) + { + char *res = xmalloc (end - beg + 1); + memcpy (res, beg, end - beg); + res[end - beg] = '\0'; + return res; + } + + return xstrdup(""); +} + +/* Parse a string containing comma-separated elements, and return a + vector of char pointers with the elements. Spaces following the + commas are ignored. */ +char ** +sepstring (const char *s) +{ + char **res; + const char *p; + int i = 0; + + if (!s || !*s) + return NULL; + res = NULL; + p = s; + while (*s) + { + if (*s == ',') + { + res = xrealloc (res, (i + 2) * sizeof (char *)); + res[i] = strdupdelim (p, s); + res[++i] = NULL; + ++s; + /* Skip the blanks following the ','. */ + while (c_isspace (*s)) + ++s; + p = s; + } + else + ++s; + } + res = xrealloc (res, (i + 2) * sizeof (char *)); + res[i] = strdupdelim (p, s); + res[i + 1] = NULL; + return res; +} + +/* Like sprintf, but prints into a string of sufficient size freshly + allocated with malloc, which is returned. If unable to print due + to invalid format, returns NULL. Inability to allocate needed + memory results in abort, as with xmalloc. This is in spirit + similar to the GNU/BSD extension asprintf, but somewhat easier to + use. + + Internally the function either calls vasprintf or loops around + vsnprintf until the correct size is found. Since Wget also ships a + fallback implementation of vsnprintf, this should be portable. */ + +char * +aprintf (const char *fmt, ...) +{ +#if defined HAVE_VASPRINTF && !defined DEBUG_MALLOC + /* Use vasprintf. */ + int ret; + va_list args; + char *str; + va_start (args, fmt); + ret = vasprintf (&str, fmt, args); + va_end (args); + if (ret < 0 && errno == ENOMEM) + memfatal ("aprintf", UNKNOWN_ATTEMPTED_SIZE); /* for consistency + with xmalloc/xrealloc */ + else if (ret < 0) + return NULL; + return str; +#else /* not HAVE_VASPRINTF */ + +/* Constant is using for limits memory allocation for text buffer. + Applicable in situation when: vasprintf is not available in the system + and vsnprintf return -1 when long line is truncated (in old versions of + glibc and in other system where C99 doesn`t support) */ + +#define FMT_MAX_LENGTH 1048576 + + /* vasprintf is unavailable. snprintf into a small buffer and + resize it as necessary. */ + int size = 32; + char *str = xmalloc (size); + + /* #### This code will infloop and eventually abort in xrealloc if + passed a FMT that causes snprintf to consistently return -1. */ + + while (1) + { + int n; + va_list args; + + va_start (args, fmt); + n = vsnprintf (str, size, fmt, args); + va_end (args); + + /* If the printing worked, return the string. */ + if (n > -1 && n < size) + return str; + + /* Else try again with a larger buffer. */ + if (n > -1) /* C99 */ + size = n + 1; /* precisely what is needed */ + else if (size >= FMT_MAX_LENGTH) /* We have a huge buffer, */ + { /* maybe we have some wrong + format string? */ + logprintf (LOG_ALWAYS, + _("%s: aprintf: text buffer is too big (%d bytes), " + "aborting.\n"), + exec_name, size); /* printout a log message */ + abort (); /* and abort... */ + } + else + { + /* else, we continue to grow our + * buffer: Twice the old size. */ + size <<= 1; + } + str = xrealloc (str, size); + } +#endif /* not HAVE_VASPRINTF */ +} + +#ifndef HAVE_STRLCPY +/* strlcpy() is a BSD function that sometimes is really handy. + * It is the same as snprintf(dst,dstsize,"%s",src), but much faster. */ + +size_t +strlcpy (char *dst, const char *src, size_t size) +{ + const char *old = src; + + /* Copy as many bytes as will fit */ + if (size) + { + while (--size) + { + if (!(*dst++ = *src++)) + return src - old - 1; + } + + *dst = 0; + } + + while (*src++); + return src - old - 1; +} +#endif + +/* Concatenate the NULL-terminated list of string arguments into + freshly allocated space. */ + +char * +concat_strings (const char *str0, ...) +{ + va_list args; + const char *arg; + size_t length = 0, pos = 0; + char *s; + + if (!str0) + return NULL; + + /* calculate the length of the resulting string */ + va_start (args, str0); + for (arg = str0; arg; arg = va_arg (args, const char *)) + length += strlen(arg); + va_end (args); + + s = xmalloc (length + 1); + + /* concatenate strings */ + va_start (args, str0); + for (arg = str0; arg; arg = va_arg (args, const char *)) + pos += strlcpy(s + pos, arg, length - pos + 1); + va_end (args); + + return s; +} + +/* Format the provided time according to the specified format. The + format is a string with format elements supported by strftime. */ + +static char * +fmttime (time_t t, const char *fmt) +{ + static char output[32]; + struct tm *tm = localtime(&t); + if (!tm) + abort (); + if (!strftime(output, sizeof(output), fmt, tm)) + abort (); + return output; +} + +/* Return pointer to a static char[] buffer in which zero-terminated + string-representation of TM (in form hh:mm:ss) is printed. + + If TM is NULL, the current time will be used. */ + +char * +time_str (time_t t) +{ + return fmttime(t, "%H:%M:%S"); +} + +/* Like the above, but include the date: YYYY-MM-DD hh:mm:ss. */ + +char * +datetime_str (time_t t) +{ + return fmttime(t, "%Y-%m-%d %H:%M:%S"); +} + +/* The Windows versions of the following two functions are defined in + mswindows.c. On MSDOS this function should never be called. */ + +#ifdef __VMS + +bool +fork_to_background (void) +{ + return false; +} + +#else /* def __VMS */ + +#if !defined(WINDOWS) && !defined(MSDOS) +bool +fork_to_background (void) +{ + pid_t pid; + /* Whether we arrange our own version of opt.lfilename here. */ + bool logfile_changed = false; + + if (!opt.lfilename && (!opt.quiet || opt.server_response)) + { + /* We must create the file immediately to avoid either a race + condition (which arises from using unique_name and failing to + use fopen_excl) or lying to the user about the log file name + (which arises from using unique_name, printing the name, and + using fopen_excl later on.) */ + FILE *new_log_fp = unique_create (DEFAULT_LOGFILE, false, &opt.lfilename); + if (new_log_fp) + { + logfile_changed = true; + fclose (new_log_fp); + } + } + pid = fork (); + if (pid < 0) + { + /* parent, error */ + perror ("fork"); + exit (WGET_EXIT_GENERIC_ERROR); + } + else if (pid != 0) + { + /* parent, no error */ + printf (_("Continuing in background, pid %d.\n"), (int) pid); + if (logfile_changed) + printf (_("Output will be written to %s.\n"), quote (opt.lfilename)); + exit (WGET_EXIT_SUCCESS); /* #### should we use _exit()? */ + } + + /* child: give up the privileges and keep running. */ + setsid (); + if (freopen ("/dev/null", "r", stdin) == NULL) + DEBUGP (("Failed to redirect stdin to /dev/null.\n")); + if (freopen ("/dev/null", "w", stdout) == NULL) + DEBUGP (("Failed to redirect stdout to /dev/null.\n")); + if (freopen ("/dev/null", "w", stderr) == NULL) + DEBUGP (("Failed to redirect stderr to /dev/null.\n")); + + return logfile_changed; +} +#endif /* !WINDOWS && !MSDOS */ + +#endif /* def __VMS [else] */ + + +/* "Touch" FILE, i.e. make its mtime ("modified time") equal the time + specified with TM. The atime ("access time") is set to the current + time. */ + +void +touch (const char *file, time_t tm) +{ + struct utimbuf times; + + times.modtime = tm; + times.actime = time (NULL); + + if (utime (file, ×) == -1) + logprintf (LOG_NOTQUIET, "utime(%s): %s\n", file, strerror (errno)); +} + +/* Checks if FILE is a symbolic link, and removes it if it is. Does + nothing under MS-Windows. */ +int +remove_link (const char *file) +{ + int err = 0; + struct stat st; + + if (lstat (file, &st) == 0 && S_ISLNK (st.st_mode)) + { + DEBUGP (("Unlinking %s (symlink).\n", file)); + err = unlink (file); + if (err != 0) + logprintf (LOG_VERBOSE, _("Failed to unlink symlink %s: %s\n"), + quote (file), strerror (errno)); + } + return err; +} + +/* Does FILENAME exist? */ +bool +file_exists_p (const char *filename, file_stats_t *fstats) +{ + struct stat buf; + + if (!filename) + return false; + +#if defined(WINDOWS) || defined(__VMS) + int ret = stat (filename, &buf); + if (ret >= 0) + { + if (fstats != NULL) + fstats->access_err = errno; + } + return ret >= 0; +#else + errno = 0; + if (stat (filename, &buf) == 0 && S_ISREG(buf.st_mode) && + (((S_IRUSR & buf.st_mode) && (getuid() == buf.st_uid)) || + ((S_IRGRP & buf.st_mode) && group_member(buf.st_gid)) || + (S_IROTH & buf.st_mode))) { + if (fstats != NULL) + { + fstats->access_err = 0; + fstats->st_ino = buf.st_ino; + fstats->st_dev = buf.st_dev; + } + return true; + } + else + { + if (fstats != NULL) + fstats->access_err = (errno == 0 ? EACCES : errno); + errno = 0; + return false; + } + /* NOTREACHED */ +#endif +} + +/* Returns 0 if PATH is a directory, 1 otherwise (any kind of file). + Returns 0 on error. */ +bool +file_non_directory_p (const char *path) +{ + struct stat buf; + /* Use lstat() rather than stat() so that symbolic links pointing to + directories can be identified correctly. */ + if (lstat (path, &buf) != 0) + return false; + return S_ISDIR (buf.st_mode) ? false : true; +} + +/* Return the size of file named by FILENAME, or -1 if it cannot be + opened or sought into. */ +wgint +file_size (const char *filename) +{ +#if defined(HAVE_FSEEKO) && defined(HAVE_FTELLO) + wgint size; + /* We use fseek rather than stat to determine the file size because + that way we can also verify that the file is readable without + explicitly checking for permissions. Inspired by the POST patch + by Arnaud Wylie. */ + FILE *fp = fopen (filename, "rb"); + if (!fp) + return -1; + fseeko (fp, 0, SEEK_END); + size = ftello (fp); + fclose (fp); + return size; +#else + struct stat st; + if (stat (filename, &st) < 0) + return -1; + return st.st_size; +#endif +} + +/* 2005-02-19 SMS. + If no UNIQ_SEP is defined (as on VMS), have unique_name() return the + original name. With the VMS file systems' versioning, everything + should be fine, and appending ".NN" just causes trouble. +*/ + +#ifdef UNIQ_SEP + +/* stat file names named PREFIX.1, PREFIX.2, etc., until one that + doesn't exist is found. Return a freshly allocated copy of the + unused file name. */ + +static char * +unique_name_1 (const char *prefix) +{ + int count = 1; + int plen = strlen (prefix); + char *template = xmalloc (plen + 1 + 24); + char *template_tail = template + plen; + + memcpy (template, prefix, plen); + *template_tail++ = UNIQ_SEP; + + do + number_to_string (template_tail, count++); + while (file_exists_p (template, NULL) && count < 999999); + + return template; +} + +/* Return a unique file name, based on FILE. + + More precisely, if FILE doesn't exist, it is returned unmodified. + If not, FILE.1 is tried, then FILE.2, etc. The first FILE.<number> + file name that doesn't exist is returned. + + 2005-02-19 SMS. "." is now UNIQ_SEP, and may be different. + + The resulting file is not created, only verified that it didn't + exist at the point in time when the function was called. + Therefore, where security matters, don't rely that the file created + by this function exists until you open it with O_EXCL or + equivalent. + + unique_name() always returns a freshly allocated string. + + unique_name_passthrough() may return FILE if the file doesn't exist + (and therefore doesn't need changing). */ + +char * +unique_name_passthrough (const char *file) +{ + /* If the FILE itself doesn't exist, return it without + modification. Otherwise, find a numeric suffix that results in unused + file name and return it. */ + return file_exists_p (file, NULL) ? unique_name_1 (file) : (char *) file; +} + +char * +unique_name (const char *file) +{ + /* If the FILE itself doesn't exist, return it without + modification. Otherwise, find a numeric suffix that results in unused + file name and return it. */ + return file_exists_p (file, NULL) ? unique_name_1 (file) : xstrdup (file); +} + +#else /* def UNIQ_SEP */ + +/* Dummy unique_name() for VMS. Return the original name as easily as + possible. +*/ +char * +unique_name_passthrough (const char *file, bool allow_passthrough) +{ + /* Return the FILE itself, without modification, irregardful. */ + return (char *) file); +} +char * + +unique_name (const char *file) +{ + /* Return the FILE itself, without modification, irregardful. */ + return xstrdup (file); +} + +#endif /* def UNIQ_SEP [else] */ + +/* Create a file based on NAME, except without overwriting an existing + file with that name. Providing O_EXCL is correctly implemented, + this function does not have the race condition associated with + opening the file returned by unique_name. */ + +FILE * +unique_create (const char *name, bool binary, char **opened_name) +{ + /* unique file name, based on NAME */ + char *uname = unique_name (name); + FILE *fp; + while ((fp = fopen_excl (uname, binary)) == NULL && errno == EEXIST) + { + xfree (uname); + uname = unique_name (name); + } + if (opened_name) + { + if (fp) + *opened_name = uname; + else + { + *opened_name = NULL; + xfree (uname); + } + } + else + xfree (uname); + return fp; +} + +/* Open the file for writing, with the addition that the file is + opened "exclusively". This means that, if the file already exists, + this function will *fail* and errno will be set to EEXIST. If + BINARY is set, the file will be opened in binary mode, equivalent + to fopen's "wb". + + If opening the file fails for any reason, including the file having + previously existed, this function returns NULL and sets errno + appropriately. */ + +FILE * +fopen_excl (const char *fname, int binary) +{ + int fd; +#ifdef O_EXCL + +/* 2005-04-14 SMS. + VMS lacks O_BINARY, but makes up for it in weird and wonderful ways. + It also has file versions which obviate all the O_EXCL effort. + O_TRUNC (something of a misnomer) requests a new version. +*/ +# ifdef __VMS +/* Common open() optional arguments: + sequential access only, access callback function. +*/ +# define OPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id + + int open_id; + int flags = O_WRONLY | O_CREAT | O_TRUNC; + + if (binary > 1) + { + open_id = 11; + fd = open( fname, /* File name. */ + flags, /* Flags. */ + 0777, /* Mode for default protection. */ + "ctx=bin,stm", /* Binary, stream access. */ + "rfm=stmlf", /* Stream_LF. */ + OPEN_OPT_ARGS); /* Access callback. */ + } + else if (binary) + { + open_id = 12; + fd = open( fname, /* File name. */ + flags, /* Flags. */ + 0777, /* Mode for default protection. */ + "ctx=bin,stm", /* Binary, stream access. */ + "rfm=fix", /* Fixed-length, */ + "mrs=512", /* 512-byte records. */ + OPEN_OPT_ARGS); /* Access callback. */ + } + else + { + open_id = 13; + fd = open( fname, /* File name. */ + flags, /* Flags. */ + 0777, /* Mode for default protection. */ + "rfm=stmlf", /* Stream_LF. */ + OPEN_OPT_ARGS); /* Access callback. */ + } +# else /* def __VMS */ + int flags = O_WRONLY | O_CREAT | O_EXCL; +# ifdef O_BINARY + if (binary) + flags |= O_BINARY; +# endif + fd = open (fname, flags, 0666); +# endif /* def __VMS [else] */ + + if (fd < 0) + return NULL; + return fdopen (fd, binary ? "wb" : "w"); +#else /* not O_EXCL */ + /* Manually check whether the file exists. This is prone to race + conditions, but systems without O_EXCL haven't deserved + better. */ + if (file_exists_p (fname, NULL)) + { + errno = EEXIST; + return NULL; + } + return fopen (fname, binary ? "wb" : "w"); +#endif /* not O_EXCL */ +} + +/* fopen_stat() assumes that file_exists_p() was called earlier. + file_stats_t passed to this function was returned from file_exists_p() + This is to prevent TOCTTOU race condition. + Details : FIO45-C from https://www.securecoding.cert.org/ + Note that for creating a new file, this check is not useful + + Input: + fname => Name of file to open + mode => File open mode + fstats => Saved file_stats_t about file that was checked for existence + + Returns: + NULL if there was an error + FILE * of opened file stream +*/ +FILE * +fopen_stat(const char *fname, const char *mode, file_stats_t *fstats) +{ + int fd; + FILE *fp; + struct stat fdstats; + +#if defined FUZZING && defined TESTING + fp = fopen_wgetrc (fname, mode); + return fp; +#else + fp = fopen (fname, mode); +#endif + if (fp == NULL) + { + logprintf (LOG_NOTQUIET, _("Failed to Fopen file %s\n"), fname); + return NULL; + } + fd = fileno (fp); + if (fd < 0) + { + logprintf (LOG_NOTQUIET, _("Failed to get FD for file %s\n"), fname); + fclose (fp); + return NULL; + } + memset(&fdstats, 0, sizeof(fdstats)); + if (fstat (fd, &fdstats) == -1) + { + logprintf (LOG_NOTQUIET, _("Failed to stat file %s, (check permissions)\n"), fname); + fclose (fp); + return NULL; + } +#if !(defined(WINDOWS) || defined(__VMS)) + if (fstats != NULL && + (fdstats.st_dev != fstats->st_dev || + fdstats.st_ino != fstats->st_ino)) + { + /* File changed since file_exists_p() : NOT SAFE */ + logprintf (LOG_NOTQUIET, _("File %s changed since the last check. Security check failed.\n"), fname); + fclose (fp); + return NULL; + } +#endif + + return fp; +} + +/* open_stat assumes that file_exists_p() was called earlier to save file_stats + file_stats_t passed to this function was returned from file_exists_p() + This is to prevent TOCTTOU race condition. + Details : FIO45-C from https://www.securecoding.cert.org/ + Note that for creating a new file, this check is not useful + + + Input: + fname => Name of file to open + flags => File open flags + mode => File open mode + fstats => Saved file_stats_t about file that was checked for existence + + Returns: + -1 if there was an error + file descriptor of opened file stream +*/ +int +open_stat(const char *fname, int flags, mode_t mode, file_stats_t *fstats) +{ + int fd; + struct stat fdstats; + + fd = open (fname, flags, mode); + if (fd < 0) + { + logprintf (LOG_NOTQUIET, _("Failed to open file %s, reason :%s\n"), fname, strerror(errno)); + return -1; + } + memset(&fdstats, 0, sizeof(fdstats)); + if (fstat (fd, &fdstats) == -1) + { + logprintf (LOG_NOTQUIET, _("Failed to stat file %s, error: %s\n"), fname, strerror(errno)); + close (fd); + return -1; + } +#if !(defined(WINDOWS) || defined(__VMS)) + if (fstats != NULL && + (fdstats.st_dev != fstats->st_dev || + fdstats.st_ino != fstats->st_ino)) + { + /* File changed since file_exists_p() : NOT SAFE */ + logprintf (LOG_NOTQUIET, _("Trying to open file %s but it changed since last check. Security check failed.\n"), fname); + close (fd); + return -1; + } +#endif + + return fd; +} + +/* Create DIRECTORY. If some of the pathname components of DIRECTORY + are missing, create them first. In case any mkdir() call fails, + return its error status. Returns 0 on successful completion. + + The behaviour of this function should be identical to the behaviour + of `mkdir -p' on systems where mkdir supports the `-p' option. */ +int +make_directory (const char *directory) +{ + int i, ret, quit = 0; + char buf[1024]; + char *dir; + size_t len = strlen (directory); + + /* Make a copy of dir, to be able to write to it. Otherwise, the + function is unsafe if called with a read-only char *argument. */ + if (len < sizeof(buf)) + { + memcpy(buf, directory, len + 1); + dir = buf; + } + else + dir = xstrdup(directory); + + /* If the first character of dir is '/', skip it (and thus enable + creation of absolute-pathname directories. */ + for (i = (*dir == '/'); 1; ++i) + { + for (; dir[i] && dir[i] != '/'; i++) + ; + if (!dir[i]) + quit = 1; + dir[i] = '\0'; + /* Check whether the directory already exists. Allow creation of + of intermediate directories to fail, as the initial path components + are not necessarily directories! */ + if (!file_exists_p (dir, NULL)) + ret = mkdir (dir, 0777); + else + ret = 0; + if (quit) + break; + else + dir[i] = '/'; + } + + if (dir != buf) + xfree (dir); + + return ret; +} + +/* Merge BASE with FILE. BASE can be a directory or a file name, FILE + should be a file name. + + file_merge("/foo/bar", "baz") => "/foo/baz" + file_merge("/foo/bar/", "baz") => "/foo/bar/baz" + file_merge("foo", "bar") => "bar" + + In other words, it's a simpler and gentler version of uri_merge. */ + +char * +file_merge (const char *base, const char *file) +{ + char *result; + const char *cut = (const char *)strrchr (base, '/'); + + if (!cut) + return xstrdup (file); + + result = xmalloc (cut - base + 1 + strlen (file) + 1); + memcpy (result, base, cut - base); + result[cut - base] = '/'; + strcpy (result + (cut - base) + 1, file); + + return result; +} + +/* Like fnmatch, but performs a case-insensitive match. */ + +int +fnmatch_nocase (const char *pattern, const char *string, int flags) +{ + /* The FNM_CASEFOLD flag started as a GNU extension, but it is now + also present on *BSD platforms, and possibly elsewhere. + Gnulib provides this flag in case it doesn't exist. */ + return fnmatch (pattern, string, flags | FNM_CASEFOLD); +} + +static bool in_acclist (const char *const *, const char *, bool); + +/* Determine whether a file is acceptable to be followed, according to + lists of patterns to accept/reject. */ +bool +acceptable (const char *s) +{ + const char *p; + + if (opt.output_document && strcmp (s, opt.output_document) == 0) + return true; + + if ((p = strrchr (s, '/'))) + s = p + 1; + + if (opt.accepts) + { + if (opt.rejects) + return (in_acclist ((const char *const *)opt.accepts, s, true) + && !in_acclist ((const char *const *)opt.rejects, s, true)); + else + return in_acclist ((const char *const *)opt.accepts, s, true); + } + else if (opt.rejects) + return !in_acclist ((const char *const *)opt.rejects, s, true); + + return true; +} + +/* Determine whether an URL is acceptable to be followed, according to + regex patterns to accept/reject. */ +bool +accept_url (const char *s) +{ + if (opt.acceptregex && !opt.regex_match_fun (opt.acceptregex, s)) + return false; + if (opt.rejectregex && opt.regex_match_fun (opt.rejectregex, s)) + return false; + + return true; +} + +/* Check if D2 is a subdirectory of D1. E.g. if D1 is `/something', subdir_p() + will return true if and only if D2 begins with `/something/' or is exactly + '/something'. */ +bool +subdir_p (const char *d1, const char *d2) +{ + if (*d1 == '\0') + return true; + if (!opt.ignore_case) + for (; *d1 && *d2 && (*d1 == *d2); ++d1, ++d2) + ; + else + for (; *d1 && *d2 && (c_tolower (*d1) == c_tolower (*d2)); ++d1, ++d2) + ; + + return *d1 == '\0' && (*d2 == '\0' || *d2 == '/'); +} + +/* Iterate through DIRLIST (which must be NULL-terminated), and return the + first element that matches DIR, through wildcards or front comparison (as + appropriate). */ +static bool +dir_matches_p (const char **dirlist, const char *dir) +{ + const char **x; + int (*matcher) (const char *, const char *, int) + = opt.ignore_case ? fnmatch_nocase : fnmatch; + + for (x = dirlist; *x; x++) + { + /* Remove leading '/' */ + const char *p = *x + (**x == '/'); + if (has_wildcards_p (p)) + { + if (matcher (p, dir, FNM_PATHNAME) == 0) + break; + } + else + { + if (subdir_p (p, dir)) + break; + } + } + + return *x ? true : false; +} + +/* Returns whether DIRECTORY is acceptable for download, wrt the + include/exclude lists. + + The leading `/' is ignored in paths; relative and absolute paths + may be freely intermixed. */ + +bool +accdir (const char *directory) +{ + /* Remove starting '/'. */ + if (*directory == '/') + ++directory; + if (opt.includes) + { + if (!dir_matches_p (opt.includes, directory)) + return false; + } + if (opt.excludes) + { + if (dir_matches_p (opt.excludes, directory)) + return false; + } + return true; +} + +/* Return true if STRING ends with TAIL. For instance: + + match_tail ("abc", "bc", false) -> 1 + match_tail ("abc", "ab", false) -> 0 + match_tail ("abc", "abc", false) -> 1 + + If FOLD_CASE is true, the comparison will be case-insensitive. */ + +bool +match_tail (const char *string, const char *tail, bool fold_case) +{ + int pos = (int) strlen (string) - (int) strlen (tail); + + if (pos < 0) + return false; /* tail is longer than string. */ + + if (!fold_case) + return !strcmp (string + pos, tail); + else + return !strcasecmp (string + pos, tail); +} + +/* Checks whether string S matches each element of ACCEPTS. A list + element are matched either with fnmatch() or match_tail(), + according to whether the element contains wildcards or not. + + If the BACKWARD is false, don't do backward comparison -- just compare + them normally. */ +static bool +in_acclist (const char *const *accepts, const char *s, bool backward) +{ + for (; *accepts; accepts++) + { + if (has_wildcards_p (*accepts)) + { + int res = opt.ignore_case + ? fnmatch_nocase (*accepts, s, 0) : fnmatch (*accepts, s, 0); + /* fnmatch returns 0 if the pattern *does* match the string. */ + if (res == 0) + return true; + } + else + { + if (backward) + { + if (match_tail (s, *accepts, opt.ignore_case)) + return true; + } + else + { + int cmp = opt.ignore_case + ? strcasecmp (s, *accepts) : strcmp (s, *accepts); + if (cmp == 0) + return true; + } + } + } + return false; +} + +/* Return the location of STR's suffix (file extension). Examples: + suffix ("foo.bar") -> "bar" + suffix ("foo.bar.baz") -> "baz" + suffix ("/foo/bar") -> NULL + suffix ("/foo.bar/baz") -> NULL */ +char * +suffix (const char *str) +{ + char *p; + + if ((p = strrchr (str, '.')) && !strchr (p + 1, '/')) + return p + 1; + + return NULL; +} + +/* Return true if S contains globbing wildcards (`*', `?', `[' or + `]'). */ + +bool +has_wildcards_p (const char *s) +{ + return !!strpbrk (s, "*?[]"); +} + +/* Return true if FNAME ends with a typical HTML suffix. The + following (case-insensitive) suffixes are presumed to be HTML + files: + + html + htm + ?html (`?' matches one character) + + #### CAVEAT. This is not necessarily a good indication that FNAME + refers to a file that contains HTML! */ +bool +has_html_suffix_p (const char *fname) +{ + char *suf; + + if ((suf = suffix (fname)) == NULL) + return false; + if (!c_strcasecmp (suf, "html")) + return true; + if (!c_strcasecmp (suf, "htm")) + return true; + if (suf[0] && !c_strcasecmp (suf + 1, "html")) + return true; + return false; +} + +/* Read FILE into memory. A pointer to `struct file_memory' are + returned; use struct element `content' to access file contents, and + the element `length' to know the file length. `content' is *not* + zero-terminated, and you should *not* read or write beyond the [0, + length) range of characters. + + After you are done with the file contents, call wget_read_file_free to + release the memory. + + Depending on the operating system and the type of file that is + being read, wget_read_file() either mmap's the file into memory, or + reads the file into the core using read(). + + If file is named "-", fileno(stdin) is used for reading instead. + If you want to read from a real file named "-", use "./-" instead. */ + +struct file_memory * +wget_read_file (const char *file) +{ + int fd; + struct file_memory *fm; + long size; + bool inhibit_close = false; + + /* Some magic in the finest tradition of Perl and its kin: if FILE + is "-", just use stdin. */ +#ifndef FUZZING + if (HYPHENP (file)) + { + fd = fileno (stdin); + inhibit_close = true; + /* Note that we don't inhibit mmap() in this case. If stdin is + redirected from a regular file, mmap() will still work. */ + } + else +#endif + fd = open (file, O_RDONLY); + if (fd < 0) + return NULL; + fm = xnew (struct file_memory); + +#ifdef HAVE_MMAP + { + struct stat buf; + if (fstat (fd, &buf) < 0) + goto mmap_lose; + fm->length = buf.st_size; + /* NOTE: As far as I know, the callers of this function never + modify the file text. Relying on this would enable us to + specify PROT_READ and MAP_SHARED for a marginal gain in + efficiency, but at some cost to generality. */ + fm->content = mmap (NULL, fm->length, PROT_READ | PROT_WRITE, + MAP_PRIVATE, fd, 0); + if (fm->content == (char *)MAP_FAILED) + goto mmap_lose; + if (!inhibit_close) + close (fd); + + fm->mmap_p = 1; + return fm; + } + + mmap_lose: + /* The most common reason why mmap() fails is that FD does not point + to a plain file. However, it's also possible that mmap() doesn't + work for a particular type of file. Therefore, whenever mmap() + fails, we just fall back to the regular method. */ +#endif /* HAVE_MMAP */ + + fm->length = 0; + size = 512; /* number of bytes fm->contents can + hold at any given time. */ + fm->content = xmalloc (size); + while (1) + { + wgint nread; + if (fm->length > size / 2) + { + /* #### I'm not sure whether the whole exponential-growth + thing makes sense with kernel read. On Linux at least, + read() refuses to read more than 4K from a file at a + single chunk anyway. But other Unixes might optimize it + better, and it doesn't *hurt* anything, so I'm leaving + it. */ + + /* Normally, we grow SIZE exponentially to make the number + of calls to read() and realloc() logarithmic in relation + to file size. However, read() can read an amount of data + smaller than requested, and it would be unreasonable to + double SIZE every time *something* was read. Therefore, + we double SIZE only when the length exceeds half of the + entire allocated size. */ + size <<= 1; + fm->content = xrealloc (fm->content, size); + } + nread = read (fd, fm->content + fm->length, size - fm->length); + if (nread > 0) + /* Successful read. */ + fm->length += nread; + else if (nread < 0) + /* Error. */ + goto lose; + else + /* EOF */ + break; + } + if (!inhibit_close) + close (fd); + if (size > fm->length && fm->length != 0) + /* Due to exponential growth of fm->content, the allocated region + might be much larger than what is actually needed. */ + fm->content = xrealloc (fm->content, fm->length); + fm->mmap_p = 0; + return fm; + + lose: + if (!inhibit_close) + close (fd); + xfree (fm->content); + xfree (fm); + return NULL; +} + +/* Release the resources held by FM. Specifically, this calls + munmap() or xfree() on fm->content, depending whether mmap or + malloc/read were used to read in the file. It also frees the + memory needed to hold the FM structure itself. */ + +void +wget_read_file_free (struct file_memory *fm) +{ +#ifdef HAVE_MMAP + if (fm->mmap_p) + { + munmap (fm->content, fm->length); + } + else +#endif + { + xfree (fm->content); + } + xfree (fm); +} + +/* Free the pointers in a NULL-terminated vector of pointers, then + free the pointer itself. */ +void +free_vec (char **vec) +{ + if (vec) + { + char **p = vec; + while (*p) + { + xfree (*p); + p++; + } + xfree (vec); + } +} + +/* Append vector V2 to vector V1. The function frees V2 and + reallocates V1 (thus you may not use the contents of neither + pointer after the call). If V1 is NULL, V2 is returned. */ +char ** +merge_vecs (char **v1, char **v2) +{ + int i, j; + + if (!v1) + return v2; + if (!v2) + return v1; + if (!*v2) + { + /* To avoid j == 0 */ + xfree (v2); + return v1; + } + /* Count v1. */ + for (i = 0; v1[i]; i++) + ; + /* Count v2. */ + for (j = 0; v2[j]; j++) + ; + /* Reallocate v1. */ + v1 = xrealloc (v1, (i + j + 1) * sizeof (char *)); + memcpy (v1 + i, v2, (j + 1) * sizeof (char *)); + xfree (v2); + return v1; +} + +/* Append a freshly allocated copy of STR to VEC. If VEC is NULL, it + is allocated as needed. Return the new value of the vector. */ + +char ** +vec_append (char **vec, const char *str) +{ + int cnt; /* count of vector elements, including + the one we're about to append */ + if (vec != NULL) + { + for (cnt = 0; vec[cnt]; cnt++) + ; + ++cnt; + } + else + cnt = 1; + /* Reallocate the array to fit the new element and the NULL. */ + vec = xrealloc (vec, (cnt + 1) * sizeof (char *)); + /* Append a copy of STR to the vector. */ + vec[cnt - 1] = xstrdup (str); + vec[cnt] = NULL; + return vec; +} + +/* Sometimes it's useful to create "sets" of strings, i.e. special + hash tables where you want to store strings as keys and merely + query for their existence. Here is a set of utility routines that + makes that transparent. */ + +void +string_set_add (struct hash_table *ht, const char *s) +{ + /* First check whether the set element already exists. If it does, + do nothing so that we don't have to free() the old element and + then strdup() a new one. */ + if (hash_table_contains (ht, s)) + return; + + /* We use "1" as value. It provides us a useful and clear arbitrary + value, and it consumes no memory -- the pointers to the same + string "1" will be shared by all the key-value pairs in all `set' + hash tables. */ + hash_table_put (ht, xstrdup (s), "1"); +} + +/* Synonym for hash_table_contains... */ + +int +string_set_contains (struct hash_table *ht, const char *s) +{ + return hash_table_contains (ht, s); +} + +/* Convert the specified string set to array. ARRAY should be large + enough to hold hash_table_count(ht) char pointers. */ + +void string_set_to_array (struct hash_table *ht, char **array) +{ + hash_table_iterator iter; + for (hash_table_iterate (ht, &iter); hash_table_iter_next (&iter); ) + *array++ = iter.key; +} + +/* Free the string set. This frees both the storage allocated for + keys and the actual hash table. (hash_table_destroy would only + destroy the hash table.) */ + +void +string_set_free (struct hash_table *ht) +{ + hash_table_iterator iter; + for (hash_table_iterate (ht, &iter); hash_table_iter_next (&iter); ) + xfree (iter.key); + hash_table_destroy (ht); +} + +/* Utility function: simply call xfree() on all keys and values of HT. */ + +void +free_keys_and_values (struct hash_table *ht) +{ + hash_table_iterator iter; + for (hash_table_iterate (ht, &iter); hash_table_iter_next (&iter); ) + { + xfree (iter.key); + xfree (iter.value); + } +} + +/* Get digit grouping data for thousand separors by calling + localeconv(). The data includes separator string and grouping info + and is cached after the first call to the function. + + In locales that don't set a thousand separator (such as the "C" + locale), this forces it to be ",". We are now only showing + thousand separators in one place, so this shouldn't be a problem in + practice. */ + +static void +get_grouping_data (const char **sep, const char **grouping) +{ + static const char *cached_sep; + static const char *cached_grouping; + static bool initialized; + if (!initialized) + { + /* Get the grouping info from the locale. */ + struct lconv *lconv = localeconv (); + cached_sep = lconv->thousands_sep; + cached_grouping = lconv->grouping; +#if ! USE_NLS_PROGRESS_BAR + /* We can't count column widths, so ensure that the separator + * is single-byte only (let check below determine what byte). */ + if (strlen(cached_sep) > 1) + cached_sep = ""; +#endif + if (!*cached_sep) + { + /* Many locales (such as "C" or "hr_HR") don't specify + grouping, which we still want to use it for legibility. + In those locales set the sep char to ',', unless that + character is used for decimal point, in which case set it + to ".". */ + if (*lconv->decimal_point != ',') + cached_sep = ","; + else + cached_sep = "."; + cached_grouping = "\x03"; + } + initialized = true; + } + *sep = cached_sep; + *grouping = cached_grouping; +} + +/* Return a printed representation of N with thousand separators. + This should respect locale settings, with the exception of the "C" + locale which mandates no separator, but we use one anyway. + + Unfortunately, we cannot use %'d (in fact it would be %'j) to get + the separators because it's too non-portable, and it's hard to test + for this feature at configure time. Besides, it wouldn't display + separators in the "C" locale, still used by many Unix users. */ + +const char * +with_thousand_seps (wgint n) +{ + static char outbuf[48]; + char *p = outbuf + sizeof outbuf; + + /* Info received from locale */ + const char *grouping, *sep; + int seplen; + + /* State information */ + int i = 0, groupsize; + const char *atgroup; + + bool negative = n < 0; + + /* Initialize grouping data. */ + get_grouping_data (&sep, &grouping); + seplen = strlen (sep); + atgroup = grouping; + groupsize = *atgroup++; + + /* This would overflow on WGINT_MIN, but printing negative numbers + is not an important goal of this fuinction. */ + if (negative) + n = -n; + + /* Write the number into the buffer, backwards, inserting the + separators as necessary. */ + *--p = '\0'; + while (1) + { + *--p = n % 10 + '0'; + n /= 10; + if (n == 0) + break; + /* Prepend SEP to every groupsize'd digit and get new groupsize. */ + if (++i == groupsize) + { + if (seplen == 1) + *--p = *sep; + else + memcpy (p -= seplen, sep, seplen); + i = 0; + if (*atgroup) + groupsize = *atgroup++; + } + } + if (negative) + *--p = '-'; + + return p; +} + +/* N, a byte quantity, is converted to a human-readable abberviated + form a la sizes printed by `ls -lh'. The result is written to a + static buffer, a pointer to which is returned. + + Unlike `with_thousand_seps', this approximates to the nearest unit. + Quoting GNU libit: "Most people visually process strings of 3-4 + digits effectively, but longer strings of digits are more prone to + misinterpretation. Hence, converting to an abbreviated form + usually improves readability." + + This intentionally uses kilobyte (KB), megabyte (MB), etc. in their + original computer-related meaning of "powers of 1024". We don't + use the "*bibyte" names invented in 1998, and seldom used in + practice. Wikipedia's entry on "binary prefix" discusses this in + some detail. */ + +char * +human_readable (wgint n, const int acc, const int decimals) +{ + /* These suffixes are compatible with those of GNU `ls -lh'. */ + static char powers[] = + { + 'K', /* kilobyte, 2^10 bytes */ + 'M', /* megabyte, 2^20 bytes */ + 'G', /* gigabyte, 2^30 bytes */ + 'T', /* terabyte, 2^40 bytes */ + 'P', /* petabyte, 2^50 bytes */ + 'E', /* exabyte, 2^60 bytes */ + }; + static char buf[8]; + size_t i; + + /* If the quantity is smaller than 1K, just print it. */ + if (n < 1024) + { + snprintf (buf, sizeof (buf), "%d", (int) n); + return buf; + } + + /* Loop over powers, dividing N with 1024 in each iteration. This + works unchanged for all sizes of wgint, while still avoiding + non-portable `long double' arithmetic. */ + for (i = 0; i < countof (powers); i++) + { + /* At each iteration N is greater than the *subsequent* power. + That way N/1024.0 produces a decimal number in the units of + *this* power. */ + if ((n / 1024) < 1024 || i == countof (powers) - 1) + { + double val = n / 1024.0; + /* Print values smaller than the accuracy level (acc) with (decimal) + * decimal digits, and others without any decimals. */ + snprintf (buf, sizeof (buf), "%.*f%c", + val < acc ? decimals : 0, val, powers[i]); + return buf; + } + n /= 1024; + } + return NULL; /* unreached */ +} + +/* Count the digits in the provided number. Used to allocate space + when printing numbers. */ + +int +numdigit (wgint number) +{ + int cnt = 1; + if (number < 0) + ++cnt; /* accommodate '-' */ + while ((number /= 10) != 0) + ++cnt; + return cnt; +} + +#define PR(mask) *p++ = n / (mask) + '0' + +/* DIGITS_<D> is used to print a D-digit number and should be called + with mask==10^(D-1). It prints n/mask (the first digit), reducing + n to n%mask (the remaining digits), and calling DIGITS_<D-1>. + Recursively this continues until DIGITS_1 is invoked. */ + +#define DIGITS_1(mask) PR (mask) +#define DIGITS_2(mask) PR (mask), n %= (mask), DIGITS_1 ((mask) / 10) +#define DIGITS_3(mask) PR (mask), n %= (mask), DIGITS_2 ((mask) / 10) +#define DIGITS_4(mask) PR (mask), n %= (mask), DIGITS_3 ((mask) / 10) +#define DIGITS_5(mask) PR (mask), n %= (mask), DIGITS_4 ((mask) / 10) +#define DIGITS_6(mask) PR (mask), n %= (mask), DIGITS_5 ((mask) / 10) +#define DIGITS_7(mask) PR (mask), n %= (mask), DIGITS_6 ((mask) / 10) +#define DIGITS_8(mask) PR (mask), n %= (mask), DIGITS_7 ((mask) / 10) +#define DIGITS_9(mask) PR (mask), n %= (mask), DIGITS_8 ((mask) / 10) +#define DIGITS_10(mask) PR (mask), n %= (mask), DIGITS_9 ((mask) / 10) + +/* DIGITS_<11-20> are only used on machines with 64-bit wgints. */ + +#define DIGITS_11(mask) PR (mask), n %= (mask), DIGITS_10 ((mask) / 10) +#define DIGITS_12(mask) PR (mask), n %= (mask), DIGITS_11 ((mask) / 10) +#define DIGITS_13(mask) PR (mask), n %= (mask), DIGITS_12 ((mask) / 10) +#define DIGITS_14(mask) PR (mask), n %= (mask), DIGITS_13 ((mask) / 10) +#define DIGITS_15(mask) PR (mask), n %= (mask), DIGITS_14 ((mask) / 10) +#define DIGITS_16(mask) PR (mask), n %= (mask), DIGITS_15 ((mask) / 10) +#define DIGITS_17(mask) PR (mask), n %= (mask), DIGITS_16 ((mask) / 10) +#define DIGITS_18(mask) PR (mask), n %= (mask), DIGITS_17 ((mask) / 10) +#define DIGITS_19(mask) PR (mask), n %= (mask), DIGITS_18 ((mask) / 10) + +/* Shorthand for casting to wgint. */ +#define W wgint + +/* Print NUMBER to BUFFER in base 10. This is equivalent to + `sprintf(buffer, "%lld", (long long) number)', only typically much + faster and portable to machines without long long. + + The speedup may make a difference in programs that frequently + convert numbers to strings. Some implementations of sprintf, + particularly the one in some versions of GNU libc, have been known + to be quite slow when converting integers to strings. + + Return the pointer to the location where the terminating zero was + printed. (Equivalent to calling buffer+strlen(buffer) after the + function is done.) + + BUFFER should be large enough to accept as many bytes as you expect + the number to take up. On machines with 64-bit wgints the maximum + needed size is 24 bytes. That includes the digits needed for the + largest 64-bit number, the `-' sign in case it's negative, and the + terminating '\0'. */ + +char * +number_to_string (char *buffer, wgint number) +{ + char *p = buffer; + wgint n = number; + + int last_digit_char = 0; + + if (n < 0) + { + if (n < -WGINT_MAX) + { + /* n = -n would overflow because -n would evaluate to a + wgint value larger than WGINT_MAX. Need to make n + smaller and handle the last digit separately. */ + int last_digit = n % 10; + /* The sign of n%10 is implementation-defined. */ + if (last_digit < 0) + last_digit_char = '0' - last_digit; + else + last_digit_char = '0' + last_digit; + /* After n is made smaller, -n will not overflow. */ + n /= 10; + } + + *p++ = '-'; + n = -n; + } + + /* Use the DIGITS_ macro appropriate for N's number of digits. That + way printing any N is fully open-coded without a loop or jump. + (Also see description of DIGITS_*.) */ + + if (n < 10) DIGITS_1 (1); + else if (n < 100) DIGITS_2 (10); + else if (n < 1000) DIGITS_3 (100); + else if (n < 10000) DIGITS_4 (1000); + else if (n < 100000) DIGITS_5 (10000); + else if (n < 1000000) DIGITS_6 (100000); + else if (n < 10000000) DIGITS_7 (1000000); + else if (n < 100000000) DIGITS_8 (10000000); + else if (n < 1000000000) DIGITS_9 (100000000); + else if (n < 10*(W)1000000000) DIGITS_10 (1000000000); + else if (n < 100*(W)1000000000) DIGITS_11 (10*(W)1000000000); + else if (n < 1000*(W)1000000000) DIGITS_12 (100*(W)1000000000); + else if (n < 10000*(W)1000000000) DIGITS_13 (1000*(W)1000000000); + else if (n < 100000*(W)1000000000) DIGITS_14 (10000*(W)1000000000); + else if (n < 1000000*(W)1000000000) DIGITS_15 (100000*(W)1000000000); + else if (n < 10000000*(W)1000000000) DIGITS_16 (1000000*(W)1000000000); + else if (n < 100000000*(W)1000000000) DIGITS_17 (10000000*(W)1000000000); + else if (n < 1000000000*(W)1000000000) DIGITS_18 (100000000*(W)1000000000); + else DIGITS_19 (1000000000*(W)1000000000); + + if (last_digit_char) + *p++ = last_digit_char; + + *p = '\0'; + + return p; +} + +#undef PR +#undef W +#undef SPRINTF_WGINT +#undef DIGITS_1 +#undef DIGITS_2 +#undef DIGITS_3 +#undef DIGITS_4 +#undef DIGITS_5 +#undef DIGITS_6 +#undef DIGITS_7 +#undef DIGITS_8 +#undef DIGITS_9 +#undef DIGITS_10 +#undef DIGITS_11 +#undef DIGITS_12 +#undef DIGITS_13 +#undef DIGITS_14 +#undef DIGITS_15 +#undef DIGITS_16 +#undef DIGITS_17 +#undef DIGITS_18 +#undef DIGITS_19 + +#define RING_SIZE 3 + +/* Print NUMBER to a statically allocated string and return a pointer + to the printed representation. + + This function is intended to be used in conjunction with printf. + It is hard to portably print wgint values: + a) you cannot use printf("%ld", number) because wgint can be long + long on 32-bit machines with LFS. + b) you cannot use printf("%lld", number) because NUMBER could be + long on 32-bit machines without LFS, or on 64-bit machines, + which do not require LFS. Also, Windows doesn't support %lld. + c) you cannot use printf("%j", (int_max_t) number) because not all + versions of printf support "%j", the most notable being the one + on Windows. + d) you cannot #define WGINT_FMT to the appropriate format and use + printf(WGINT_FMT, number) because that would break translations + for user-visible messages, such as printf("Downloaded: %d + bytes\n", number). + + What you should use instead is printf("%s", number_to_static_string + (number)). + + CAVEAT: since the function returns pointers to static data, you + must be careful to copy its result before calling it again. + However, to make it more useful with printf, the function maintains + an internal ring of static buffers to return. That way things like + printf("%s %s", number_to_static_string (num1), + number_to_static_string (num2)) work as expected. Three buffers + are currently used, which means that "%s %s %s" will work, but "%s + %s %s %s" won't. If you need to print more than three wgints, + bump the RING_SIZE (or rethink your message.) */ + +char * +number_to_static_string (wgint number) +{ + static char ring[RING_SIZE][24]; + static int ringpos; + char *buf = ring[ringpos]; + number_to_string (buf, number); + ringpos = (ringpos + 1) % RING_SIZE; + return buf; +} + +/* Converts the byte to bits format if --report-bps option is enabled + */ +wgint +convert_to_bits (wgint num) +{ + if (opt.report_bps) + return num * 8; + return num; +} + + +/* Determine the width of the terminal we're running on. If that's + not possible, return 0. */ + +int +determine_screen_width (void) +{ + /* If there's a way to get the terminal size using POSIX + tcgetattr(), somebody please tell me. */ +#ifdef TIOCGWINSZ + int fd; + struct winsize wsz; + + if (opt.lfilename != NULL && opt.show_progress != 1) + return 0; + + fd = fileno (stderr); + if (ioctl (fd, TIOCGWINSZ, &wsz) < 0) + return 0; /* most likely ENOTTY */ + + return wsz.ws_col; +#elif defined(WINDOWS) + CONSOLE_SCREEN_BUFFER_INFO csbi; + if (!GetConsoleScreenBufferInfo (GetStdHandle (STD_ERROR_HANDLE), &csbi)) + return 0; + return csbi.dwSize.X; +#else /* neither TIOCGWINSZ nor WINDOWS */ + return 0; +#endif /* neither TIOCGWINSZ nor WINDOWS */ +} + +/* Whether the rnd system (either rand or [dl]rand48) has been + seeded. */ +static int rnd_seeded; + +/* Return a random number between 0 and MAX-1, inclusive. + + If the system does not support lrand48 and MAX is greater than the + value of RAND_MAX+1 on the system, the returned value will be in + the range [0, RAND_MAX]. This may be fixed in a future release. + The random number generator is seeded automatically the first time + it is called. + + This uses lrand48 where available, rand elsewhere. DO NOT use it + for cryptography. It is only meant to be used in situations where + quality of the random numbers returned doesn't really matter. */ + +int +random_number (int max) +{ +#ifdef HAVE_RANDOM + if (!rnd_seeded) + { + srandom ((long) time (NULL) ^ (long) getpid ()); + rnd_seeded = 1; + } + return random () % max; +#elif defined HAVE_DRAND48 + if (!rnd_seeded) + { + srand48 ((long) time (NULL) ^ (long) getpid ()); + rnd_seeded = 1; + } + return lrand48 () % max; +#else /* not HAVE_DRAND48 */ + + double bounded; + int rnd; + if (!rnd_seeded) + { + srand ((unsigned) time (NULL) ^ (unsigned) getpid ()); + rnd_seeded = 1; + } + rnd = rand (); + + /* Like rand() % max, but uses the high-order bits for better + randomness on architectures where rand() is implemented using a + simple congruential generator. */ + + bounded = (double) max * rnd / (RAND_MAX + 1.0); + return (int) bounded; + +#endif /* not HAVE_DRAND48 */ +} + +/* Return a random uniformly distributed floating point number in the + [0, 1) range. Uses drand48 where available, and a really lame + kludge elsewhere. */ + +double +random_float (void) +{ +#ifdef HAVE_RANDOM + return ((double) random_number (RAND_MAX)) / RAND_MAX; +#elif defined HAVE_DRAND48 + if (!rnd_seeded) + { + srand48 ((long) time (NULL) ^ (long) getpid ()); + rnd_seeded = 1; + } + return drand48 (); +#else /* not HAVE_DRAND48 */ + return ( random_number (10000) / 10000.0 + + random_number (10000) / (10000.0 * 10000.0) + + random_number (10000) / (10000.0 * 10000.0 * 10000.0) + + random_number (10000) / (10000.0 * 10000.0 * 10000.0 * 10000.0)); +#endif /* not HAVE_DRAND48 */ +} + +/* Implementation of run_with_timeout, a generic timeout-forcing + routine for systems with Unix-like signal handling. */ + +#ifdef USE_SIGNAL_TIMEOUT +# ifdef HAVE_SIGSETJMP +# define SETJMP(env) sigsetjmp (env, 1) + +static sigjmp_buf run_with_timeout_env; + +_Noreturn static void +abort_run_with_timeout (int sig _GL_UNUSED) +{ + assert (sig == SIGALRM); + siglongjmp (run_with_timeout_env, -1); +} +# else /* not HAVE_SIGSETJMP */ +# define SETJMP(env) setjmp (env) + +static jmp_buf run_with_timeout_env; + +static void _Noreturn +abort_run_with_timeout (int sig _GL_UNUSED) +{ + assert (sig == SIGALRM); + /* We don't have siglongjmp to preserve the set of blocked signals; + if we longjumped out of the handler at this point, SIGALRM would + remain blocked. We must unblock it manually. */ + sigset_t set; + sigemptyset (&set); + sigaddset (&set, SIGALRM); + sigprocmask (SIG_BLOCK, &set, NULL); + + /* Now it's safe to longjump. */ + longjmp (run_with_timeout_env, -1); +} +# endif /* not HAVE_SIGSETJMP */ + +/* Arrange for SIGALRM to be delivered in TIMEOUT seconds. This uses + setitimer where available, alarm otherwise. + + TIMEOUT should be non-zero. If the timeout value is so small that + it would be rounded to zero, it is rounded to the least legal value + instead (1us for setitimer, 1s for alarm). That ensures that + SIGALRM will be delivered in all cases. */ + +static void +alarm_set (double timeout) +{ +#ifdef ITIMER_REAL + /* Use the modern itimer interface. */ + struct itimerval itv; + xzero (itv); + itv.it_value.tv_sec = (long) timeout; + itv.it_value.tv_usec = 1000000 * (timeout - (long)timeout); + if (itv.it_value.tv_sec == 0 && itv.it_value.tv_usec == 0) + /* Ensure that we wait for at least the minimum interval. + Specifying zero would mean "wait forever". */ + itv.it_value.tv_usec = 1; + setitimer (ITIMER_REAL, &itv, NULL); +#else /* not ITIMER_REAL */ + /* Use the old alarm() interface. */ + int secs = (int) timeout; + if (secs == 0) + /* Round TIMEOUTs smaller than 1 to 1, not to zero. This is + because alarm(0) means "never deliver the alarm", i.e. "wait + forever", which is not what someone who specifies a 0.5s + timeout would expect. */ + secs = 1; + alarm (secs); +#endif /* not ITIMER_REAL */ +} + +/* Cancel the alarm set with alarm_set. */ + +static void +alarm_cancel (void) +{ +#ifdef ITIMER_REAL + struct itimerval disable; + xzero (disable); + setitimer (ITIMER_REAL, &disable, NULL); +#else /* not ITIMER_REAL */ + alarm (0); +#endif /* not ITIMER_REAL */ +} + +/* Call FUN(ARG), but don't allow it to run for more than TIMEOUT + seconds. Returns true if the function was interrupted with a + timeout, false otherwise. + + This works by setting up SIGALRM to be delivered in TIMEOUT seconds + using setitimer() or alarm(). The timeout is enforced by + longjumping out of the SIGALRM handler. This has several + advantages compared to the traditional approach of relying on + signals causing system calls to exit with EINTR: + + * The callback function is *forcibly* interrupted after the + timeout expires, (almost) regardless of what it was doing and + whether it was in a syscall. For example, a calculation that + takes a long time is interrupted as reliably as an IO + operation. + + * It works with both SYSV and BSD signals because it doesn't + depend on the default setting of SA_RESTART. + + * It doesn't require special handler setup beyond a simple call + to signal(). (It does use sigsetjmp/siglongjmp, but they're + optional.) + + The only downside is that, if FUN allocates internal resources that + are normally freed prior to exit from the functions, they will be + lost in case of timeout. */ + +bool +run_with_timeout (double timeout, void (*fun) (void *), void *arg) +{ + int saved_errno; + + if (timeout == 0) + { + fun (arg); + return false; + } + + if (SETJMP (run_with_timeout_env) != 0) + { + /* Longjumped out of FUN with a timeout. */ + signal (SIGALRM, SIG_DFL); + return true; + } + else + { + signal (SIGALRM, abort_run_with_timeout); + } + alarm_set (timeout); + fun (arg); + + /* Preserve errno in case alarm() or signal() modifies it. */ + saved_errno = errno; + alarm_cancel (); + signal (SIGALRM, SIG_DFL); + errno = saved_errno; + + return false; +} + +#else /* not USE_SIGNAL_TIMEOUT */ + +#ifndef WINDOWS +/* A stub version of run_with_timeout that just calls FUN(ARG). Don't + define it under Windows, because Windows has its own version of + run_with_timeout that uses threads. */ + +bool +run_with_timeout (double timeout, void (*fun) (void *), void *arg) +{ + fun (arg); + return false; +} +#endif /* not WINDOWS */ +#endif /* not USE_SIGNAL_TIMEOUT */ + +#ifndef WINDOWS + +/* Sleep the specified amount of seconds. On machines without + nanosleep(), this may sleep shorter if interrupted by signals. */ + +void +xsleep (double seconds) +{ +#ifdef HAVE_NANOSLEEP + /* nanosleep is the preferred interface because it offers high + accuracy and, more importantly, because it allows us to reliably + restart receiving a signal such as SIGWINCH. (There was an + actual Debian bug report about --limit-rate malfunctioning while + the terminal was being resized.) */ + struct timespec sleep, remaining; + sleep.tv_sec = (long) seconds; + sleep.tv_nsec = 1000000000 * (seconds - (long) seconds); + while (nanosleep (&sleep, &remaining) < 0 && errno == EINTR) + /* If nanosleep has been interrupted by a signal, adjust the + sleeping period and return to sleep. */ + sleep = remaining; +#elif defined(HAVE_USLEEP) + /* If usleep is available, use it in preference to select. */ + if (seconds >= 1) + { + /* On some systems, usleep cannot handle values larger than + 1,000,000. If the period is larger than that, use sleep + first, then add usleep for subsecond accuracy. */ + sleep (seconds); + seconds -= (long) seconds; + } + usleep (seconds * 1000000); +#else /* fall back select */ + /* Note that, although Windows supports select, it can't be used to + implement sleeping because Winsock's select doesn't implement + timeout when it is passed NULL pointers for all fd sets. (But it + does under Cygwin, which implements Unix-compatible select.) */ + struct timeval sleep; + sleep.tv_sec = (long) seconds; + sleep.tv_usec = 1000000 * (seconds - (long) seconds); + select (0, NULL, NULL, NULL, &sleep); + /* If select returns -1 and errno is EINTR, it means we were + interrupted by a signal. But without knowing how long we've + actually slept, we can't return to sleep. Using gettimeofday to + track sleeps is slow and unreliable due to clock skew. */ +#endif +} + +#endif /* not WINDOWS */ + +/* Encode the octets in DATA of length LENGTH to base64 format, + storing the result to DEST. The output will be zero-terminated, + and must point to a writable buffer of at least + 1+BASE64_LENGTH(length) bytes. The function returns the length of + the resulting base64 data, not counting the terminating zero. + + This implementation does not emit newlines after 76 characters of + base64 data. */ + +size_t +wget_base64_encode (const void *data, size_t length, char *dest) +{ + /* Conversion table. */ + static const char tbl[64] = { + 'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P', + 'Q','R','S','T','U','V','W','X','Y','Z','a','b','c','d','e','f', + 'g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v', + 'w','x','y','z','0','1','2','3','4','5','6','7','8','9','+','/' + }; + /* Access bytes in DATA as unsigned char, otherwise the shifts below + don't work for data with MSB set. */ + const unsigned char *s = data; + /* Theoretical ANSI violation when length < 3. */ + const unsigned char *end = (const unsigned char *) data + length - 2; + char *p = dest; + + /* Transform the 3x8 bits to 4x6 bits, as required by base64. */ + for (; s < end; s += 3) + { + *p++ = tbl[s[0] >> 2]; + *p++ = tbl[((s[0] & 3) << 4) + (s[1] >> 4)]; + *p++ = tbl[((s[1] & 0xf) << 2) + (s[2] >> 6)]; + *p++ = tbl[s[2] & 0x3f]; + } + + /* Pad the result if necessary... */ + switch (length % 3) + { + case 1: + *p++ = tbl[s[0] >> 2]; + *p++ = tbl[(s[0] & 3) << 4]; + *p++ = '='; + *p++ = '='; + break; + case 2: + *p++ = tbl[s[0] >> 2]; + *p++ = tbl[((s[0] & 3) << 4) + (s[1] >> 4)]; + *p++ = tbl[((s[1] & 0xf) << 2)]; + *p++ = '='; + break; + } + /* ...and zero-terminate it. */ + *p = '\0'; + + return p - dest; +} + +/* Store in C the next non-whitespace character from the string, or \0 + when end of string is reached. */ +#define NEXT_CHAR(c, p) do { \ + c = (unsigned char) *p++; \ +} while (c_isspace (c)) + +#define IS_ASCII(c) (((c) & 0x80) == 0) + +/* Decode data from BASE64 (a null-terminated string) into memory + pointed to by DEST. DEST is assumed to be large enough to + accommodate the decoded data, which is guaranteed to be no more than + 3/4*strlen(base64). + + Since DEST is assumed to contain binary data, it is not + NUL-terminated. The function returns the length of the data + written to "TO". -1 is returned in case of error caused by malformed + base64 input. + + This function originates from Free Recode. */ + +ssize_t +wget_base64_decode (const char *base64, void *dest, size_t size) +{ + /* Table of base64 values for first 128 characters. Note that this + assumes ASCII (but so does Wget in other places). */ + static const signed char base64_char_to_value[128] = + { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 0- 9 */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 10- 19 */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 20- 29 */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 30- 39 */ + -1, -1, -1, 62, -1, -1, -1, 63, 52, 53, /* 40- 49 */ + 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, /* 50- 59 */ + -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, /* 60- 69 */ + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, /* 70- 79 */ + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, /* 80- 89 */ + 25, -1, -1, -1, -1, -1, -1, 26, 27, 28, /* 90- 99 */ + 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, /* 100-109 */ + 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, /* 110-119 */ + 49, 50, 51, -1, -1, -1, -1, -1 /* 120-127 */ + }; +#define BASE64_CHAR_TO_VALUE(c) ((int) base64_char_to_value[c]) +#define IS_BASE64(c) ((IS_ASCII (c) && BASE64_CHAR_TO_VALUE (c) >= 0) || c == '=') + + const char *p = base64; + unsigned char *q = dest; + ssize_t n = 0; + + while (1) + { + unsigned char c; + unsigned long value; + + /* Process first byte of a quadruplet. */ + NEXT_CHAR (c, p); + if (!c) + break; + if (c == '=' || !IS_BASE64 (c)) + return -1; /* illegal char while decoding base64 */ + value = BASE64_CHAR_TO_VALUE (c) << 18; + + /* Process second byte of a quadruplet. */ + NEXT_CHAR (c, p); + if (!c) + return -1; /* premature EOF while decoding base64 */ + if (c == '=' || !IS_BASE64 (c)) + return -1; /* illegal char while decoding base64 */ + value |= BASE64_CHAR_TO_VALUE (c) << 12; + if (size) + { + *q++ = value >> 16; + size--; + } + n++; + + /* Process third byte of a quadruplet. */ + NEXT_CHAR (c, p); + if (!c) + return -1; /* premature EOF while decoding base64 */ + if (!IS_BASE64 (c)) + return -1; /* illegal char while decoding base64 */ + + if (c == '=') + { + NEXT_CHAR (c, p); + if (!c) + return -1; /* premature EOF while decoding base64 */ + if (c != '=') + return -1; /* padding `=' expected but not found */ + continue; + } + + value |= BASE64_CHAR_TO_VALUE (c) << 6; + if (size) + { + *q++ = 0xff & value >> 8; + size--; + } + n++; + + /* Process fourth byte of a quadruplet. */ + NEXT_CHAR (c, p); + if (!c) + return -1; /* premature EOF while decoding base64 */ + if (c == '=') + continue; + if (!IS_BASE64 (c)) + return -1; /* illegal char while decoding base64 */ + + value |= BASE64_CHAR_TO_VALUE (c); + if (size) + { + *q++ = 0xff & value; + size--; + } + n++; + } +#undef IS_BASE64 +#undef BASE64_CHAR_TO_VALUE + + return n; +} + +#ifdef HAVE_LIBPCRE2 +/* Compiles the PCRE2 regex. */ +void * +compile_pcre2_regex (const char *str) +{ + int errornumber; + PCRE2_SIZE erroroffset; + pcre2_code *regex = pcre2_compile((PCRE2_SPTR) str, PCRE2_ZERO_TERMINATED, 0, &errornumber, &erroroffset, NULL); + if (! regex) + { + fprintf (stderr, _("Invalid regular expression %s, PCRE2 error %d\n"), + quote (str), errornumber); + } + return regex; +} +#endif + +#ifdef HAVE_LIBPCRE +/* Compiles the PCRE regex. */ +void * +compile_pcre_regex (const char *str) +{ + const char *errbuf; + int erroffset; + pcre *regex = pcre_compile (str, 0, &errbuf, &erroffset, 0); + if (! regex) + { + fprintf (stderr, _("Invalid regular expression %s, %s\n"), + quote (str), errbuf); + } + return regex; +} +#endif + +/* Compiles the POSIX regex. */ +void * +compile_posix_regex (const char *str) +{ + regex_t *regex = xmalloc (sizeof (regex_t)); +#ifdef TESTING + /* regcomp might be *very* cpu+memory intensive, + * see https://sourceware.org/glibc/wiki/Security%20Exceptions */ + str = "a"; +#endif + int errcode = regcomp ((regex_t *) regex, str, REG_EXTENDED | REG_NOSUB); + if (errcode != 0) + { + size_t errbuf_size = regerror (errcode, (regex_t *) regex, NULL, 0); + char *errbuf = xmalloc (errbuf_size); + regerror (errcode, (regex_t *) regex, errbuf, errbuf_size); + fprintf (stderr, _("Invalid regular expression %s, %s\n"), + quote (str), errbuf); + xfree (errbuf); + xfree (regex); + return NULL; + } + + return regex; +} + +#ifdef HAVE_LIBPCRE2 +/* Matches a PCRE2 regex. */ +bool +match_pcre2_regex (const void *regex, const char *str) +{ + int rc; + pcre2_match_data *match_data; + + match_data = pcre2_match_data_create_from_pattern(regex, NULL); + + if (match_data) + { + rc = pcre2_match(regex, (PCRE2_SPTR) str, strlen(str), 0, 0, match_data, NULL); + pcre2_match_data_free(match_data); + } + else + rc = PCRE2_ERROR_NOMEMORY; + + if (rc < 0 && rc != PCRE2_ERROR_NOMATCH) + { + logprintf (LOG_VERBOSE, _("Error while matching %s: %d\n"), + quote (str), rc); + } + + return rc >= 0; +} +#endif + +#ifdef HAVE_LIBPCRE +#define OVECCOUNT 30 +/* Matches a PCRE regex. */ +bool +match_pcre_regex (const void *regex, const char *str) +{ + size_t l = strlen (str); + int ovector[OVECCOUNT]; + + int rc = pcre_exec ((pcre *) regex, 0, str, (int) l, 0, 0, ovector, OVECCOUNT); + if (rc == PCRE_ERROR_NOMATCH) + return false; + else if (rc < 0) + { + logprintf (LOG_VERBOSE, _("Error while matching %s: %d\n"), + quote (str), rc); + return false; + } + else + return true; +} +#undef OVECCOUNT +#endif + +/* Matches a POSIX regex. */ +bool +match_posix_regex (const void *regex, const char *str) +{ + int rc = regexec ((regex_t *) regex, str, 0, NULL, 0); + if (rc == REG_NOMATCH) + return false; + else if (rc == 0) + return true; + else + { + size_t errbuf_size = regerror (rc, opt.acceptregex, NULL, 0); + char *errbuf = xmalloc (errbuf_size); + regerror (rc, opt.acceptregex, errbuf, errbuf_size); + logprintf (LOG_VERBOSE, _("Error while matching %s: %d\n"), + quote (str), rc); + xfree (errbuf); + return false; + } +} + +#undef IS_ASCII +#undef NEXT_CHAR + +/* Simple merge sort for use by stable_sort. Implementation courtesy + Zeljko Vrba with additional debugging by Nenad Barbutov. */ + +static void +mergesort_internal (void *base, void *temp, size_t size, size_t from, size_t to, + int (*cmpfun) (const void *, const void *)) +{ +#define ELT(array, pos) ((char *)(array) + (pos) * size) + if (from < to) + { + size_t i, j, k; + size_t mid = (to + from) / 2; + mergesort_internal (base, temp, size, from, mid, cmpfun); + mergesort_internal (base, temp, size, mid + 1, to, cmpfun); + i = from; + j = mid + 1; + for (k = from; (i <= mid) && (j <= to); k++) + if (cmpfun (ELT (base, i), ELT (base, j)) <= 0) + memcpy (ELT (temp, k), ELT (base, i++), size); + else + memcpy (ELT (temp, k), ELT (base, j++), size); + while (i <= mid) + memcpy (ELT (temp, k++), ELT (base, i++), size); + while (j <= to) + memcpy (ELT (temp, k++), ELT (base, j++), size); + for (k = from; k <= to; k++) + memcpy (ELT (base, k), ELT (temp, k), size); + } +#undef ELT +} + +/* Stable sort with interface exactly like standard library's qsort. + Uses mergesort internally. */ + +void +stable_sort (void *base, size_t nmemb, size_t size, + int (*cmpfun) (const void *, const void *)) +{ + if (nmemb > 1 && size > 1) + { + void *temp = xmalloc (nmemb * size); + mergesort_internal (base, temp, size, 0, nmemb - 1, cmpfun); + xfree(temp); + } +} + +/* Print a decimal number. If it is equal to or larger than ten, the + number is rounded. Otherwise it is printed with one significant + digit without trailing zeros and with no more than three fractional + digits total. For example, 0.1 is printed as "0.1", 0.035 is + printed as "0.04", 0.0091 as "0.009", and 0.0003 as simply "0". + + This is useful for displaying durations because it provides + order-of-magnitude information without unnecessary clutter -- + long-running downloads are shown without the fractional part, and + short ones still retain one significant digit. */ + +const char * +print_decimal (double number) +{ + static char buf[32]; + double n = number >= 0 ? number : -number; + + if (n >= 9.95) + /* Cut off at 9.95 because the below %.1f would round 9.96 to + "10.0" instead of "10". OTOH 9.94 will print as "9.9". */ + snprintf (buf, sizeof buf, "%.0f", number); + else if (n >= 0.95) + snprintf (buf, sizeof buf, "%.1f", number); + else if (n >= 0.001) + snprintf (buf, sizeof buf, "%.1g", number); + else if (n >= 0.0005) + /* round [0.0005, 0.001) to 0.001 */ + snprintf (buf, sizeof buf, "%.3f", number); + else + /* print numbers close to 0 as 0, not 0.000 */ + strcpy (buf, "0"); + + return buf; +} + +/* Get the maximum name length for the given path. */ +/* Return 0 if length is unknown. */ +long +get_max_length (const char *path, int length, int name) +{ + long ret; + char *p, *d; + + /* Make a copy of the path that we can modify. */ + p = path ? strdupdelim (path, path + length) : strdup (""); + + for (;;) + { + errno = 0; + /* For an empty path query the current directory. */ +#if HAVE_PATHCONF + ret = pathconf (*p ? p : ".", name); + if (!(ret < 0 && errno == ENOENT)) + break; +#else + ret = PATH_MAX; +#endif + + /* The path does not exist yet, but may be created. */ + /* Already at current or root directory, give up. */ + if (!*p || strcmp (p, "/") == 0) + break; + + /* Remove one directory level and try again. */ + d = strrchr (p, '/'); + if (d == p) + p[1] = '\0'; /* check root directory */ + else if (d) + *d = '\0'; /* remove last directory part */ + else + *p = '\0'; /* check current directory */ + } + + xfree (p); + + if (ret < 0) + { + /* pathconf() has a message for us. */ + if (errno != 0) + perror ("pathconf"); + + /* If (errno == 0) then there is no max length. + Even on error return 0 so the caller can continue. */ + return 0; + } + + return ret; +} + +void +wg_hex_to_string (char *str_buffer, const char *hex_buffer, size_t hex_len) +{ + size_t i; + + for (i = 0; i < hex_len; i++) + { + /* Each byte takes 2 characters. */ + sprintf (str_buffer + 2 * i, "%02x", (unsigned) (hex_buffer[i] & 0xFF)); + } + + /* Null-terminate result. */ + str_buffer[2 * i] = '\0'; +} + +#ifdef HAVE_SSL + +/* + * Public key pem to der conversion + */ + +static bool +wg_pubkey_pem_to_der (const char *pem, unsigned char **der, size_t *der_len) +{ + char *stripped_pem, *begin_pos, *end_pos; + size_t pem_count, stripped_pem_count = 0, pem_len; + ssize_t size; + unsigned char *base64data; + + *der = NULL; + *der_len = 0; + + /* if no pem, exit. */ + if (!pem) + return false; + + begin_pos = strstr (pem, "-----BEGIN PUBLIC KEY-----"); + if (!begin_pos) + return false; + + pem_count = begin_pos - pem; + /* Invalid if not at beginning AND not directly following \n */ + if (0 != pem_count && '\n' != pem[pem_count - 1]) + return false; + + /* 26 is length of "-----BEGIN PUBLIC KEY-----" */ + pem_count += 26; + + /* Invalid if not directly following \n */ + end_pos = strstr (pem + pem_count, "\n-----END PUBLIC KEY-----"); + if (!end_pos) + return false; + + pem_len = end_pos - pem; + + stripped_pem = xmalloc (pem_len - pem_count + 1); + + /* + * Here we loop through the pem array one character at a time between the + * correct indices, and place each character that is not '\n' or '\r' + * into the stripped_pem array, which should represent the raw base64 string + */ + while (pem_count < pem_len) { + if ('\n' != pem[pem_count] && '\r' != pem[pem_count]) + stripped_pem[stripped_pem_count++] = pem[pem_count]; + ++pem_count; + } + /* Place the null terminator in the correct place */ + stripped_pem[stripped_pem_count] = '\0'; + + base64data = xmalloc (BASE64_LENGTH(stripped_pem_count)); + + size = wget_base64_decode (stripped_pem, base64data, BASE64_LENGTH(stripped_pem_count)); + + if (size < 0) { + xfree (base64data); /* malformed base64 from server */ + } else { + *der = base64data; + *der_len = (size_t) size; + } + + xfree (stripped_pem); + + return *der_len > 0; +} + +/* + * Generic pinned public key check. + */ + +bool +wg_pin_peer_pubkey (const char *pinnedpubkey, const char *pubkey, size_t pubkeylen) +{ + struct file_memory *fm; + unsigned char *buf = NULL, *pem_ptr = NULL; + size_t size, pem_len; + bool pem_read; + bool result = false; + + size_t pinkeylen; + ssize_t decoded_hash_length; + char *pinkeycopy, *begin_pos, *end_pos; + unsigned char *sha256sumdigest = NULL, *expectedsha256sumdigest = NULL; + + /* if a path wasn't specified, don't pin */ + if (!pinnedpubkey) + return true; + if (!pubkey || !pubkeylen) + return result; + + /* only do this if pinnedpubkey starts with "sha256//", length 8 */ + if (strncmp (pinnedpubkey, "sha256//", 8) == 0) + { + /* compute sha256sum of public key */ + sha256sumdigest = xmalloc (SHA256_DIGEST_SIZE); + sha256_buffer (pubkey, pubkeylen, sha256sumdigest); + expectedsha256sumdigest = xmalloc (SHA256_DIGEST_SIZE); + + /* it starts with sha256//, copy so we can modify it */ + pinkeylen = strlen (pinnedpubkey) + 1; + pinkeycopy = xmalloc (pinkeylen); + memcpy (pinkeycopy, pinnedpubkey, pinkeylen); + + /* point begin_pos to the copy, and start extracting keys */ + begin_pos = pinkeycopy; + do + { + end_pos = strstr (begin_pos, ";sha256//"); + /* + * if there is an end_pos, null terminate, + * otherwise it'll go to the end of the original string + */ + if (end_pos) + end_pos[0] = '\0'; + + /* decode base64 pinnedpubkey, 8 is length of "sha256//" */ + decoded_hash_length = wget_base64_decode (begin_pos + 8, expectedsha256sumdigest, SHA256_DIGEST_SIZE); + + /* if valid base64, compare sha256 digests directly */ + if (SHA256_DIGEST_SIZE == decoded_hash_length) + { + if (!memcmp (sha256sumdigest, expectedsha256sumdigest, SHA256_DIGEST_SIZE)) + { + result = true; + break; + } + } + else + logprintf (LOG_VERBOSE, _ ("Skipping key with wrong size (%d/%d): %s\n"), + (int) (strlen (begin_pos + 8) * 3) / 4, SHA256_DIGEST_SIZE, + quote (begin_pos + 8)); + + /* + * change back the null-terminator we changed earlier, + * and look for next begin + */ + if (end_pos) + { + end_pos[0] = ';'; + begin_pos = strstr (end_pos, "sha256//"); + } + } + while (end_pos && begin_pos); + + xfree (sha256sumdigest); + xfree (expectedsha256sumdigest); + xfree (pinkeycopy); + + return result; + } + + /* fall back to assuming this is a file path */ + fm = wget_read_file (pinnedpubkey); + if (!fm) + return result; + + /* Check the file's size */ + if (fm->length < 0 || fm->length > MAX_PINNED_PUBKEY_SIZE) + goto cleanup; + + /* + * if the size of our certificate is bigger than the file + * size then it can't match + */ + size = (size_t) fm->length; + if (pubkeylen > size) + goto cleanup; + + /* If the sizes are the same, it can't be base64 encoded, must be der */ + if (pubkeylen == size) + { + if (!memcmp (pubkey, fm->content, pubkeylen)) + result = true; + goto cleanup; + } + + /* + * Otherwise we will assume it's PEM and try to decode it + * after placing null terminator + */ + buf = xmalloc (size + 1); + memcpy (buf, fm->content, size); + buf[size] = '\0'; + + pem_read = wg_pubkey_pem_to_der ((const char *) buf, &pem_ptr, &pem_len); + /* if it wasn't read successfully, exit */ + if (!pem_read) + goto cleanup; + + /* + * if the size of our certificate doesn't match the size of + * the decoded file, they can't be the same, otherwise compare + */ + if (pubkeylen == pem_len && !memcmp (pubkey, pem_ptr, pubkeylen)) + result = true; + +cleanup: + xfree (buf); + xfree (pem_ptr); + wget_read_file_free (fm); + + return result; +} + +#endif /* HAVE_SSL */ + +#ifdef TESTING + +const char * +test_subdir_p(void) +{ + static const struct { + const char *d1; + const char *d2; + bool result; + } test_array[] = { + { "/somedir", "/somedir", true }, + { "/somedir", "/somedir/d2", true }, + { "/somedir/d1", "/somedir", false }, + }; + unsigned i; + + for (i = 0; i < countof(test_array); ++i) + { + bool res = subdir_p (test_array[i].d1, test_array[i].d2); + + mu_assert ("test_subdir_p: wrong result", + res == test_array[i].result); + } + + return NULL; +} + +const char * +test_dir_matches_p(void) +{ + static struct { + const char *dirlist[3]; + const char *dir; + bool result; + } test_array[] = { + { { "/somedir", "/someotherdir", NULL }, "somedir", true }, + { { "/somedir", "/someotherdir", NULL }, "anotherdir", false }, + { { "/somedir", "/*otherdir", NULL }, "anotherdir", true }, + { { "/somedir/d1", "/someotherdir", NULL }, "somedir/d1", true }, + { { "*/*d1", "/someotherdir", NULL }, "somedir/d1", true }, + { { "/somedir/d1", "/someotherdir", NULL }, "d1", false }, + { { "!COMPLETE", NULL, NULL }, "!COMPLETE", true }, + { { "*COMPLETE", NULL, NULL }, "!COMPLETE", true }, + { { "*/!COMPLETE", NULL, NULL }, "foo/!COMPLETE", true }, + { { "*COMPLETE", NULL, NULL }, "foo/!COMPLETE", false }, + { { "*/*COMPLETE", NULL, NULL }, "foo/!COMPLETE", true }, + { { "/dir with spaces", NULL, NULL }, "dir with spaces", true }, + { { "/dir*with*spaces", NULL, NULL }, "dir with spaces", true }, + { { "/Tmp/has", NULL, NULL }, "/Tmp/has space", false }, + { { "/Tmp/has", NULL, NULL }, "/Tmp/has,comma", false }, + }; + unsigned i; + + for (i = 0; i < countof(test_array); ++i) + { + bool res = dir_matches_p (test_array[i].dirlist, test_array[i].dir); + + mu_assert ("test_dir_matches_p: wrong result", + res == test_array[i].result); + } + + return NULL; +} + +#endif /* TESTING */ diff --git a/src/utils.h b/src/utils.h new file mode 100644 index 0000000..b1ab901 --- /dev/null +++ b/src/utils.h @@ -0,0 +1,179 @@ +/* Declarations for utils.c. + Copyright (C) 1996-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef UTILS_H +#define UTILS_H + +#include <stdlib.h> +#include <wget.h> + +/* Constant is using when we don`t know attempted size exactly */ +#define UNKNOWN_ATTEMPTED_SIZE -3 + +#ifndef MAX_PINNED_PUBKEY_SIZE +#define MAX_PINNED_PUBKEY_SIZE 1048576 /* 1MB */ +#endif + +/* Macros that interface to malloc, but know about type sizes, and + cast the result to the appropriate type. The casts are not + necessary in standard C, but Wget performs them anyway for the sake + of pre-standard environments and possibly C++. */ + +#define xnew(type) (xmalloc (sizeof (type))) +#define xnew0(type) (xcalloc (1, sizeof (type))) +#define xnew_array(type, len) (xmalloc ((len) * sizeof (type))) +#define xnew0_array(type, len) (xcalloc ((len), sizeof (type))) + +#define xfree(p) do { free ((void *) (p)); p = NULL; } while (0) + +struct hash_table; + +struct file_memory { + char *content; + long length; + int mmap_p; +}; + +#define HYPHENP(x) (*(x) == '-' && !*((x) + 1)) + +char *time_str (time_t); +char *datetime_str (time_t); + +char *xstrdup_lower (const char *); + +char *strdupdelim (const char *, const char *); +char **sepstring (const char *); +bool subdir_p (const char *, const char *); +bool fork_to_background (void); + +char *aprintf (const char *, ...) GCC_FORMAT_ATTR (1, 2); +char *concat_strings (const char *, ...); + +typedef struct file_stat_s { + int access_err; /* Error in accecssing file : Not present vs permission */ + ino_t st_ino; /* st_ino from stats() on the file before open() */ + dev_t st_dev; /* st_dev from stats() on the file before open() */ +} file_stats_t; + +void touch (const char *, time_t); +int remove_link (const char *); +bool file_exists_p (const char *, file_stats_t *); +bool file_non_directory_p (const char *); +wgint file_size (const char *); +int make_directory (const char *); +char *unique_name_passthrough (const char *); +char *unique_name (const char *); +FILE *unique_create (const char *, bool, char **); +FILE *fopen_excl (const char *, int); +FILE *fopen_stat (const char *, const char *, file_stats_t *); +int open_stat (const char *, int, mode_t, file_stats_t *); +char *file_merge (const char *, const char *); + +int fnmatch_nocase (const char *, const char *, int); +bool acceptable (const char *); +bool accept_url (const char *); +bool accdir (const char *s); +char *suffix (const char *s); +bool match_tail (const char *, const char *, bool); +bool has_wildcards_p (const char *); + +bool has_html_suffix_p (const char *); + +struct file_memory *wget_read_file (const char *); +void wget_read_file_free (struct file_memory *); + +void free_vec (char **); +char **merge_vecs (char **, char **); +char **vec_append (char **, const char *); + +void string_set_add (struct hash_table *, const char *); +int string_set_contains (struct hash_table *, const char *); +void string_set_to_array (struct hash_table *, char **); +void string_set_free (struct hash_table *); +void free_keys_and_values (struct hash_table *); + +const char *with_thousand_seps (wgint); + +/* human_readable must be able to accept wgint and SUM_SIZE_INT + arguments. */ +char *human_readable (wgint, const int, const int); + + +int numdigit (wgint); +char *number_to_string (char *, wgint); +char *number_to_static_string (wgint); +wgint convert_to_bits (wgint); + +int determine_screen_width (void); +int random_number (int); +double random_float (void); + +bool run_with_timeout (double, void (*) (void *), void *); +void xsleep (double); + +/* How many bytes it will take to store LEN bytes in base64. */ +#define BASE64_LENGTH(len) (4 * (((len) + 2) / 3)) + +size_t wget_base64_encode (const void *, size_t, char *); +ssize_t wget_base64_decode (const char *, void *, size_t); + +#ifdef HAVE_LIBPCRE2 +void *compile_pcre2_regex (const char *); +bool match_pcre2_regex (const void *, const char *); +#endif + +#ifdef HAVE_LIBPCRE +void *compile_pcre_regex (const char *); +bool match_pcre_regex (const void *, const char *); +#endif + +void *compile_posix_regex (const char *); +bool match_posix_regex (const void *, const char *); + +void stable_sort (void *, size_t, size_t, int (*) (const void *, const void *)); + +const char *print_decimal (double); + +long get_max_length (const char *path, int length, int name); + +#ifndef HAVE_STRLCPY +size_t strlcpy (char *dst, const char *src, size_t size); +#endif + +void wg_hex_to_string (char *str_buffer, const char *hex_buffer, size_t hex_len); + +extern unsigned char char_prop[]; + +#ifdef HAVE_SSL +/* Check pinned public key. */ +bool wg_pin_peer_pubkey (const char *pinnedpubkey, const char *pubkey, size_t pubkeylen); +#endif + +#endif /* UTILS_H */ diff --git a/src/version.h b/src/version.h new file mode 100644 index 0000000..49fb931 --- /dev/null +++ b/src/version.h @@ -0,0 +1,41 @@ +/* Extern declarations for printing version information + Copyright (C) 2013, 2015, 2018-2020 Free Software Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#ifndef WGET_VERSION_H +#define WGET_VERSION_H + +/* Extern declarations for strings in version.c */ +extern const char *version_string; +extern const char *compilation_string; +extern const char *link_string; + +/* Extern declaration for string in build_info.c */ +extern const char *compiled_features[]; + +#endif /* WGET_VERSION_H */ diff --git a/src/warc.c b/src/warc.c new file mode 100644 index 0000000..f602444 --- /dev/null +++ b/src/warc.c @@ -0,0 +1,1662 @@ +/* Utility functions for writing WARC files. + Copyright (C) 2011-2012, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" +#include "hash.h" +#include "utils.h" +#include "version.h" +#include "dirname.h" +#include "url.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <tmpdir.h> +#include <sha1.h> +#include <base32.h> +#include <unistd.h> +#ifdef HAVE_LIBZ +#include <zlib.h> +#endif + +#ifdef HAVE_LIBUUID +#include <uuid/uuid.h> +#elif HAVE_UUID_CREATE +#include <uuid.h> +#endif + +#include "warc.h" +#include "exits.h" + +#ifdef WINDOWS +/* we need this on Windows to have O_TEMPORARY defined */ +# include <fcntl.h> +# include <rpc.h> +#endif + +#ifndef O_TEMPORARY +#define O_TEMPORARY 0 +#endif + +#include "warc.h" +#include "exits.h" + + +/* The log file (a temporary file that contains a copy + of the wget log). */ +static FILE *warc_log_fp; + +/* The manifest file (a temporary file that contains the + warcinfo uuid of every file in this crawl). */ +static FILE *warc_manifest_fp; + +/* The current WARC file (or NULL, if WARC is disabled). */ +static FILE *warc_current_file; + +#ifdef HAVE_LIBZ +/* The gzip stream for the current WARC file + (or NULL, if WARC or gzip is disabled). */ +static gzFile warc_current_gzfile; + +/* The offset of the current gzip record in the WARC file. */ +static off_t warc_current_gzfile_offset; + +/* The uncompressed size (so far) of the current record. */ +static off_t warc_current_gzfile_uncompressed_size; +# endif + +/* This is true until a warc_write_* method fails. */ +static bool warc_write_ok; + +/* The current CDX file (or NULL, if CDX is disabled). */ +static FILE *warc_current_cdx_file; + +/* The record id of the warcinfo record of the current WARC file. */ +static char warc_current_warcinfo_uuid_str[48]; + +/* The file name of the current WARC file. */ +static char *warc_current_filename; + +/* The serial number of the current WARC file. This number is + incremented each time a new file is opened and is used in the + WARC file's filename. */ +static int warc_current_file_number; + +/* The table of CDX records, if deduplication is enabled. */ +static struct hash_table * warc_cdx_dedup_table; + +static bool warc_start_new_file (bool meta); + + +struct warc_cdx_record +{ + char *url; + char *uuid; + char digest[SHA1_DIGEST_SIZE]; +}; + +static unsigned long +warc_hash_sha1_digest (const void *key) +{ + /* We just use some of the first bytes of the digest. */ + unsigned long v = 0; + memcpy (&v, key, sizeof (unsigned long)); + return v; +} + +static int +warc_cmp_sha1_digest (const void *digest1, const void *digest2) +{ + return !memcmp (digest1, digest2, SHA1_DIGEST_SIZE); +} + + + +/* Writes SIZE bytes from BUFFER to the current WARC file, + through gzwrite if compression is enabled. + Returns the number of uncompressed bytes written. */ +static size_t +warc_write_buffer (const char *buffer, size_t size) +{ +#ifdef HAVE_LIBZ + if (warc_current_gzfile) + { + warc_current_gzfile_uncompressed_size += size; + return gzwrite (warc_current_gzfile, buffer, size); + } + else +#endif + return fwrite (buffer, 1, size, warc_current_file); +} + +/* Writes STR to the current WARC file. + Returns false and set warc_write_ok to false if there + is an error. */ +static bool +warc_write_string (const char *str) +{ + size_t n; + + if (!warc_write_ok) + return false; + + n = strlen (str); + if (n != warc_write_buffer (str, n)) + warc_write_ok = false; + + return warc_write_ok; +} + + +#define EXTRA_GZIP_HEADER_SIZE 14 +#define GZIP_STATIC_HEADER_SIZE 10 +#define FLG_FEXTRA 0x04 +#define OFF_FLG 3 + +/* Starts a new WARC record. Writes the version header. + If opt.warc_maxsize is set and the current file is becoming + too large, this will open a new WARC file. + + If compression is enabled, this will start a new + gzip stream in the current WARC file. + + Returns false and set warc_write_ok to false if there + is an error. */ +static bool +warc_write_start_record (void) +{ + if (!warc_write_ok) + return false; + + fflush (warc_current_file); + if (opt.warc_maxsize > 0 && ftello (warc_current_file) >= opt.warc_maxsize) + warc_start_new_file (false); + +#ifdef HAVE_LIBZ + /* Start a GZIP stream, if required. */ + if (opt.warc_compression_enabled) + { + int dup_fd; + /* Record the starting offset of the new record. */ + warc_current_gzfile_offset = ftello (warc_current_file); + + /* Reserve space for the extra GZIP header field. + In warc_write_end_record we will fill this space + with information about the uncompressed and + compressed size of the record. */ + if (fseek (warc_current_file, EXTRA_GZIP_HEADER_SIZE, SEEK_CUR) < 0) + { + logprintf (LOG_NOTQUIET, _("Error setting WARC file position.\n")); + warc_write_ok = false; + return false; + } + + if (fflush (warc_current_file) != 0) + { + logprintf (LOG_NOTQUIET, _("Error flushing WARC file to disk.\n")); + warc_write_ok = false; + return false; + } + + /* Start a new GZIP stream. */ + dup_fd = dup (fileno (warc_current_file)); + if (dup_fd < 0) + { + logprintf (LOG_NOTQUIET, +_("Error duplicating WARC file file descriptor.\n")); + warc_write_ok = false; + return false; + } + + warc_current_gzfile = gzdopen (dup_fd, "wb9"); + warc_current_gzfile_uncompressed_size = 0; + + if (warc_current_gzfile == NULL) + { + logprintf (LOG_NOTQUIET, +_("Error opening GZIP stream to WARC file.\n")); + close (dup_fd); + warc_write_ok = false; + return false; + } + } +#endif + + warc_write_string ("WARC/1.0\r\n"); + return warc_write_ok; +} + +/* Writes a WARC header to the current WARC record. + This method may be run after warc_write_start_record and + before warc_write_block_from_file. */ +static bool +warc_write_header (const char *name, const char *value) +{ + if (value) + { + warc_write_string (name); + warc_write_string (": "); + warc_write_string (value); + warc_write_string ("\r\n"); + } + return warc_write_ok; +} + +/* Writes a WARC header with a URI as value to the current WARC record. + This method may be run after warc_write_start_record and + before warc_write_block_from_file. */ +static bool +warc_write_header_uri (const char *name, const char *value) +{ + if (value) + { + warc_write_string (name); + warc_write_string (": <"); + warc_write_string (value); + warc_write_string (">\r\n"); + } + return warc_write_ok; +} + +/* Copies the contents of DATA_IN to the WARC record. + Adds a Content-Length header to the WARC record. + Run this method after warc_write_header, + then run warc_write_end_record. */ +static bool +warc_write_block_from_file (FILE *data_in) +{ + /* Add the Content-Length header. */ + char content_length[MAX_INT_TO_STRING_LEN(off_t)]; + char buffer[BUFSIZ]; + size_t s; + + fseeko (data_in, 0L, SEEK_END); + number_to_string (content_length, ftello (data_in)); + warc_write_header ("Content-Length", content_length); + + /* End of the WARC header section. */ + warc_write_string ("\r\n"); + + if (fseeko (data_in, 0L, SEEK_SET) != 0) + warc_write_ok = false; + + /* Copy the data in the file to the WARC record. */ + while (warc_write_ok && (s = fread (buffer, 1, BUFSIZ, data_in)) > 0) + { + if (warc_write_buffer (buffer, s) < s) + warc_write_ok = false; + } + + return warc_write_ok; +} + +/* Run this method to close the current WARC record. + + If compression is enabled, this method closes the + current GZIP stream and fills the extra GZIP header + with the uncompressed and compressed length of the + record. */ +static bool +warc_write_end_record (void) +{ + if (warc_write_buffer ("\r\n\r\n", 4) != 4) + { + warc_write_ok = false; + return false; + } + +#ifdef HAVE_LIBZ + /* We start a new gzip stream for each record. */ + if (warc_write_ok && warc_current_gzfile) + { + char extra_header[EXTRA_GZIP_HEADER_SIZE]; + char static_header[GZIP_STATIC_HEADER_SIZE]; + off_t current_offset, uncompressed_size, compressed_size; + size_t result; + + if (gzclose (warc_current_gzfile) != Z_OK) + { + warc_write_ok = false; + return false; + } + + fflush (warc_current_file); + fseeko (warc_current_file, 0, SEEK_END); + + /* The WARC standard suggests that we add 'skip length' data in the + extra header field of the GZIP stream. + + In warc_write_start_record we reserved space for this extra header. + This extra space starts at warc_current_gzfile_offset and fills + EXTRA_GZIP_HEADER_SIZE bytes. The static GZIP header starts at + warc_current_gzfile_offset + EXTRA_GZIP_HEADER_SIZE. + + We need to do three things: + 1. Move the static GZIP header to warc_current_gzfile_offset; + 2. Set the FEXTRA flag in the GZIP header; + 3. Write the extra GZIP header after the static header, that is, + starting at warc_current_gzfile_offset + GZIP_STATIC_HEADER_SIZE. + */ + + /* Calculate the uncompressed and compressed sizes. */ + current_offset = ftello (warc_current_file); + uncompressed_size = current_offset - warc_current_gzfile_offset; + compressed_size = warc_current_gzfile_uncompressed_size; + + /* Go back to the static GZIP header. */ + result = fseeko (warc_current_file, warc_current_gzfile_offset + + EXTRA_GZIP_HEADER_SIZE, SEEK_SET); + if (result != 0) + { + warc_write_ok = false; + return false; + } + + /* Read the header. */ + result = fread (static_header, 1, GZIP_STATIC_HEADER_SIZE, + warc_current_file); + if (result != GZIP_STATIC_HEADER_SIZE) + { + warc_write_ok = false; + return false; + } + + /* Set the FEXTRA flag in the flags byte of the header. */ + static_header[OFF_FLG] = static_header[OFF_FLG] | FLG_FEXTRA; + + /* Write the header back to the file, but starting at + warc_current_gzfile_offset. */ + fseeko (warc_current_file, warc_current_gzfile_offset, SEEK_SET); + fwrite (static_header, 1, GZIP_STATIC_HEADER_SIZE, warc_current_file); + + /* Prepare the extra GZIP header. */ + /* XLEN, the length of the extra header fields. */ + extra_header[0] = ((EXTRA_GZIP_HEADER_SIZE - 2) & 255); + extra_header[1] = ((EXTRA_GZIP_HEADER_SIZE - 2) >> 8) & 255; + /* The extra header field identifier for the WARC skip length. */ + extra_header[2] = 's'; + extra_header[3] = 'l'; + /* The size of the field value (8 bytes). */ + extra_header[4] = (8 & 255); + extra_header[5] = ((8 >> 8) & 255); + /* The size of the uncompressed record. */ + extra_header[6] = (uncompressed_size & 255); + extra_header[7] = (uncompressed_size >> 8) & 255; + extra_header[8] = (uncompressed_size >> 16) & 255; + extra_header[9] = (uncompressed_size >> 24) & 255; + /* The size of the compressed record. */ + extra_header[10] = (compressed_size & 255); + extra_header[11] = (compressed_size >> 8) & 255; + extra_header[12] = (compressed_size >> 16) & 255; + extra_header[13] = (compressed_size >> 24) & 255; + + /* Write the extra header after the static header. */ + fseeko (warc_current_file, warc_current_gzfile_offset + + GZIP_STATIC_HEADER_SIZE, SEEK_SET); + fwrite (extra_header, 1, EXTRA_GZIP_HEADER_SIZE, warc_current_file); + + /* Done, move back to the end of the file. */ + fflush (warc_current_file); + fseeko (warc_current_file, 0, SEEK_END); + } +#endif /* HAVE_LIBZ */ + + return warc_write_ok; +} + + +/* Writes the WARC-Date header for the given timestamp to + the current WARC record. + If timestamp is NULL, the current time will be used. */ +static bool +warc_write_date_header (const char *timestamp) +{ + char current_timestamp[21]; + + return warc_write_header ("WARC-Date", timestamp ? timestamp : + warc_timestamp (current_timestamp, sizeof(current_timestamp))); +} + +/* Writes the WARC-IP-Address header for the given IP to + the current WARC record. If IP is NULL, no header will + be written. */ +static bool +warc_write_ip_header (const ip_address *ip) +{ + if (ip != NULL) + return warc_write_header ("WARC-IP-Address", print_address (ip)); + else + return warc_write_ok; +} + + +/* warc_sha1_stream_with_payload is a modified copy of sha1_stream + from gnulib/sha1.c. This version calculates two digests in one go. + + Compute SHA1 message digests for bytes read from STREAM. The + digest of the complete file will be written into the 16 bytes + beginning at RES_BLOCK. + + If payload_offset >= 0, a second digest will be calculated of the + portion of the file starting at payload_offset and continuing to + the end of the file. The digest number will be written into the + 16 bytes beginning ad RES_PAYLOAD. */ +static int +warc_sha1_stream_with_payload (FILE *stream, void *res_block, void *res_payload, + off_t payload_offset) +{ +#define BLOCKSIZE 32768 + + struct sha1_ctx ctx_block; + struct sha1_ctx ctx_payload; + off_t pos; + off_t sum; + + char *buffer = xmalloc (BLOCKSIZE + 72); + + /* Initialize the computation context. */ + sha1_init_ctx (&ctx_block); + if (payload_offset >= 0) + sha1_init_ctx (&ctx_payload); + + pos = 0; + + /* Iterate over full file contents. */ + while (1) + { + /* We read the file in blocks of BLOCKSIZE bytes. One call of the + computation function processes the whole buffer so that with the + next round of the loop another block can be read. */ + off_t n; + sum = 0; + + /* Read block. Take care for partial reads. */ + while (1) + { + n = fread (buffer + sum, 1, BLOCKSIZE - sum, stream); + + sum += n; + pos += n; + + if (sum == BLOCKSIZE) + break; + + if (n == 0) + { + /* Check for the error flag IF N == 0, so that we don't + exit the loop after a partial read due to e.g., EAGAIN + or EWOULDBLOCK. */ + if (ferror (stream)) + { + xfree (buffer); + return 1; + } + goto process_partial_block; + } + + /* We've read at least one byte, so ignore errors. But always + check for EOF, since feof may be true even though N > 0. + Otherwise, we could end up calling fread after EOF. */ + if (feof (stream)) + goto process_partial_block; + } + + /* Process buffer with BLOCKSIZE bytes. Note that + BLOCKSIZE % 64 == 0 + */ + sha1_process_block (buffer, BLOCKSIZE, &ctx_block); + if (payload_offset >= 0 && payload_offset < pos) + { + /* At least part of the buffer contains data from payload. */ + off_t start_of_payload = payload_offset - (pos - BLOCKSIZE); + if (start_of_payload <= 0) + /* All bytes in the buffer belong to the payload. */ + start_of_payload = 0; + + /* Process the payload part of the buffer. + Note: we can't use sha1_process_block here even if we + process the complete buffer. Because the payload doesn't + have to start with a full block, there may still be some + bytes left from the previous buffer. Therefore, we need + to continue with sha1_process_bytes. */ + sha1_process_bytes (buffer + start_of_payload, + BLOCKSIZE - start_of_payload, &ctx_payload); + } + } + + process_partial_block:; + + /* Process any remaining bytes. */ + if (sum > 0) + { + sha1_process_bytes (buffer, sum, &ctx_block); + if (payload_offset >= 0 && payload_offset < pos) + { + /* At least part of the buffer contains data from payload. */ + off_t start_of_payload = payload_offset - (pos - sum); + if (start_of_payload <= 0) + /* All bytes in the buffer belong to the payload. */ + start_of_payload = 0; + + /* Process the payload part of the buffer. */ + sha1_process_bytes (buffer + start_of_payload, + sum - start_of_payload, &ctx_payload); + } + } + + /* Construct result in desired memory. */ + sha1_finish_ctx (&ctx_block, res_block); + if (payload_offset >= 0) + sha1_finish_ctx (&ctx_payload, res_payload); + xfree (buffer); + return 0; + +#undef BLOCKSIZE +} + +/* Converts the SHA1 digest to a base32-encoded string. + "sha1:DIGEST\0" (Allocates a new string for the response.) */ +static char * +warc_base32_sha1_digest (const char *sha1_digest, char *sha1_base32, size_t sha1_base32_size) +{ + if (sha1_base32_size >= BASE32_LENGTH(SHA1_DIGEST_SIZE) + 5 + 1) + { + memcpy (sha1_base32, "sha1:", 5); + base32_encode (sha1_digest, SHA1_DIGEST_SIZE, sha1_base32 + 5, + sha1_base32_size - 5); + } + else + *sha1_base32 = 0; + + return sha1_base32; +} + + +/* Sets the digest headers of the record. + This method will calculate the block digest and, if payload_offset >= 0, + will also calculate the payload digest of the payload starting at the + provided offset. */ +static void +warc_write_digest_headers (FILE *file, long payload_offset) +{ + if (opt.warc_digests_enabled) + { + /* Calculate the block and payload digests. */ + char sha1_res_block[SHA1_DIGEST_SIZE]; + char sha1_res_payload[SHA1_DIGEST_SIZE]; + + rewind (file); + if (warc_sha1_stream_with_payload (file, sha1_res_block, + sha1_res_payload, payload_offset) == 0) + { + char digest[BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1 + 5]; + + warc_write_header ("WARC-Block-Digest", + warc_base32_sha1_digest (sha1_res_block, digest, sizeof(digest))); + + if (payload_offset >= 0) + warc_write_header ("WARC-Payload-Digest", + warc_base32_sha1_digest (sha1_res_payload, digest, sizeof(digest))); + } + } +} + + +/* Fills timestamp with the current time and date. + The UTC time is formatted following ISO 8601, as required + for use in the WARC-Date header. + The timestamp will be 21 characters long. */ +char * +warc_timestamp (char *timestamp, size_t timestamp_size) +{ + time_t rawtime = time (NULL); + struct tm * timeinfo = gmtime (&rawtime); + + if (strftime (timestamp, timestamp_size, "%Y-%m-%dT%H:%M:%SZ", timeinfo) == 0 && timestamp_size > 0) + *timestamp = 0; + + return timestamp; +} + +/* Fills urn_str with a UUID in the format required + for the WARC-Record-Id header. + The string will be 47 characters long. */ +#if HAVE_LIBUUID +void +warc_uuid_str (char *urn_str, size_t urn_size) +{ + char uuid_str[37]; + uuid_t record_id; + + uuid_generate (record_id); + uuid_unparse (record_id, uuid_str); + + snprintf (urn_str, urn_size, "<urn:uuid:%s>", uuid_str); +} +#elif HAVE_UUID_CREATE +void +warc_uuid_str (char *urn_str, size_t urn_size) +{ + char *uuid_str; + uuid_t record_id; + + uuid_create (&record_id, NULL); + uuid_to_string (&record_id, &uuid_str, NULL); + + snprintf (urn_str, urn_size, "<urn:uuid:%s>", uuid_str); + xfree (uuid_str); +} +#else +# ifdef WINDOWS + +typedef RPC_STATUS (RPC_ENTRY * UuidCreate_proc) (UUID *); +typedef RPC_STATUS (RPC_ENTRY * UuidToString_proc) (UUID *, unsigned char **); +typedef RPC_STATUS (RPC_ENTRY * RpcStringFree_proc) (unsigned char **); + +static int +windows_uuid_str (char *urn_str, size_t urn_size) +{ + static UuidCreate_proc pfn_UuidCreate = NULL; + static UuidToString_proc pfn_UuidToString = NULL; + static RpcStringFree_proc pfn_RpcStringFree = NULL; + static int rpc_uuid_avail = -1; + + /* Rpcrt4.dll is not available on older versions of Windows, so we + need to test its availability at run time. */ + if (rpc_uuid_avail == -1) + { + HMODULE hm_rpcrt4 = LoadLibrary ("Rpcrt4.dll"); + + if (hm_rpcrt4) + { + pfn_UuidCreate = + (UuidCreate_proc) GetProcAddress (hm_rpcrt4, "UuidCreate"); + pfn_UuidToString = + (UuidToString_proc) GetProcAddress (hm_rpcrt4, "UuidToStringA"); + pfn_RpcStringFree = + (RpcStringFree_proc) GetProcAddress (hm_rpcrt4, "RpcStringFreeA"); + if (pfn_UuidCreate && pfn_UuidToString && pfn_RpcStringFree) + rpc_uuid_avail = 1; + else + rpc_uuid_avail = 0; + } + else + rpc_uuid_avail = 0; + } + + if (rpc_uuid_avail) + { + BYTE *uuid_str; + UUID uuid; + + if (pfn_UuidCreate (&uuid) == RPC_S_OK) + { + if (pfn_UuidToString (&uuid, &uuid_str) == RPC_S_OK) + { + snprintf (urn_str, urn_size, "<urn:uuid:%s>", uuid_str); + pfn_RpcStringFree (&uuid_str); + return 1; + } + } + } + return 0; +} +#endif +/* Fills urn_str with a UUID based on random numbers in the format + required for the WARC-Record-Id header. + (See RFC 4122, UUID version 4.) + + Note: this is a fallback method, it is much better to use the + methods provided by libuuid. + + The string will be 47 characters long. */ +void +warc_uuid_str (char *urn_str, size_t urn_size) +{ + /* RFC 4122, a version 4 UUID with only random numbers */ + + unsigned char uuid_data[16]; + int i; + +#ifdef WINDOWS + /* If the native method fails (expected on older Windows versions), + use the fallback below. */ + if (windows_uuid_str (urn_str, urn_size)) + return; +#endif + + for (i=0; i<16; i++) + uuid_data[i] = random_number (255); + + /* Set the four most significant bits (bits 12 through 15) of the + * time_hi_and_version field to the 4-bit version number */ + uuid_data[6] = (uuid_data[6] & 0x0F) | 0x40; + + /* Set the two most significant bits (bits 6 and 7) of the + * clock_seq_hi_and_reserved to zero and one, respectively. */ + uuid_data[8] = (uuid_data[8] & 0xBF) | 0x80; + + snprintf (urn_str, urn_size, + "<urn:uuid:%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x>", + uuid_data[0], uuid_data[1], uuid_data[2], uuid_data[3], uuid_data[4], + uuid_data[5], uuid_data[6], uuid_data[7], uuid_data[8], uuid_data[9], + uuid_data[10], uuid_data[11], uuid_data[12], uuid_data[13], uuid_data[14], + uuid_data[15]); +} +#endif + +/* Write a warcinfo record to the current file. + Updates warc_current_warcinfo_uuid_str. */ +static bool +warc_write_warcinfo_record (const char *filename) +{ + FILE *warc_tmp; + char timestamp[22]; + char *filename_basename; + + /* Write warc-info record as the first record of the file. */ + /* We add the record id of this info record to the other records in the + file. */ + warc_uuid_str (warc_current_warcinfo_uuid_str, sizeof (warc_current_warcinfo_uuid_str)); + + warc_timestamp (timestamp, sizeof(timestamp)); + + filename_basename = base_name (filename); + + warc_write_start_record (); + warc_write_header ("WARC-Type", "warcinfo"); + warc_write_header ("Content-Type", "application/warc-fields"); + warc_write_header ("WARC-Date", timestamp); + warc_write_header ("WARC-Record-ID", warc_current_warcinfo_uuid_str); + warc_write_header ("WARC-Filename", filename_basename); + + xfree (filename_basename); + + /* Create content. */ + warc_tmp = warc_tempfile (); + if (warc_tmp == NULL) + { + return false; + } + + fprintf (warc_tmp, "software: Wget/%s (%s)\r\n", version_string, OS_TYPE); + fprintf (warc_tmp, "format: WARC File Format 1.0\r\n"); + fprintf (warc_tmp, +"conformsTo: http://bibnum.bnf.fr/WARC/WARC_ISO_28500_version1_latestdraft.pdf\r\n"); + fprintf (warc_tmp, "robots: %s\r\n", (opt.use_robots ? "classic" : "off")); + fprintf (warc_tmp, "wget-arguments: %s\r\n", program_argstring); + /* Add the user headers, if any. */ + if (opt.warc_user_headers) + { + int i; + for (i = 0; opt.warc_user_headers[i]; i++) + fprintf (warc_tmp, "%s\r\n", opt.warc_user_headers[i]); + } + fprintf(warc_tmp, "\r\n"); + + warc_write_digest_headers (warc_tmp, -1); + warc_write_block_from_file (warc_tmp); + warc_write_end_record (); + + if (! warc_write_ok) + logprintf (LOG_NOTQUIET, _("Error writing warcinfo record to WARC file.\n")); + + fclose (warc_tmp); + return warc_write_ok; +} + +/* Opens a new WARC file. + If META is true, generates a filename ending with 'meta.warc.gz'. + + This method will: + 1. close the current WARC file (if there is one); + 2. increment warc_current_file_number; + 3. open a new WARC file; + 4. write the initial warcinfo record. + + Returns true on success, false otherwise. + */ +static bool +warc_start_new_file (bool meta) +{ +#ifdef __VMS +# define WARC_GZ "warc-gz" +#else /* def __VMS */ +# define WARC_GZ "warc.gz" +#endif /* def __VMS [else] */ + +#ifdef HAVE_LIBZ + const char *extension = (opt.warc_compression_enabled ? WARC_GZ : "warc"); +#else + const char *extension = "warc"; +#endif + + int base_filename_length; + char *new_filename; + + if (opt.warc_filename == NULL) + return false; + + if (warc_current_file != NULL) + fclose (warc_current_file); + + *warc_current_warcinfo_uuid_str = 0; + xfree (warc_current_filename); + + warc_current_file_number++; + + base_filename_length = strlen (opt.warc_filename); + /* filename format: base + "-" + 5 digit serial number + ".warc.gz" */ + new_filename = xmalloc (base_filename_length + 1 + 5 + 8 + 1); + + warc_current_filename = new_filename; + + /* If max size is enabled, we add a serial number to the file names. */ + if (meta) + sprintf (new_filename, "%s-meta.%s", opt.warc_filename, extension); + else if (opt.warc_maxsize > 0) + { + sprintf (new_filename, "%s-%05d.%s", opt.warc_filename, + warc_current_file_number, extension); + } + else + sprintf (new_filename, "%s.%s", opt.warc_filename, extension); + + logprintf (LOG_VERBOSE, _("Opening WARC file %s.\n\n"), quote (new_filename)); + + /* Open the WARC file. */ + warc_current_file = fopen (new_filename, "wb+"); + if (warc_current_file == NULL) + { + logprintf (LOG_NOTQUIET, _("Error opening WARC file %s.\n"), + quote (new_filename)); + return false; + } + + if (! warc_write_warcinfo_record (new_filename)) + return false; + + /* Add warcinfo uuid to manifest. */ + if (warc_manifest_fp) + fprintf (warc_manifest_fp, "%s\n", warc_current_warcinfo_uuid_str); + + return true; +} + +/* Opens the CDX file for output. */ +static bool +warc_start_cdx_file (void) +{ + char *cdx_filename = aprintf("%s.cdx", opt.warc_filename); + warc_current_cdx_file = fopen (cdx_filename, "a+"); + free(cdx_filename); + + if (warc_current_cdx_file == NULL) + return false; + + /* Print the CDX header. + * + * a - original url + * b - date + * m - mime type + * s - response code + * k - new style checksum + * r - redirect + * M - meta tags + * V - compressed arc file offset + * g - file name + * u - record-id + */ + fprintf (warc_current_cdx_file, " CDX a b a m s k r M V g u\n"); + fflush (warc_current_cdx_file); + + return true; +} + +#define CDX_FIELDSEP " \t\r\n" + +/* Parse the CDX header and find the field numbers of the original url, + checksum and record ID fields. */ +static bool +warc_parse_cdx_header (char *lineptr, int *field_num_original_url, + int *field_num_checksum, int *field_num_record_id) +{ + char *token; + char *save_ptr; + + *field_num_original_url = -1; + *field_num_checksum = -1; + *field_num_record_id = -1; + + token = strtok_r (lineptr, CDX_FIELDSEP, &save_ptr); + + if (token != NULL && strcmp (token, "CDX") == 0) + { + int field_num = 0; + while (token != NULL) + { + token = strtok_r (NULL, CDX_FIELDSEP, &save_ptr); + if (token != NULL) + { + switch (token[0]) + { + case 'a': + *field_num_original_url = field_num; + break; + case 'k': + *field_num_checksum = field_num; + break; + case 'u': + *field_num_record_id = field_num; + break; + } + } + field_num++; + } + } + + return *field_num_original_url != -1 + && *field_num_checksum != -1 + && *field_num_record_id != -1; +} + +/* Parse the CDX record and add it to the warc_cdx_dedup_table hash table. */ +static void +warc_process_cdx_line (char *lineptr, int field_num_original_url, + int field_num_checksum, int field_num_record_id) +{ + char *original_url = NULL; + char *checksum = NULL; + char *record_id = NULL; + char *token; + char *save_ptr; + int field_num = 0; + + /* Read this line to get the fields we need. */ + token = strtok_r (lineptr, CDX_FIELDSEP, &save_ptr); + while (token != NULL) + { + char **val; + if (field_num == field_num_original_url) + val = &original_url; + else if (field_num == field_num_checksum) + val = &checksum; + else if (field_num == field_num_record_id) + val = &record_id; + else + val = NULL; + + if (val != NULL) + *val = strdup (token); + + token = strtok_r (NULL, CDX_FIELDSEP, &save_ptr); + field_num++; + } + + if (original_url != NULL && checksum != NULL && record_id != NULL) + { + /* For some extra efficiency, we decode the base32 encoded + checksum value. This should produce exactly SHA1_DIGEST_SIZE + bytes. */ + size_t checksum_l; + char * checksum_v; + base32_decode_alloc (checksum, strlen (checksum), &checksum_v, + &checksum_l); + xfree (checksum); + + if (checksum_v != NULL && checksum_l == SHA1_DIGEST_SIZE) + { + /* This is a valid line with a valid checksum. */ + struct warc_cdx_record *rec; + rec = xmalloc (sizeof (struct warc_cdx_record)); + rec->url = original_url; + rec->uuid = record_id; + memcpy (rec->digest, checksum_v, SHA1_DIGEST_SIZE); + hash_table_put (warc_cdx_dedup_table, rec->digest, rec); + xfree (checksum_v); + } + else + { + xfree (original_url); + xfree (checksum_v); + xfree (record_id); + } + } + else + { + xfree(checksum); + xfree(original_url); + xfree(record_id); + } +} + +/* Loads the CDX file from opt.warc_cdx_dedup_filename and fills + the warc_cdx_dedup_table. */ +static bool +warc_load_cdx_dedup_file (void) +{ + FILE *f; + char *lineptr = NULL; + size_t n = 0; + ssize_t line_length; + int field_num_original_url = -1; + int field_num_checksum = -1; + int field_num_record_id = -1; + + f = fopen (opt.warc_cdx_dedup_filename, "r"); + if (f == NULL) + return false; + + /* The first line should contain the CDX header. + Format: " CDX x x x x x" + where x are field type indicators. For our purposes, we only + need 'a' (the original url), 'k' (the SHA1 checksum) and + 'u' (the WARC record id). */ + line_length = getline (&lineptr, &n, f); + if (line_length != -1) + warc_parse_cdx_header (lineptr, &field_num_original_url, + &field_num_checksum, &field_num_record_id); + + /* If the file contains all three fields, read the complete file. */ + if (field_num_original_url == -1 + || field_num_checksum == -1 + || field_num_record_id == -1) + { + if (field_num_original_url == -1) + logprintf (LOG_NOTQUIET, +_("CDX file does not list original urls. (Missing column 'a'.)\n")); + if (field_num_checksum == -1) + logprintf (LOG_NOTQUIET, +_("CDX file does not list checksums. (Missing column 'k'.)\n")); + if (field_num_record_id == -1) + logprintf (LOG_NOTQUIET, +_("CDX file does not list record ids. (Missing column 'u'.)\n")); + } + else + { + int nrecords; + + /* Initialize the table. */ + warc_cdx_dedup_table = hash_table_new (1000, warc_hash_sha1_digest, + warc_cmp_sha1_digest); + + do + { + line_length = getline (&lineptr, &n, f); + if (line_length != -1) + { + warc_process_cdx_line (lineptr, field_num_original_url, + field_num_checksum, field_num_record_id); + } + + } + while (line_length != -1); + + /* Print results. */ + nrecords = hash_table_count (warc_cdx_dedup_table); + logprintf (LOG_VERBOSE, ngettext ("Loaded %d record from CDX.\n\n", + "Loaded %d records from CDX.\n\n", + nrecords), + nrecords); + } + + xfree (lineptr); + fclose (f); + + return true; +} +#undef CDX_FIELDSEP + +/* Returns the existing duplicate CDX record for the given url and payload + digest. Returns NULL if the url is not found or if the payload digest + does not match, or if CDX deduplication is disabled. */ +static struct warc_cdx_record * +warc_find_duplicate_cdx_record (const char *url, char *sha1_digest_payload) +{ + struct warc_cdx_record *rec_existing; + + if (warc_cdx_dedup_table == NULL) + return NULL; + + rec_existing = hash_table_get (warc_cdx_dedup_table, sha1_digest_payload); + + if (rec_existing && strcmp (rec_existing->url, url) == 0) + return rec_existing; + else + return NULL; +} + +/* Initializes the WARC writer (if opt.warc_filename is set). + This should be called before any WARC record is written. */ +void +warc_init (void) +{ + warc_write_ok = true; + + if (opt.warc_filename != NULL) + { + if (opt.warc_cdx_dedup_filename != NULL) + { + if (! warc_load_cdx_dedup_file ()) + { + logprintf (LOG_NOTQUIET, + _("Could not read CDX file %s for deduplication.\n"), + quote (opt.warc_cdx_dedup_filename)); + exit (WGET_EXIT_GENERIC_ERROR); + } + } + + warc_manifest_fp = warc_tempfile (); + if (warc_manifest_fp == NULL) + { + logprintf (LOG_NOTQUIET, + _("Could not open temporary WARC manifest file.\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + + if (opt.warc_keep_log) + { + warc_log_fp = warc_tempfile (); + if (warc_log_fp == NULL) + { + logprintf (LOG_NOTQUIET, + _("Could not open temporary WARC log file.\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + log_set_warc_log_fp (warc_log_fp); + } + + warc_current_file_number = -1; + if (! warc_start_new_file (false)) + { + logprintf (LOG_NOTQUIET, _("Could not open WARC file.\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + + if (opt.warc_cdx_enabled) + { + if (! warc_start_cdx_file ()) + { + logprintf (LOG_NOTQUIET, + _("Could not open CDX file for output.\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + } + } +} + +/* Writes metadata (manifest, configuration, log file) to the WARC file. */ +static void +warc_write_metadata (void) +{ + char manifest_uuid[48]; + FILE *warc_tmp_fp; + + /* If there are multiple WARC files, the metadata should be written to a separate file. */ + if (opt.warc_maxsize > 0) + warc_start_new_file (true); + + warc_uuid_str (manifest_uuid, sizeof (manifest_uuid)); + + fflush (warc_manifest_fp); + warc_write_metadata_record (manifest_uuid, + "metadata://gnu.org/software/wget/warc/MANIFEST.txt", + NULL, NULL, NULL, "text/plain", + warc_manifest_fp, -1); + /* warc_write_resource_record has closed warc_manifest_fp. */ + + warc_tmp_fp = warc_tempfile (); + if (warc_tmp_fp == NULL) + { + logprintf (LOG_NOTQUIET, _("Could not open temporary WARC file.\n")); + exit (WGET_EXIT_GENERIC_ERROR); + } + fflush (warc_tmp_fp); + fprintf (warc_tmp_fp, "%s\n", program_argstring); + + warc_write_resource_record (NULL, + "metadata://gnu.org/software/wget/warc/wget_arguments.txt", + NULL, manifest_uuid, NULL, "text/plain", + warc_tmp_fp, -1); + /* warc_write_resource_record has closed warc_tmp_fp. */ + + if (warc_log_fp != NULL) + { + warc_write_resource_record (NULL, + "metadata://gnu.org/software/wget/warc/wget.log", + NULL, manifest_uuid, NULL, "text/plain", + warc_log_fp, -1); + /* warc_write_resource_record has closed warc_log_fp. */ + + warc_log_fp = NULL; + log_set_warc_log_fp (NULL); + } +} + +/* Finishes the WARC writing. + This should be called at the end of the program. */ +void +warc_close (void) +{ + if (warc_current_file != NULL) + { + warc_write_metadata (); + *warc_current_warcinfo_uuid_str = 0; + fclose (warc_current_file); + warc_current_file = NULL; + } + + if (warc_current_cdx_file != NULL) + { + fclose (warc_current_cdx_file); + warc_current_cdx_file = NULL; + } + + if (warc_log_fp != NULL) + { + fclose (warc_log_fp); + log_set_warc_log_fp (NULL); + } +} + +/* Creates a temporary file for writing WARC output. + The temporary file will be created in opt.warc_tempdir. + Returns the pointer to the temporary file, or NULL. */ +FILE * +warc_tempfile (void) +{ + char filename[100]; + int fd; + + if (path_search (filename, 100, opt.warc_tempdir, "wget", true) == -1) + return NULL; + +#ifdef __VMS + /* 2013-07-12 SMS. + * mkostemp()+unlink()+fdopen() scheme causes trouble on VMS, so use + * mktemp() to uniquify the (VMS-style) name, and then use a normal + * fopen() with a "create temp file marked for delete" option. + */ + { + char *tfn; + + tfn = mktemp (filename); /* Get unique name from template. */ + if (tfn == NULL) + return NULL; + return fopen (tfn, "w+", "fop=tmd"); /* Create auto-delete temp file. */ + } +#else /* def __VMS */ + fd = mkostemp (filename, O_TEMPORARY); + if (fd < 0) + return NULL; + +#if !O_TEMPORARY + if (unlink (filename) < 0) + { + close(fd); + return NULL; + } +#endif + + return fdopen (fd, "wb+"); +#endif /* def __VMS [else] */ +} + + +/* Writes a request record to the WARC file. + url is the target uri of the request, + timestamp_str is the timestamp of the request (generated with warc_timestamp), + record_uuid is the uuid of the request (generated with warc_uuid_str), + body is a pointer to a file containing the request headers and body. + ip is the ip address of the server (or NULL), + Calling this function will close body. + Returns true on success, false on error. */ +bool +warc_write_request_record (const char *url, const char *timestamp_str, + const char *record_uuid, const ip_address *ip, + FILE *body, off_t payload_offset) +{ + warc_write_start_record (); + warc_write_header ("WARC-Type", "request"); + warc_write_header_uri ("WARC-Target-URI", url); + warc_write_header ("Content-Type", "application/http;msgtype=request"); + warc_write_date_header (timestamp_str); + warc_write_header ("WARC-Record-ID", record_uuid); + warc_write_ip_header (ip); + warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str); + warc_write_digest_headers (body, payload_offset); + warc_write_block_from_file (body); + warc_write_end_record (); + + fclose (body); + + return warc_write_ok; +} + +/* Writes a response record to the CDX file. + url is the target uri of the request/response, + timestamp_str is the timestamp of the request that generated this response, + (generated with warc_timestamp), + mime_type is the mime type of the response body (will be printed to CDX), + response_code is the HTTP response code (will be printed to CDX), + payload_digest is the sha1 digest of the payload, + redirect_location is the contents of the Location: header, or NULL (will be printed to CDX), + offset is the position of the WARC record in the WARC file, + warc_filename is the filename of the WARC, + response_uuid is the uuid of the response. + Returns true on success, false on error. */ +static bool +warc_write_cdx_record (const char *url, const char *timestamp_str, + const char *mime_type, int response_code, + const char *payload_digest, const char *redirect_location, + off_t offset, const char *warc_filename _GL_UNUSED, + const char *response_uuid) +{ + /* Transform the timestamp. */ + char timestamp_str_cdx[15]; + char offset_string[MAX_INT_TO_STRING_LEN(off_t)]; + const char *checksum; + char *tmp_location = NULL; + + memcpy (timestamp_str_cdx , timestamp_str , 4); /* "YYYY" "-" */ + memcpy (timestamp_str_cdx + 4, timestamp_str + 5, 2); /* "mm" "-" */ + memcpy (timestamp_str_cdx + 6, timestamp_str + 8, 2); /* "dd" "T" */ + memcpy (timestamp_str_cdx + 8, timestamp_str + 11, 2); /* "HH" ":" */ + memcpy (timestamp_str_cdx + 10, timestamp_str + 14, 2); /* "MM" ":" */ + memcpy (timestamp_str_cdx + 12, timestamp_str + 17, 2); /* "SS" "Z" */ + timestamp_str_cdx[14] = '\0'; + + /* Rewrite the checksum. */ + if (payload_digest != NULL) + checksum = payload_digest + 5; /* Skip the "sha1:" */ + else + checksum = "-"; + + if (mime_type == NULL || strlen(mime_type) == 0) + mime_type = "-"; + if (redirect_location == NULL || strlen(redirect_location) == 0) + tmp_location = strdup ("-"); + else + tmp_location = url_escape(redirect_location); + + number_to_string (offset_string, offset); + + /* Print the CDX line. */ + fprintf (warc_current_cdx_file, "%s %s %s %s %d %s %s - %s %s %s\n", url, + timestamp_str_cdx, url, mime_type, response_code, checksum, + tmp_location, offset_string, warc_current_filename, + response_uuid); + fflush (warc_current_cdx_file); + free (tmp_location); + + return true; +} + +/* Writes a revisit record to the WARC file. + url is the target uri of the request/response, + timestamp_str is the timestamp of the request that generated this response + (generated with warc_timestamp), + concurrent_to_uuid is the uuid of the request for that generated this response + (generated with warc_uuid_str), + refers_to_uuid is the uuid of the original response + (generated with warc_uuid_str), + payload_digest is the sha1 digest of the payload, + ip is the ip address of the server (or NULL), + body is a pointer to a file containing the response headers (without payload). + Calling this function will close body. + Returns true on success, false on error. */ +static bool +warc_write_revisit_record (const char *url, const char *timestamp_str, + const char *concurrent_to_uuid, const char *payload_digest, + const char *refers_to, const ip_address *ip, FILE *body) +{ + char revisit_uuid [48]; + char block_digest[BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1 + 5]; + char sha1_res_block[SHA1_DIGEST_SIZE]; + + warc_uuid_str (revisit_uuid, sizeof (revisit_uuid)); + + sha1_stream (body, sha1_res_block); + warc_base32_sha1_digest (sha1_res_block, block_digest, sizeof(block_digest)); + + warc_write_start_record (); + warc_write_header ("WARC-Type", "revisit"); + warc_write_header ("WARC-Record-ID", revisit_uuid); + warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str); + warc_write_header ("WARC-Concurrent-To", concurrent_to_uuid); + warc_write_header ("WARC-Refers-To", refers_to); + warc_write_header ("WARC-Profile", "http://netpreserve.org/warc/1.0/revisit/identical-payload-digest"); + warc_write_header ("WARC-Truncated", "length"); + warc_write_header_uri ("WARC-Target-URI", url); + warc_write_date_header (timestamp_str); + warc_write_ip_header (ip); + warc_write_header ("Content-Type", "application/http;msgtype=response"); + warc_write_header ("WARC-Block-Digest", block_digest); + warc_write_header ("WARC-Payload-Digest", payload_digest); + warc_write_block_from_file (body); + warc_write_end_record (); + + fclose (body); + + return warc_write_ok; +} + +/* Writes a response record to the WARC file. + url is the target uri of the request/response, + timestamp_str is the timestamp of the request that generated this response + (generated with warc_timestamp), + concurrent_to_uuid is the uuid of the request for that generated this response + (generated with warc_uuid_str), + ip is the ip address of the server (or NULL), + body is a pointer to a file containing the response headers and body. + mime_type is the mime type of the response body (will be printed to CDX), + response_code is the HTTP response code (will be printed to CDX), + redirect_location is the contents of the Location: header, or NULL (will be printed to CDX), + Calling this function will close body. + Returns true on success, false on error. */ +bool +warc_write_response_record (const char *url, const char *timestamp_str, + const char *concurrent_to_uuid, const ip_address *ip, + FILE *body, off_t payload_offset, const char *mime_type, + int response_code, const char *redirect_location) +{ + char block_digest[BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1 + 5]; + char payload_digest[BASE32_LENGTH(SHA1_DIGEST_SIZE) + 1 + 5]; + char sha1_res_block[SHA1_DIGEST_SIZE]; + char sha1_res_payload[SHA1_DIGEST_SIZE]; + char response_uuid [48]; + off_t offset; + + if (opt.warc_digests_enabled) + { + /* Calculate the block and payload digests. */ + rewind (body); + if (warc_sha1_stream_with_payload (body, sha1_res_block, sha1_res_payload, + payload_offset) == 0) + { + /* Decide (based on url + payload digest) if we have seen this + data before. */ + struct warc_cdx_record *rec_existing; + rec_existing = warc_find_duplicate_cdx_record (url, sha1_res_payload); + if (rec_existing != NULL) + { + bool result; + + /* Found an existing record. */ + logprintf (LOG_VERBOSE, + _("Found exact match in CDX file. Saving revisit record to WARC.\n")); + + /* Remove the payload from the file. */ + if (payload_offset > 0) + { + if (ftruncate (fileno (body), payload_offset) == -1) + return false; + } + + /* Send the original payload digest. */ + warc_base32_sha1_digest (sha1_res_payload, payload_digest, sizeof(payload_digest)); + result = warc_write_revisit_record (url, timestamp_str, + concurrent_to_uuid, payload_digest, rec_existing->uuid, + ip, body); + + return result; + } + + warc_base32_sha1_digest (sha1_res_block, block_digest, sizeof(block_digest)); + warc_base32_sha1_digest (sha1_res_payload, payload_digest, sizeof(payload_digest)); + } + } + + /* Not a revisit, just store the record. */ + + warc_uuid_str (response_uuid, sizeof (response_uuid)); + + fseeko (warc_current_file, 0L, SEEK_END); + offset = ftello (warc_current_file); + + warc_write_start_record (); + warc_write_header ("WARC-Type", "response"); + warc_write_header ("WARC-Record-ID", response_uuid); + warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str); + warc_write_header ("WARC-Concurrent-To", concurrent_to_uuid); + warc_write_header_uri ("WARC-Target-URI", url); + warc_write_date_header (timestamp_str); + warc_write_ip_header (ip); + warc_write_header ("WARC-Block-Digest", block_digest); + warc_write_header ("WARC-Payload-Digest", payload_digest); + warc_write_header ("Content-Type", "application/http;msgtype=response"); + warc_write_block_from_file (body); + warc_write_end_record (); + + fclose (body); + + if (warc_write_ok && opt.warc_cdx_enabled) + { + /* Add this record to the CDX. */ + warc_write_cdx_record (url, timestamp_str, mime_type, response_code, + payload_digest, redirect_location, offset, warc_current_filename, + response_uuid); + } + + return warc_write_ok; +} + +/* Writes a resource or metadata record to the WARC file. + warc_type is either "resource" or "metadata", + resource_uuid is the uuid of the resource (or NULL), + url is the target uri of the resource, + timestamp_str is the timestamp (generated with warc_timestamp), + concurrent_to_uuid is the uuid of the record that generated this, + resource (generated with warc_uuid_str) or NULL, + ip is the ip address of the server (or NULL), + content_type is the mime type of the body (or NULL), + body is a pointer to a file containing the resource data. + Calling this function will close body. + Returns true on success, false on error. */ +static bool +warc_write_record (const char *record_type, const char *resource_uuid, + const char *url, const char *timestamp_str, + const char *concurrent_to_uuid, + const ip_address *ip, const char *content_type, FILE *body, + off_t payload_offset) +{ + char uuid_buf[48]; + + if (resource_uuid == NULL) + { + warc_uuid_str (uuid_buf, sizeof (uuid_buf)); + resource_uuid = uuid_buf; + } + + if (content_type == NULL) + content_type = "application/octet-stream"; + + warc_write_start_record (); + warc_write_header ("WARC-Type", record_type); + warc_write_header ("WARC-Record-ID", resource_uuid); + warc_write_header ("WARC-Warcinfo-ID", warc_current_warcinfo_uuid_str); + warc_write_header ("WARC-Concurrent-To", concurrent_to_uuid); + warc_write_header_uri ("WARC-Target-URI", url); + warc_write_date_header (timestamp_str); + warc_write_ip_header (ip); + warc_write_digest_headers (body, payload_offset); + warc_write_header ("Content-Type", content_type); + warc_write_block_from_file (body); + warc_write_end_record (); + + fclose (body); + + return warc_write_ok; +} + +/* Writes a resource record to the WARC file. + resource_uuid is the uuid of the resource (or NULL), + url is the target uri of the resource, + timestamp_str is the timestamp (generated with warc_timestamp), + concurrent_to_uuid is the uuid of the record that generated this, + resource (generated with warc_uuid_str) or NULL, + ip is the ip address of the server (or NULL), + content_type is the mime type of the body (or NULL), + body is a pointer to a file containing the resource data. + Calling this function will close body. + Returns true on success, false on error. */ +bool +warc_write_resource_record (const char *resource_uuid, const char *url, + const char *timestamp_str, const char *concurrent_to_uuid, + const ip_address *ip, const char *content_type, FILE *body, + off_t payload_offset) +{ + return warc_write_record ("resource", + resource_uuid, url, timestamp_str, concurrent_to_uuid, + ip, content_type, body, payload_offset); +} + +/* Writes a metadata record to the WARC file. + record_uuid is the uuid of the record (or NULL), + url is the target uri of the record, + timestamp_str is the timestamp (generated with warc_timestamp), + concurrent_to_uuid is the uuid of the record that generated this, + record (generated with warc_uuid_str) or NULL, + ip is the ip address of the server (or NULL), + content_type is the mime type of the body (or NULL), + body is a pointer to a file containing the record data. + Calling this function will close body. + Returns true on success, false on error. */ +bool +warc_write_metadata_record (const char *record_uuid, const char *url, + const char *timestamp_str, const char *concurrent_to_uuid, + ip_address *ip, const char *content_type, FILE *body, + off_t payload_offset) +{ + return warc_write_record ("metadata", + record_uuid, url, timestamp_str, concurrent_to_uuid, + ip, content_type, body, payload_offset); +} diff --git a/src/warc.h b/src/warc.h new file mode 100644 index 0000000..7238cff --- /dev/null +++ b/src/warc.h @@ -0,0 +1,27 @@ +/* Declarations of WARC helper methods. */ +#ifndef WARC_H +#define WARC_H + +#include "host.h" + +void warc_init (void); +void warc_close (void); +void warc_uuid_str (char *id_str, size_t urn_size); + +char * warc_timestamp (char *timestamp, size_t timestamp_size); + +FILE * warc_tempfile (void); + +bool warc_write_request_record (const char *url, const char *timestamp_str, + const char *concurrent_to_uuid, const ip_address *ip, FILE *body, off_t payload_offset); +bool warc_write_response_record (const char *url, const char *timestamp_str, + const char *concurrent_to_uuid, const ip_address *ip, FILE *body, off_t payload_offset, + const char *mime_type, int response_code, const char *redirect_location); +bool warc_write_resource_record (const char *resource_uuid, const char *url, + const char *timestamp_str, const char *concurrent_to_uuid, const ip_address *ip, + const char *content_type, FILE *body, off_t payload_offset); +bool warc_write_metadata_record (const char *record_uuid, const char *url, + const char *timestamp_str, const char *concurrent_to_uuid, ip_address *ip, + const char *content_type, FILE *body, off_t payload_offset); + +#endif /* WARC_H */ diff --git a/src/wget.h b/src/wget.h new file mode 100644 index 0000000..b6a9dca --- /dev/null +++ b/src/wget.h @@ -0,0 +1,333 @@ +/* Miscellaneous declarations. + Copyright (C) 1996-2011, 2015, 2018-2020 Free Software Foundation, + Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +/* This file contains declarations that are universally useful and + those that don't fit elsewhere. It also includes sysdep.h which + includes some often-needed system includes, like the obnoxious + <time.h> inclusion. */ + +#ifndef WGET_H +#define WGET_H + +#include "config.h" + +#if ((defined _WIN32 || defined __WIN32__) && !defined __CYGWIN__) +# define WINDOWS +#endif + +/* Include these, so random files need not include them. */ +#include "sysdep.h" + +/* Disable assertions when debug support is not compiled in. */ +#ifndef ENABLE_DEBUG +#ifndef NDEBUG +# define NDEBUG +#endif +#endif + +/* Is OpenSSL or GNUTLS available? */ +#if defined HAVE_LIBSSL || defined HAVE_LIBSSL32 || defined HAVE_LIBGNUTLS +# define HAVE_SSL +# define HAVE_HSTS /* There's no sense in enabling HSTS without SSL */ +#endif + +/* `gettext (FOO)' is long to write, so we use `_(FOO)'. If NLS is + unavailable, _(STRING) simply returns STRING. */ +#include "gettext.h" +#define _(STRING) gettext(STRING) + +/* A pseudo function call that serves as a marker for the automated + extraction of messages, but does not call gettext(). The run-time + translation is done at a different place in the code. The purpose + of the N_("...") call is to make the message snarfer aware that the + "..." string needs to be translated. STRING should be a string + literal. Concatenated strings and other string expressions won't + work. The macro's expansion is not parenthesized, so that it is + suitable as initializer for static 'char[]' or 'const char[]' + variables. -- explanation partly taken from GNU make. */ +#define N_(string) string + +#if HAVE_WCWIDTH && HAVE_MBTOWC +# define USE_NLS_PROGRESS_BAR 1 +#else +/* Just to be a little paranoid about it. */ +# undef USE_NLS_PROGRESS_BAR +#endif + +/* I18N NOTE: You will notice that none of the DEBUGP messages are + marked as translatable. This is intentional, for a few reasons: + + 1) The debug messages are not meant for the users to look at, but + for the developers; as such, they should be considered more like + source comments than real program output. + + 2) The messages are numerous, and yet they are random and frivolous + ("double yuck!" and such). There would be a lot of work with no + gain. + + 3) Finally, the debug messages are meant to be a clue for me to + debug problems with Wget. If I get them in a language I don't + understand, debugging will become a new challenge of its own! */ + +/* locale independent replacement for ctype.h */ +#include "c-ctype.h" + +/* Conditionalize the use of GCC's __attribute__((format)) and + __builtin_expect features using macros. */ + +#if defined(__GNUC__) && __GNUC__ >= 3 +# define GCC_FORMAT_ATTR(a, b) __attribute__ ((format (printf, a, b))) +# define LIKELY(exp) __builtin_expect (!!(exp), 1) +# define UNLIKELY(exp) __builtin_expect ((exp), 0) +#else +# define GCC_FORMAT_ATTR(a, b) +# define LIKELY(exp) (exp) +# define UNLIKELY(exp) (exp) +#endif + +/* Execute the following statement if debugging is both enabled at + compile-time and requested at run-time; a no-op otherwise. */ + +#ifdef ENABLE_DEBUG +# define IF_DEBUG if (UNLIKELY (opt.debug)) +#else +# define IF_DEBUG if (0) +#endif + +/* Print ARGS if debugging is enabled and requested, otherwise do + nothing. This must be called with an extra level of parentheses + because it's not possible to pass a variable number of arguments to + a macro (in portable C89). ARGS are like arguments to printf. */ + +#define DEBUGP(args) do { IF_DEBUG { debug_logprintf args; } } while (0) + +/* Pick an integer type large enough for file sizes, content lengths, + and such. Because today's files can be very large, it should be a + signed integer at least 64 bits wide. This can't be typedeffed to + off_t because: a) off_t is always 32-bit on Windows, and b) we + don't necessarily want to tie having a 64-bit type for internal + calculations to having LFS support. */ + +/* Gnulib's stdint.h module essentially guarantees the existence of int64_t. + * Thus we can simply assume it always exists and use it. + */ +#include <stdint.h> + +typedef int64_t wgint; +#define WGINT_MAX INT64_MAX +typedef wgint SUM_SIZE_INT; + +#define str_to_wgint strtol + +#include "options.h" + +/* Everything uses this, so include them here directly. */ +#ifdef __cplusplus +# undef _Noreturn +#endif +#include "xalloc.h" + +/* Likewise for logging functions. */ +#include "log.h" + +/* Likewise for quoting functions. */ +#include "quote.h" +#include "quotearg.h" + +/* Likewise for struct iri definition */ +#include "iri.h" + +/* Useful macros used across the code: */ + +/* The number of elements in an array. For example: + static char a[] = "foo"; -- countof(a) == 4 (note terminating \0) + int a[5] = {1, 2}; -- countof(a) == 5 + char *a[] = { -- countof(a) == 3 + "foo", "bar", "baz" + }; */ +#define countof(array) (sizeof (array) / sizeof ((array)[0])) + +/* Zero out a value. */ +#define xzero(x) memset (&(x), '\0', sizeof (x)) + +/* Convert an ASCII hex digit to the corresponding number between 0 + and 15. c should be a hexadecimal digit that satisfies c_isxdigit; + otherwise, the result is undefined. */ +static inline unsigned char _unhex(unsigned char c) +{ + return c <= '9' ? c - '0' : (c <= 'F' ? c - 'A' + 10 : c - 'a' + 10); +} +#define X2DIGITS_TO_NUM(h1, h2) ((_unhex (h1) << 4) + _unhex (h2)) + +/* The reverse of the above: convert a number in the [0, 16) range to + the ASCII representation of the corresponding hexadecimal digit. + `+ 0' is there so you can't accidentally use it as an lvalue. */ +#define XNUM_TO_DIGIT(x) ("0123456789ABCDEF"[x] + 0) +#define XNUM_TO_digit(x) ("0123456789abcdef"[x] + 0) + +/* Return non-zero if string bounded between BEG and END is equal to + STRING_LITERAL. The comparison is case-sensitive. */ +#define BOUNDED_EQUAL(beg, end, string_literal) \ + ((end) - (beg) == sizeof (string_literal) - 1 \ + && !memcmp (beg, string_literal, sizeof (string_literal) - 1)) + +/* The same as above, except the comparison is case-insensitive. */ +#define BOUNDED_EQUAL_NO_CASE(beg, end, string_literal) \ + ((end) - (beg) == sizeof (string_literal) - 1 \ + && !c_strncasecmp (beg, string_literal, sizeof (string_literal) - 1)) + +/* Generally useful if you want to avoid arbitrary size limits but + don't need a full dynamic array. Assumes that BASEVAR points to a + malloced array of TYPE objects (or possibly a NULL pointer, if + SIZEVAR is 0), with the total size stored in SIZEVAR. This macro + will realloc BASEVAR as necessary so that it can hold at least + NEEDED_SIZE objects. The reallocing is done by doubling, which + ensures constant amortized time per element. */ + +#define DO_REALLOC(basevar, sizevar, needed_size, type) do { \ + long DR_needed_size = (needed_size); \ + long DR_newsize = 0; \ + while ((sizevar) < (DR_needed_size)) { \ + DR_newsize = sizevar << 1; \ + if (DR_newsize < 16) \ + DR_newsize = 16; \ + (sizevar) = DR_newsize; \ + } \ + if (DR_newsize) \ + basevar = xrealloc (basevar, DR_newsize * sizeof (type)); \ +} while (0) + +/* Used to print pointers (usually for debugging). Print pointers + using printf ("0x%0*lx", PTR_FORMAT (p)). (%p is too unpredictable; + some implementations prepend 0x, while some don't, and most don't + 0-pad the address.) */ +#define PTR_FORMAT(p) (int) (2 * sizeof (void *)), (unsigned long) (p) + +/* Find the maximum buffer length needed to print an integer of type `x' + in base 10. 24082 / 10000 = 8*log_{10}(2). */ +#define MAX_INT_TO_STRING_LEN(x) ((sizeof(x) * 24082 / 10000) + 2) + +/* Find the minimum or maximum of two provided values */ +# define MIN(i, j) ((i) <= (j) ? (i) : (j)) +# define MAX(i, j) ((i) >= (j) ? (i) : (j)) + + +extern const char *exec_name; +extern const char *program_name; +extern const char *program_argstring; + +/* Document type ("dt") flags */ +enum +{ + TEXTHTML = 0x0001, /* document is of type text/html + or application/xhtml+xml */ + RETROKF = 0x0002, /* retrieval was OK */ + HEAD_ONLY = 0x0004, /* only send the HEAD request */ + SEND_NOCACHE = 0x0008, /* send Cache-Control: no-cache and Pragma: no-cache directive */ + ACCEPTRANGES = 0x0010, /* Accept-ranges header was found */ + ADDED_HTML_EXTENSION = 0x0020, /* added ".html" extension due to -E */ + TEXTCSS = 0x0040, /* document is of type text/css */ + IF_MODIFIED_SINCE = 0x0080, /* use if-modified-since header */ + METALINK_METADATA = 0x0100 /* use HTTP response for Metalink metadata */ +}; + +/* Universal error type -- used almost everywhere. Error reporting of + this detail is not generally used or needed and should be + simplified. */ +typedef enum +{ + NOCONERROR, HOSTERR, CONSOCKERR, CONERROR, CONSSLERR, + CONIMPOSSIBLE, NEWLOCATION, + FTPOK, FTPLOGINC, FTPLOGREFUSED, FTPPORTERR, FTPSYSERR, + FTPNSFOD, FTPUNKNOWNTYPE, FTPRERR, + FTPSRVERR, FTPRETRINT, FTPRESTFAIL, URLERROR, FOPENERR, + FOPEN_EXCL_ERR, FWRITEERR, HEOF, GATEWAYTIMEOUT, + HERR, RETROK, RECLEVELEXC, WRONGCODE, + FTPINVPASV, FTPNOPASV, FTPNOPBSZ, FTPNOPROT, FTPNOAUTH, + CONTNOTSUPPORTED, RETRUNNEEDED, RETRFINISHED, + READERR, TRYLIMEXC, FILEBADFILE, RANGEERR, + RETRBADPATTERN, PROXERR, + AUTHFAILED, QUOTEXC, WRITEFAILED, SSLINITFAILED, VERIFCERTERR, + UNLINKERR, NEWLOCATION_KEEP_POST, CLOSEFAILED, ATTRMISSING, UNKNOWNATTR, + WARC_ERR, WARC_TMP_FOPENERR, WARC_TMP_FWRITEERR, + TIMECONV_ERR, + METALINK_PARSE_ERROR, METALINK_RETR_ERROR, + METALINK_CHKSUM_ERROR, METALINK_SIG_ERROR, METALINK_MISSING_RESOURCE, + RETR_WITH_METALINK, + METALINK_SIZE_ERROR +} uerr_t; + +/* 2005-02-19 SMS. + Select an appropriate "orig" suffix and a separator character for + adding a unique suffix to a file name. + + A VMS ODS2 file system can't tolerate multiple dots. An ODS5 file + system can, but even there not all dots are equal, and heroic effort + would be needed to get ".html^.orig" rather than (the less desirable) + "^.html.orig". It's more satisfactory always to use "_orig" on VMS + (rather than including "vms.h", testing "ods5_dest", and acting + accordingly). + + Note that code in various places assumes that this string is five + characters long. +*/ +# ifdef __VMS +# define ORIG_SFX "_orig" +# else /* def __VMS */ +# define ORIG_SFX ".orig" +# endif /* def __VMS [else] */ + +/* ".NNN" unique-ifying suffix separator character for unique_name() in + url.c (and anywhere else). Note that on VMS, the file system's + version numbers solve the problem that unique_name() is designed to + handle, obviating this whole exercise. Other systems may specify a + character different from "." here, if desired. +*/ +# ifndef __VMS +# define UNIQ_SEP '.' +# endif /* ndef __VMS */ + +#if defined FUZZING && defined TESTING +/* Rename fopen so we can have our own version in fuzz/main.c to + not create random files. */ +# define fopen(fp, mode) fopen_wget(fp, mode) +# define exit(status) exit_wget(status) + +/* In run_wgetrc() we call fopen_wgetrc() instead of fopen, so we can catch + the call in our fuzzers. */ +FILE *fopen_wget(const char *pathname, const char *mode); +FILE *fopen_wgetrc(const char *pathname, const char *mode); +void exit_wget(int status); +#else +/* When not fuzzing, we want to call fopen() instead of fopen_wgetrc() */ +# define fopen_wgetrc(fp, mode) fopen(fp, mode) +#endif /* FUZZING && TESTING */ + +#endif /* WGET_H */ diff --git a/src/xattr.c b/src/xattr.c new file mode 100644 index 0000000..a5261f1 --- /dev/null +++ b/src/xattr.c @@ -0,0 +1,95 @@ +/* xattr.h -- POSIX Extended Attribute support. + + Copyright (C) 2016, 2018-2020 Free Software Foundation, Inc. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. */ + +#include "wget.h" + +#include <stdio.h> +#include <string.h> + +#include "log.h" +#include "utils.h" +#include "xattr.h" + +#ifdef USE_XATTR + +static int +write_xattr_metadata (const char *name, const char *value, FILE *fp) +{ + int retval = -1; + + if (name && value && fp) + { + retval = fsetxattr (fileno (fp), name, value, strlen (value), 0); + /* FreeBSD's extattr_set_fd returns the length of the extended attribute. */ + retval = (retval < 0) ? retval : 0; + if (retval) + DEBUGP (("Failed to set xattr %s.\n", quote(name))); + } + + return retval; +} + +#else /* USE_XATTR */ + +static int +write_xattr_metadata (const char *name, const char *value, FILE *fp) +{ + (void)name; + (void)value; + (void)fp; + + return 0; +} + +#endif /* USE_XATTR */ + +int +set_file_metadata (const struct url *origin_url, const struct url *referrer_url, FILE *fp) +{ + /* Save metadata about where the file came from (requested, final URLs) to + * user POSIX Extended Attributes of retrieved file. + * + * For more details about the user namespace see + * [http://freedesktop.org/wiki/CommonExtendedAttributes] and + * [http://0pointer.de/lennart/projects/mod_mime_xattr/]. + */ + int retval = -1; + char *value; + + if (!origin_url || !fp) + return retval; + + value = url_string (origin_url, URL_AUTH_HIDE); + retval = write_xattr_metadata ("user.xdg.origin.url", escnonprint_uri (value), fp); + xfree (value); + + if (!retval && referrer_url) + { + struct url u; + + memset(&u, 0, sizeof(u)); + u.scheme = referrer_url->scheme; + u.host = referrer_url->host; + u.port = referrer_url->port; + + value = url_string (&u, 0); + retval = write_xattr_metadata ("user.xdg.referrer.url", escnonprint_uri (value), fp); + xfree (value); + } + + return retval; +} diff --git a/src/xattr.h b/src/xattr.h new file mode 100644 index 0000000..20eebe0 --- /dev/null +++ b/src/xattr.h @@ -0,0 +1,46 @@ +/* xattr.h -- POSIX Extended Attribute function mappings. + + Copyright (C) 2016, 2018-2020 Free Software Foundation, Inc. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. */ + +#include <stdio.h> +#include <url.h> + +#ifndef _XATTR_H +#define _XATTR_H + +/* Store metadata name/value attributes against fp. */ +int set_file_metadata (const struct url *origin_url, const struct url *referrer_url, FILE *fp); + +#if defined(__linux) +/* libc on Linux has fsetxattr (5 arguments). */ +# include <sys/xattr.h> +# define USE_XATTR +#elif defined(__APPLE__) +/* libc on OS/X has fsetxattr (6 arguments). */ +# include <sys/xattr.h> +# define fsetxattr(file, name, buffer, size, flags) \ + fsetxattr ((file), (name), (buffer), (size), 0, (flags)) +# define USE_XATTR +#elif defined(__FreeBSD_version) && (__FreeBSD_version > 500000) +/* FreeBSD */ +# include <sys/types.h> +# include <sys/extattr.h> +# define fsetxattr(file, name, buffer, size, flags) \ + extattr_set_fd ((file), EXTATTR_NAMESPACE_USER, (name), (buffer), (size)) +# define USE_XATTR +#endif + +#endif /* _XATTR_H */ |