diff options
Diffstat (limited to '')
56 files changed, 31826 insertions, 0 deletions
diff --git a/src/lib-http/Makefile.am b/src/lib-http/Makefile.am new file mode 100644 index 0000000..081d39f --- /dev/null +++ b/src/lib-http/Makefile.am @@ -0,0 +1,211 @@ +noinst_LTLIBRARIES = libhttp.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-dns \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -I$(top_srcdir)/src/lib-master \ + -DPKG_RUNDIR=\""$(rundir)"\" + +libhttp_la_SOURCES = \ + http-date.c \ + http-url.c \ + http-parser.c \ + http-header.c \ + http-header-parser.c \ + http-transfer-chunked.c \ + http-auth.c \ + http-message-parser.c \ + http-request.c \ + http-request-parser.c \ + http-response.c \ + http-response-parser.c \ + http-client-request.c \ + http-client-connection.c \ + http-client-peer.c \ + http-client-queue.c \ + http-client-host.c \ + http-client.c \ + http-server-ostream.c \ + http-server-response.c \ + http-server-request.c \ + http-server-connection.c \ + http-server-resource.c \ + http-server.c + +headers = \ + http-common.h \ + http-date.h \ + http-url.h \ + http-parser.h \ + http-header.h \ + http-header-parser.h \ + http-transfer.h \ + http-auth.h \ + http-message-parser.h \ + http-request.h \ + http-request-parser.h \ + http-response.h \ + http-response-parser.h \ + http-client-private.h \ + http-client.h \ + http-server-private.h \ + http-server.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) + +test_programs = \ + test-http-date \ + test-http-url \ + test-http-header-parser \ + test-http-transfer \ + test-http-auth \ + test-http-response-parser \ + test-http-request-parser \ + test-http-payload \ + test-http-client-errors \ + test-http-client-request \ + test-http-server-errors + +test_nocheck_programs = \ + test-http-client \ + test-http-server + +noinst_PROGRAMS = $(test_programs) $(test_nocheck_programs) + +test_libs = \ + ../lib-test/libtest.la \ + ../lib/liblib.la \ + $(MODULE_LIBS) + +test_deps = \ + $(noinst_LTLIBRARIES) \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_http_url_SOURCES = test-http-url.c +test_http_url_LDADD = http-url.lo http-header.lo $(test_libs) +test_http_url_DEPENDENCIES = $(test_deps) + +test_http_date_SOURCES = test-http-date.c +test_http_date_LDADD = http-date.lo $(test_libs) +test_http_date_DEPENDENCIES = $(test_deps) + +test_http_header_parser_SOURCES = test-http-header-parser.c +test_http_header_parser_LDADD = http-parser.lo http-header-parser.lo http-header.lo $(test_libs) +test_http_header_parser_DEPENDENCIES = $(test_deps) + +test_http_transfer_SOURCES = test-http-transfer.c +test_http_transfer_LDADD = \ + http-parser.lo \ + http-header-parser.lo \ + http-transfer-chunked.lo \ + http-header.lo \ + $(test_libs) +test_http_transfer_DEPENDENCIES = $(test_deps) + +test_http_auth_SOURCES = test-http-auth.c +test_http_auth_LDADD = \ + http-auth.lo \ + http-parser.lo \ + $(test_libs) +test_http_auth_DEPENDENCIES = $(test_deps) + +test_http_response_parser_SOURCES = test-http-response-parser.c +test_http_response_parser_LDADD = \ + http-date.lo \ + http-parser.lo \ + http-header.lo \ + http-header-parser.lo \ + http-transfer-chunked.lo \ + http-message-parser.lo \ + http-response-parser.lo \ + $(test_libs) +test_http_response_parser_DEPENDENCIES = $(test_deps) + +test_http_request_parser_SOURCES = test-http-request-parser.c +test_http_request_parser_LDADD = \ + http-date.lo \ + http-parser.lo \ + http-url.lo \ + http-header.lo \ + http-header-parser.lo \ + http-transfer-chunked.lo \ + http-message-parser.lo \ + http-request-parser.lo \ + $(test_libs) +test_http_request_parser_DEPENDENCIES = $(test_deps) + +test_http_libs = \ + libhttp.la \ + ../lib-dns/libdns.la \ + ../lib-ssl-iostream/libssl_iostream.la \ + ../lib-master/libmaster.la \ + ../lib-auth/libauth.la \ + ../lib-settings/libsettings.la \ + $(test_libs) +test_http_deps = \ + libhttp.la \ + ../lib-dns/libdns.la \ + ../lib-ssl-iostream/libssl_iostream.la \ + ../lib-master/libmaster.la \ + ../lib-auth/libauth.la \ + ../lib-settings/libsettings.la \ + $(test_deps) + +test_http_libs_ssl= +if BUILD_OPENSSL +test_http_libs_ssl += ../lib-ssl-iostream/libssl_iostream_openssl.la +endif + +test_http_payload_SOURCES = test-http-payload.c +test_http_payload_LDFLAGS = -export-dynamic +test_http_payload_LDADD = \ + $(test_http_libs) \ + $(test_http_libs_ssl) +test_http_payload_DEPENDENCIES = \ + $(test_http_deps) + +test_http_client_SOURCES = test-http-client.c +test_http_client_LDFLAGS = -export-dynamic +test_http_client_LDADD = \ + $(test_http_libs) \ + $(test_http_libs_ssl) + +test_http_client_DEPENDENCIES = \ + $(test_http_deps) + +test_http_client_errors_SOURCES = test-http-client-errors.c +test_http_client_errors_LDFLAGS = -export-dynamic +test_http_client_errors_LDADD = \ + $(test_http_libs) +test_http_client_errors_DEPENDENCIES = \ + $(test_http_deps) + +test_http_client_request_SOURCES = test-http-client-request.c +test_http_client_request_LDFLAGS = -export-dynamic +test_http_client_request_LDADD = \ + $(test_http_libs) +test_http_client_request_DEPENDENCIES = \ + $(test_http_deps) + +test_http_server_SOURCES = test-http-server.c +test_http_server_LDFLAGS = -export-dynamic +test_http_server_LDADD = \ + $(test_http_libs) +test_http_server_DEPENDENCIES = \ + $(test_http_deps) + +test_http_server_errors_SOURCES = test-http-server-errors.c +test_http_server_errors_LDFLAGS = -export-dynamic +test_http_server_errors_LDADD = \ + $(test_http_libs) +test_http_server_errors_DEPENDENCIES = \ + $(test_http_deps) + +check-local: + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done diff --git a/src/lib-http/Makefile.in b/src/lib-http/Makefile.in new file mode 100644 index 0000000..6379099 --- /dev/null +++ b/src/lib-http/Makefile.in @@ -0,0 +1,1310 @@ +# Makefile.in generated by automake 1.16.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2018 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +noinst_PROGRAMS = $(am__EXEEXT_1) $(am__EXEEXT_2) +@BUILD_OPENSSL_TRUE@am__append_1 = ../lib-ssl-iostream/libssl_iostream_openssl.la +subdir = src/lib-http +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(pkginc_lib_HEADERS) \ + $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__EXEEXT_1 = test-http-date$(EXEEXT) test-http-url$(EXEEXT) \ + test-http-header-parser$(EXEEXT) test-http-transfer$(EXEEXT) \ + test-http-auth$(EXEEXT) test-http-response-parser$(EXEEXT) \ + test-http-request-parser$(EXEEXT) test-http-payload$(EXEEXT) \ + test-http-client-errors$(EXEEXT) \ + test-http-client-request$(EXEEXT) \ + test-http-server-errors$(EXEEXT) +am__EXEEXT_2 = test-http-client$(EXEEXT) test-http-server$(EXEEXT) +PROGRAMS = $(noinst_PROGRAMS) +LTLIBRARIES = $(noinst_LTLIBRARIES) +libhttp_la_LIBADD = +am_libhttp_la_OBJECTS = http-date.lo http-url.lo http-parser.lo \ + http-header.lo http-header-parser.lo http-transfer-chunked.lo \ + http-auth.lo http-message-parser.lo http-request.lo \ + http-request-parser.lo http-response.lo \ + http-response-parser.lo http-client-request.lo \ + http-client-connection.lo http-client-peer.lo \ + http-client-queue.lo http-client-host.lo http-client.lo \ + http-server-ostream.lo http-server-response.lo \ + http-server-request.lo http-server-connection.lo \ + http-server-resource.lo http-server.lo +libhttp_la_OBJECTS = $(am_libhttp_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +am_test_http_auth_OBJECTS = test-http-auth.$(OBJEXT) +test_http_auth_OBJECTS = $(am_test_http_auth_OBJECTS) +am__DEPENDENCIES_1 = +am__DEPENDENCIES_2 = ../lib-test/libtest.la ../lib/liblib.la \ + $(am__DEPENDENCIES_1) +am_test_http_client_OBJECTS = test-http-client.$(OBJEXT) +test_http_client_OBJECTS = $(am_test_http_client_OBJECTS) +am__DEPENDENCIES_3 = libhttp.la ../lib-dns/libdns.la \ + ../lib-ssl-iostream/libssl_iostream.la \ + ../lib-master/libmaster.la ../lib-auth/libauth.la \ + ../lib-settings/libsettings.la $(am__DEPENDENCIES_2) +test_http_client_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \ + $(AM_CFLAGS) $(CFLAGS) $(test_http_client_LDFLAGS) $(LDFLAGS) \ + -o $@ +am_test_http_client_errors_OBJECTS = \ + test-http-client-errors.$(OBJEXT) +test_http_client_errors_OBJECTS = \ + $(am_test_http_client_errors_OBJECTS) +test_http_client_errors_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \ + $(AM_CFLAGS) $(CFLAGS) $(test_http_client_errors_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_test_http_client_request_OBJECTS = \ + test-http-client-request.$(OBJEXT) +test_http_client_request_OBJECTS = \ + $(am_test_http_client_request_OBJECTS) +test_http_client_request_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \ + $(AM_CFLAGS) $(CFLAGS) $(test_http_client_request_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_test_http_date_OBJECTS = test-http-date.$(OBJEXT) +test_http_date_OBJECTS = $(am_test_http_date_OBJECTS) +am_test_http_header_parser_OBJECTS = \ + test-http-header-parser.$(OBJEXT) +test_http_header_parser_OBJECTS = \ + $(am_test_http_header_parser_OBJECTS) +am_test_http_payload_OBJECTS = test-http-payload.$(OBJEXT) +test_http_payload_OBJECTS = $(am_test_http_payload_OBJECTS) +test_http_payload_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \ + $(AM_CFLAGS) $(CFLAGS) $(test_http_payload_LDFLAGS) $(LDFLAGS) \ + -o $@ +am_test_http_request_parser_OBJECTS = \ + test-http-request-parser.$(OBJEXT) +test_http_request_parser_OBJECTS = \ + $(am_test_http_request_parser_OBJECTS) +am_test_http_response_parser_OBJECTS = \ + test-http-response-parser.$(OBJEXT) +test_http_response_parser_OBJECTS = \ + $(am_test_http_response_parser_OBJECTS) +am_test_http_server_OBJECTS = test-http-server.$(OBJEXT) +test_http_server_OBJECTS = $(am_test_http_server_OBJECTS) +test_http_server_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \ + $(AM_CFLAGS) $(CFLAGS) $(test_http_server_LDFLAGS) $(LDFLAGS) \ + -o $@ +am_test_http_server_errors_OBJECTS = \ + test-http-server-errors.$(OBJEXT) +test_http_server_errors_OBJECTS = \ + $(am_test_http_server_errors_OBJECTS) +test_http_server_errors_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \ + $(AM_CFLAGS) $(CFLAGS) $(test_http_server_errors_LDFLAGS) \ + $(LDFLAGS) -o $@ +am_test_http_transfer_OBJECTS = test-http-transfer.$(OBJEXT) +test_http_transfer_OBJECTS = $(am_test_http_transfer_OBJECTS) +am_test_http_url_OBJECTS = test-http-url.$(OBJEXT) +test_http_url_OBJECTS = $(am_test_http_url_OBJECTS) +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/http-auth.Plo \ + ./$(DEPDIR)/http-client-connection.Plo \ + ./$(DEPDIR)/http-client-host.Plo \ + ./$(DEPDIR)/http-client-peer.Plo \ + ./$(DEPDIR)/http-client-queue.Plo \ + ./$(DEPDIR)/http-client-request.Plo \ + ./$(DEPDIR)/http-client.Plo ./$(DEPDIR)/http-date.Plo \ + ./$(DEPDIR)/http-header-parser.Plo ./$(DEPDIR)/http-header.Plo \ + ./$(DEPDIR)/http-message-parser.Plo \ + ./$(DEPDIR)/http-parser.Plo \ + ./$(DEPDIR)/http-request-parser.Plo \ + ./$(DEPDIR)/http-request.Plo \ + ./$(DEPDIR)/http-response-parser.Plo \ + ./$(DEPDIR)/http-response.Plo \ + ./$(DEPDIR)/http-server-connection.Plo \ + ./$(DEPDIR)/http-server-ostream.Plo \ + ./$(DEPDIR)/http-server-request.Plo \ + ./$(DEPDIR)/http-server-resource.Plo \ + ./$(DEPDIR)/http-server-response.Plo \ + ./$(DEPDIR)/http-server.Plo \ + ./$(DEPDIR)/http-transfer-chunked.Plo ./$(DEPDIR)/http-url.Plo \ + ./$(DEPDIR)/test-http-auth.Po \ + ./$(DEPDIR)/test-http-client-errors.Po \ + ./$(DEPDIR)/test-http-client-request.Po \ + ./$(DEPDIR)/test-http-client.Po ./$(DEPDIR)/test-http-date.Po \ + ./$(DEPDIR)/test-http-header-parser.Po \ + ./$(DEPDIR)/test-http-payload.Po \ + ./$(DEPDIR)/test-http-request-parser.Po \ + ./$(DEPDIR)/test-http-response-parser.Po \ + ./$(DEPDIR)/test-http-server-errors.Po \ + ./$(DEPDIR)/test-http-server.Po \ + ./$(DEPDIR)/test-http-transfer.Po ./$(DEPDIR)/test-http-url.Po +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libhttp_la_SOURCES) $(test_http_auth_SOURCES) \ + $(test_http_client_SOURCES) $(test_http_client_errors_SOURCES) \ + $(test_http_client_request_SOURCES) $(test_http_date_SOURCES) \ + $(test_http_header_parser_SOURCES) \ + $(test_http_payload_SOURCES) \ + $(test_http_request_parser_SOURCES) \ + $(test_http_response_parser_SOURCES) \ + $(test_http_server_SOURCES) $(test_http_server_errors_SOURCES) \ + $(test_http_transfer_SOURCES) $(test_http_url_SOURCES) +DIST_SOURCES = $(libhttp_la_SOURCES) $(test_http_auth_SOURCES) \ + $(test_http_client_SOURCES) $(test_http_client_errors_SOURCES) \ + $(test_http_client_request_SOURCES) $(test_http_date_SOURCES) \ + $(test_http_header_parser_SOURCES) \ + $(test_http_payload_SOURCES) \ + $(test_http_request_parser_SOURCES) \ + $(test_http_response_parser_SOURCES) \ + $(test_http_server_SOURCES) $(test_http_server_errors_SOURCES) \ + $(test_http_transfer_SOURCES) $(test_http_url_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkginc_libdir)" +HEADERS = $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libhttp.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-dns \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -I$(top_srcdir)/src/lib-master \ + -DPKG_RUNDIR=\""$(rundir)"\" + +libhttp_la_SOURCES = \ + http-date.c \ + http-url.c \ + http-parser.c \ + http-header.c \ + http-header-parser.c \ + http-transfer-chunked.c \ + http-auth.c \ + http-message-parser.c \ + http-request.c \ + http-request-parser.c \ + http-response.c \ + http-response-parser.c \ + http-client-request.c \ + http-client-connection.c \ + http-client-peer.c \ + http-client-queue.c \ + http-client-host.c \ + http-client.c \ + http-server-ostream.c \ + http-server-response.c \ + http-server-request.c \ + http-server-connection.c \ + http-server-resource.c \ + http-server.c + +headers = \ + http-common.h \ + http-date.h \ + http-url.h \ + http-parser.h \ + http-header.h \ + http-header-parser.h \ + http-transfer.h \ + http-auth.h \ + http-message-parser.h \ + http-request.h \ + http-request-parser.h \ + http-response.h \ + http-response-parser.h \ + http-client-private.h \ + http-client.h \ + http-server-private.h \ + http-server.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) +test_programs = \ + test-http-date \ + test-http-url \ + test-http-header-parser \ + test-http-transfer \ + test-http-auth \ + test-http-response-parser \ + test-http-request-parser \ + test-http-payload \ + test-http-client-errors \ + test-http-client-request \ + test-http-server-errors + +test_nocheck_programs = \ + test-http-client \ + test-http-server + +test_libs = \ + ../lib-test/libtest.la \ + ../lib/liblib.la \ + $(MODULE_LIBS) + +test_deps = \ + $(noinst_LTLIBRARIES) \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_http_url_SOURCES = test-http-url.c +test_http_url_LDADD = http-url.lo http-header.lo $(test_libs) +test_http_url_DEPENDENCIES = $(test_deps) +test_http_date_SOURCES = test-http-date.c +test_http_date_LDADD = http-date.lo $(test_libs) +test_http_date_DEPENDENCIES = $(test_deps) +test_http_header_parser_SOURCES = test-http-header-parser.c +test_http_header_parser_LDADD = http-parser.lo http-header-parser.lo http-header.lo $(test_libs) +test_http_header_parser_DEPENDENCIES = $(test_deps) +test_http_transfer_SOURCES = test-http-transfer.c +test_http_transfer_LDADD = \ + http-parser.lo \ + http-header-parser.lo \ + http-transfer-chunked.lo \ + http-header.lo \ + $(test_libs) + +test_http_transfer_DEPENDENCIES = $(test_deps) +test_http_auth_SOURCES = test-http-auth.c +test_http_auth_LDADD = \ + http-auth.lo \ + http-parser.lo \ + $(test_libs) + +test_http_auth_DEPENDENCIES = $(test_deps) +test_http_response_parser_SOURCES = test-http-response-parser.c +test_http_response_parser_LDADD = \ + http-date.lo \ + http-parser.lo \ + http-header.lo \ + http-header-parser.lo \ + http-transfer-chunked.lo \ + http-message-parser.lo \ + http-response-parser.lo \ + $(test_libs) + +test_http_response_parser_DEPENDENCIES = $(test_deps) +test_http_request_parser_SOURCES = test-http-request-parser.c +test_http_request_parser_LDADD = \ + http-date.lo \ + http-parser.lo \ + http-url.lo \ + http-header.lo \ + http-header-parser.lo \ + http-transfer-chunked.lo \ + http-message-parser.lo \ + http-request-parser.lo \ + $(test_libs) + +test_http_request_parser_DEPENDENCIES = $(test_deps) +test_http_libs = \ + libhttp.la \ + ../lib-dns/libdns.la \ + ../lib-ssl-iostream/libssl_iostream.la \ + ../lib-master/libmaster.la \ + ../lib-auth/libauth.la \ + ../lib-settings/libsettings.la \ + $(test_libs) + +test_http_deps = \ + libhttp.la \ + ../lib-dns/libdns.la \ + ../lib-ssl-iostream/libssl_iostream.la \ + ../lib-master/libmaster.la \ + ../lib-auth/libauth.la \ + ../lib-settings/libsettings.la \ + $(test_deps) + +test_http_libs_ssl = $(am__append_1) +test_http_payload_SOURCES = test-http-payload.c +test_http_payload_LDFLAGS = -export-dynamic +test_http_payload_LDADD = \ + $(test_http_libs) \ + $(test_http_libs_ssl) + +test_http_payload_DEPENDENCIES = \ + $(test_http_deps) + +test_http_client_SOURCES = test-http-client.c +test_http_client_LDFLAGS = -export-dynamic +test_http_client_LDADD = \ + $(test_http_libs) \ + $(test_http_libs_ssl) + +test_http_client_DEPENDENCIES = \ + $(test_http_deps) + +test_http_client_errors_SOURCES = test-http-client-errors.c +test_http_client_errors_LDFLAGS = -export-dynamic +test_http_client_errors_LDADD = \ + $(test_http_libs) + +test_http_client_errors_DEPENDENCIES = \ + $(test_http_deps) + +test_http_client_request_SOURCES = test-http-client-request.c +test_http_client_request_LDFLAGS = -export-dynamic +test_http_client_request_LDADD = \ + $(test_http_libs) + +test_http_client_request_DEPENDENCIES = \ + $(test_http_deps) + +test_http_server_SOURCES = test-http-server.c +test_http_server_LDFLAGS = -export-dynamic +test_http_server_LDADD = \ + $(test_http_libs) + +test_http_server_DEPENDENCIES = \ + $(test_http_deps) + +test_http_server_errors_SOURCES = test-http-server-errors.c +test_http_server_errors_LDFLAGS = -export-dynamic +test_http_server_errors_LDADD = \ + $(test_http_libs) + +test_http_server_errors_DEPENDENCIES = \ + $(test_http_deps) + +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/lib-http/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/lib-http/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libhttp.la: $(libhttp_la_OBJECTS) $(libhttp_la_DEPENDENCIES) $(EXTRA_libhttp_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libhttp_la_OBJECTS) $(libhttp_la_LIBADD) $(LIBS) + +test-http-auth$(EXEEXT): $(test_http_auth_OBJECTS) $(test_http_auth_DEPENDENCIES) $(EXTRA_test_http_auth_DEPENDENCIES) + @rm -f test-http-auth$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_http_auth_OBJECTS) $(test_http_auth_LDADD) $(LIBS) + +test-http-client$(EXEEXT): $(test_http_client_OBJECTS) $(test_http_client_DEPENDENCIES) $(EXTRA_test_http_client_DEPENDENCIES) + @rm -f test-http-client$(EXEEXT) + $(AM_V_CCLD)$(test_http_client_LINK) $(test_http_client_OBJECTS) $(test_http_client_LDADD) $(LIBS) + +test-http-client-errors$(EXEEXT): $(test_http_client_errors_OBJECTS) $(test_http_client_errors_DEPENDENCIES) $(EXTRA_test_http_client_errors_DEPENDENCIES) + @rm -f test-http-client-errors$(EXEEXT) + $(AM_V_CCLD)$(test_http_client_errors_LINK) $(test_http_client_errors_OBJECTS) $(test_http_client_errors_LDADD) $(LIBS) + +test-http-client-request$(EXEEXT): $(test_http_client_request_OBJECTS) $(test_http_client_request_DEPENDENCIES) $(EXTRA_test_http_client_request_DEPENDENCIES) + @rm -f test-http-client-request$(EXEEXT) + $(AM_V_CCLD)$(test_http_client_request_LINK) $(test_http_client_request_OBJECTS) $(test_http_client_request_LDADD) $(LIBS) + +test-http-date$(EXEEXT): $(test_http_date_OBJECTS) $(test_http_date_DEPENDENCIES) $(EXTRA_test_http_date_DEPENDENCIES) + @rm -f test-http-date$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_http_date_OBJECTS) $(test_http_date_LDADD) $(LIBS) + +test-http-header-parser$(EXEEXT): $(test_http_header_parser_OBJECTS) $(test_http_header_parser_DEPENDENCIES) $(EXTRA_test_http_header_parser_DEPENDENCIES) + @rm -f test-http-header-parser$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_http_header_parser_OBJECTS) $(test_http_header_parser_LDADD) $(LIBS) + +test-http-payload$(EXEEXT): $(test_http_payload_OBJECTS) $(test_http_payload_DEPENDENCIES) $(EXTRA_test_http_payload_DEPENDENCIES) + @rm -f test-http-payload$(EXEEXT) + $(AM_V_CCLD)$(test_http_payload_LINK) $(test_http_payload_OBJECTS) $(test_http_payload_LDADD) $(LIBS) + +test-http-request-parser$(EXEEXT): $(test_http_request_parser_OBJECTS) $(test_http_request_parser_DEPENDENCIES) $(EXTRA_test_http_request_parser_DEPENDENCIES) + @rm -f test-http-request-parser$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_http_request_parser_OBJECTS) $(test_http_request_parser_LDADD) $(LIBS) + +test-http-response-parser$(EXEEXT): $(test_http_response_parser_OBJECTS) $(test_http_response_parser_DEPENDENCIES) $(EXTRA_test_http_response_parser_DEPENDENCIES) + @rm -f test-http-response-parser$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_http_response_parser_OBJECTS) $(test_http_response_parser_LDADD) $(LIBS) + +test-http-server$(EXEEXT): $(test_http_server_OBJECTS) $(test_http_server_DEPENDENCIES) $(EXTRA_test_http_server_DEPENDENCIES) + @rm -f test-http-server$(EXEEXT) + $(AM_V_CCLD)$(test_http_server_LINK) $(test_http_server_OBJECTS) $(test_http_server_LDADD) $(LIBS) + +test-http-server-errors$(EXEEXT): $(test_http_server_errors_OBJECTS) $(test_http_server_errors_DEPENDENCIES) $(EXTRA_test_http_server_errors_DEPENDENCIES) + @rm -f test-http-server-errors$(EXEEXT) + $(AM_V_CCLD)$(test_http_server_errors_LINK) $(test_http_server_errors_OBJECTS) $(test_http_server_errors_LDADD) $(LIBS) + +test-http-transfer$(EXEEXT): $(test_http_transfer_OBJECTS) $(test_http_transfer_DEPENDENCIES) $(EXTRA_test_http_transfer_DEPENDENCIES) + @rm -f test-http-transfer$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_http_transfer_OBJECTS) $(test_http_transfer_LDADD) $(LIBS) + +test-http-url$(EXEEXT): $(test_http_url_OBJECTS) $(test_http_url_DEPENDENCIES) $(EXTRA_test_http_url_DEPENDENCIES) + @rm -f test-http-url$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_http_url_OBJECTS) $(test_http_url_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-auth.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-client-connection.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-client-host.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-client-peer.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-client-queue.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-client-request.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-client.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-date.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-header-parser.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-header.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-message-parser.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-parser.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-request-parser.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-request.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-response-parser.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-response.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-server-connection.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-server-ostream.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-server-request.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-server-resource.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-server-response.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-server.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-transfer-chunked.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/http-url.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-http-auth.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-http-client-errors.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-http-client-request.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-http-client.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-http-date.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-http-header-parser.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-http-payload.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-http-request-parser.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-http-response-parser.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-http-server-errors.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-http-server.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-http-transfer.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-http-url.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) check-local +check: check-am +all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + clean-noinstPROGRAMS mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/http-auth.Plo + -rm -f ./$(DEPDIR)/http-client-connection.Plo + -rm -f ./$(DEPDIR)/http-client-host.Plo + -rm -f ./$(DEPDIR)/http-client-peer.Plo + -rm -f ./$(DEPDIR)/http-client-queue.Plo + -rm -f ./$(DEPDIR)/http-client-request.Plo + -rm -f ./$(DEPDIR)/http-client.Plo + -rm -f ./$(DEPDIR)/http-date.Plo + -rm -f ./$(DEPDIR)/http-header-parser.Plo + -rm -f ./$(DEPDIR)/http-header.Plo + -rm -f ./$(DEPDIR)/http-message-parser.Plo + -rm -f ./$(DEPDIR)/http-parser.Plo + -rm -f ./$(DEPDIR)/http-request-parser.Plo + -rm -f ./$(DEPDIR)/http-request.Plo + -rm -f ./$(DEPDIR)/http-response-parser.Plo + -rm -f ./$(DEPDIR)/http-response.Plo + -rm -f ./$(DEPDIR)/http-server-connection.Plo + -rm -f ./$(DEPDIR)/http-server-ostream.Plo + -rm -f ./$(DEPDIR)/http-server-request.Plo + -rm -f ./$(DEPDIR)/http-server-resource.Plo + -rm -f ./$(DEPDIR)/http-server-response.Plo + -rm -f ./$(DEPDIR)/http-server.Plo + -rm -f ./$(DEPDIR)/http-transfer-chunked.Plo + -rm -f ./$(DEPDIR)/http-url.Plo + -rm -f ./$(DEPDIR)/test-http-auth.Po + -rm -f ./$(DEPDIR)/test-http-client-errors.Po + -rm -f ./$(DEPDIR)/test-http-client-request.Po + -rm -f ./$(DEPDIR)/test-http-client.Po + -rm -f ./$(DEPDIR)/test-http-date.Po + -rm -f ./$(DEPDIR)/test-http-header-parser.Po + -rm -f ./$(DEPDIR)/test-http-payload.Po + -rm -f ./$(DEPDIR)/test-http-request-parser.Po + -rm -f ./$(DEPDIR)/test-http-response-parser.Po + -rm -f ./$(DEPDIR)/test-http-server-errors.Po + -rm -f ./$(DEPDIR)/test-http-server.Po + -rm -f ./$(DEPDIR)/test-http-transfer.Po + -rm -f ./$(DEPDIR)/test-http-url.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/http-auth.Plo + -rm -f ./$(DEPDIR)/http-client-connection.Plo + -rm -f ./$(DEPDIR)/http-client-host.Plo + -rm -f ./$(DEPDIR)/http-client-peer.Plo + -rm -f ./$(DEPDIR)/http-client-queue.Plo + -rm -f ./$(DEPDIR)/http-client-request.Plo + -rm -f ./$(DEPDIR)/http-client.Plo + -rm -f ./$(DEPDIR)/http-date.Plo + -rm -f ./$(DEPDIR)/http-header-parser.Plo + -rm -f ./$(DEPDIR)/http-header.Plo + -rm -f ./$(DEPDIR)/http-message-parser.Plo + -rm -f ./$(DEPDIR)/http-parser.Plo + -rm -f ./$(DEPDIR)/http-request-parser.Plo + -rm -f ./$(DEPDIR)/http-request.Plo + -rm -f ./$(DEPDIR)/http-response-parser.Plo + -rm -f ./$(DEPDIR)/http-response.Plo + -rm -f ./$(DEPDIR)/http-server-connection.Plo + -rm -f ./$(DEPDIR)/http-server-ostream.Plo + -rm -f ./$(DEPDIR)/http-server-request.Plo + -rm -f ./$(DEPDIR)/http-server-resource.Plo + -rm -f ./$(DEPDIR)/http-server-response.Plo + -rm -f ./$(DEPDIR)/http-server.Plo + -rm -f ./$(DEPDIR)/http-transfer-chunked.Plo + -rm -f ./$(DEPDIR)/http-url.Plo + -rm -f ./$(DEPDIR)/test-http-auth.Po + -rm -f ./$(DEPDIR)/test-http-client-errors.Po + -rm -f ./$(DEPDIR)/test-http-client-request.Po + -rm -f ./$(DEPDIR)/test-http-client.Po + -rm -f ./$(DEPDIR)/test-http-date.Po + -rm -f ./$(DEPDIR)/test-http-header-parser.Po + -rm -f ./$(DEPDIR)/test-http-payload.Po + -rm -f ./$(DEPDIR)/test-http-request-parser.Po + -rm -f ./$(DEPDIR)/test-http-response-parser.Po + -rm -f ./$(DEPDIR)/test-http-server-errors.Po + -rm -f ./$(DEPDIR)/test-http-server.Po + -rm -f ./$(DEPDIR)/test-http-transfer.Po + -rm -f ./$(DEPDIR)/test-http-url.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS + +.MAKE: check-am install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am \ + check-local clean clean-generic clean-libtool \ + clean-noinstLTLIBRARIES clean-noinstPROGRAMS cscopelist-am \ + ctags ctags-am distclean distclean-compile distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-data \ + install-data-am install-dvi install-dvi-am install-exec \ + install-exec-am install-html install-html-am install-info \ + install-info-am install-man install-pdf install-pdf-am \ + install-pkginc_libHEADERS install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ + pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \ + uninstall-pkginc_libHEADERS + +.PRECIOUS: Makefile + + +check-local: + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/lib-http/http-auth.c b/src/lib-http/http-auth.c new file mode 100644 index 0000000..67b7f6b --- /dev/null +++ b/src/lib-http/http-auth.c @@ -0,0 +1,476 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "base64.h" +#include "array.h" +#include "http-parser.h" + +#include "http-auth.h" + +/* RFC 7235, Section 2.1: + + challenge = auth-scheme [ 1*SP ( token68 / #auth-param ) ] + credentials = auth-scheme [ 1*SP ( token68 / #auth-param ) ] + + auth-scheme = token + auth-param = token BWS "=" BWS ( token / quoted-string ) + token68 = 1*( ALPHA / DIGIT / + "-" / "." / "_" / "~" / "+" / "/" ) *"=" + + OWS = *( SP / HTAB ) + ; optional whitespace + BWS = OWS + ; "bad" whitespace + */ + +/* + * Parsing + */ + +static int +http_parse_token68(struct http_parser *parser, const char **token68_r) +{ + const unsigned char *first; + + /* token68 = 1*( ALPHA / DIGIT / + "-" / "." / "_" / "~" / "+" / "/" ) *"=" + */ + + /* 1*( ALPHA / DIGIT / "-" / "." / "_" / "~" / "+" / "/" ) */ + if (parser->cur >= parser->end || !http_char_is_token68(*parser->cur)) + return 0; + first = parser->cur++; + while (parser->cur < parser->end && http_char_is_token68(*parser->cur)) + parser->cur++; + + /* *"=" */ + while (parser->cur < parser->end && *parser->cur == '=') + parser->cur++; + + *token68_r = t_strndup(first, parser->cur - first); + return 1; +} + +static int +http_parse_auth_param(struct http_parser *parser, + const char **param_r, const char **value_r) +{ + const unsigned char *first = parser->cur, *end_token; + int ret; + + /* auth-param = token BWS "=" BWS ( token / quoted-string ) */ + + /* token */ + if ((ret=http_parser_skip_token(parser)) <= 0) { + parser->cur = first; + return ret; + } + end_token = parser->cur; + + /* BWS "=" BWS */ + http_parse_ows(parser); + if (parser->cur >= parser->end || *parser->cur != '=') { + parser->cur = first; + return 0; + } + parser->cur++; + http_parse_ows(parser); + + /* ( token / quoted-string ) */ + if ((ret=http_parse_token_or_qstring(parser, value_r)) <= 0) { + parser->cur = first; + return ret; + } + + *param_r = t_strndup(first, end_token - first); + return 1; +} + +static int +http_parse_auth_params(struct http_parser *parser, + ARRAY_TYPE(http_auth_param) *params) +{ + const unsigned char *last = parser->cur; + struct http_auth_param param; + unsigned int count = 0; + int ret; + + i_zero(¶m); + while ((ret=http_parse_auth_param + (parser, ¶m.name, ¶m.value)) > 0) { + if (!array_is_created(params)) + t_array_init(params, 4); + array_push_back(params, ¶m); + count++; + + last = parser->cur; + + /* OWS "," OWS + --> also allow empty elements + */ + for (;;) { + http_parse_ows(parser); + if (parser->cur >= parser->end || *parser->cur != ',') + break; + parser->cur++; + } + } + + parser->cur = last; + if (ret < 0) + return -1; + return (count > 0 ? 1 : 0); +} + +int http_auth_parse_challenges(const unsigned char *data, size_t size, + ARRAY_TYPE(http_auth_challenge) *chlngs) +{ + struct http_parser parser; + int ret; + + http_parser_init(&parser, data, size); + + /* WWW-Authenticate = 1#challenge + Proxy-Authenticate = 1#challenge + + challenge = auth-scheme [ 1*SP ( token68 / #auth-param ) ] + auth-scheme = token + */ + + /* 1#element => *( "," OWS ) ... ; RFC 7230, Section 7 */ + for (;;) { + if (parser.cur >= parser.end || *parser.cur != ',') + break; + parser.cur++; + http_parse_ows(&parser); + } + + for (;;) { + struct http_auth_challenge chlng; + + i_zero(&chlng); + + /* auth-scheme */ + if ((ret=http_parse_token(&parser, &chlng.scheme)) <= 0) { + if (ret < 0) + return -1; + break; + } + + /* [ 1*SP ... ] */ + if (parser.cur >= parser.end || *parser.cur != ' ') + return 1; + parser.cur++; + while (parser.cur < parser.end && *parser.cur == ' ') + parser.cur++; + + /* ( token68 / #auth-param ) */ + if ((ret=http_parse_auth_params(&parser, &chlng.params)) <= 0) { + if (ret < 0) + return -1; + if (http_parse_token68(&parser, &chlng.data) < 0) + return -1; + } + + if (!array_is_created(chlngs)) + t_array_init(chlngs, 4); + array_push_back(chlngs, &chlng); + + /* OWS "," OWS + --> also allow empty elements + */ + for (;;) { + http_parse_ows(&parser); + if (parser.cur >= parser.end || *parser.cur != ',') + break; + parser.cur++; + } + } + + if (parser.cur != parser.end) + return -1; + return 1; +} + +int http_auth_parse_credentials(const unsigned char *data, size_t size, + struct http_auth_credentials *crdts) +{ + struct http_parser parser; + int ret; + + http_parser_init(&parser, data, size); + + /* Authorization = credentials + Proxy-Authorization = credentials + + credentials = auth-scheme [ 1*SP ( token68 / #auth-param ) ] + auth-scheme = token + */ + + i_zero(crdts); + + /* auth-scheme */ + if (http_parse_token(&parser, &crdts->scheme) <= 0) + return -1; + + /* [ 1*SP ... ] */ + if (parser.cur >= parser.end || *parser.cur != ' ') + return 1; + parser.cur++; + while (parser.cur < parser.end && *parser.cur == ' ') + parser.cur++; + + /* ( token68 / #auth-param ) */ + if ((ret=http_parse_auth_params(&parser, &crdts->params)) <= 0) { + if (ret < 0) + return -1; + if (http_parse_token68(&parser, &crdts->data) < 0) + return -1; + } + + if (parser.cur != parser.end) + return -1; + return 1; +} + +/* + * Construction + */ + +static void +http_auth_create_param(string_t *out, const struct http_auth_param *param) +{ + const char *p, *first; + + /* auth-param = token BWS "=" BWS ( token / quoted-string ) */ + + str_append(out, param->name); + str_append_c(out, '='); + + for (p = param->value; *p != '\0' && http_char_is_token(*p); p++); + + if ( *p != '\0' ) { + str_append_c(out, '"'); + p = first = param->value; + while (*p != '\0') { + if (*p == '\\' || *p == '"') { + str_append_data(out, first, p-first); + str_append_c(out, '\\'); + first = p; + } + p++; + } + str_append_data(out, first, p-first); + str_append_c(out, '"'); + } else { + str_append(out, param->value); + } +} + +static void +http_auth_create_params(string_t *out, + const ARRAY_TYPE(http_auth_param) *params) +{ + const struct http_auth_param *prms; + unsigned int count, i; + + if (!array_is_created(params)) + return; + + prms = array_get(params, &count); + for (i = 0; i < count; i++) { + if (i > 0) + str_append(out, ", "); + http_auth_create_param(out, &prms[i]); + } +} + +static void http_auth_check_token68(const char *data) +{ + const char *p = data; + + /* Make sure we're not working with nonsense. */ + i_assert(http_char_is_token68(*p)); + for (p++; *p != '\0' && *p != '='; p++) + i_assert(http_char_is_token68(*p)); + for (; *p != '\0'; p++) + i_assert(*p == '='); +} + +void http_auth_create_challenge(string_t *out, + const struct http_auth_challenge *chlng) +{ + /* challenge = auth-scheme [ 1*SP ( token68 / #auth-param ) ] + auth-scheme = token + */ + + /* auth-scheme */ + str_append(out, chlng->scheme); + + if (chlng->data != NULL) { + /* SP token68 */ + http_auth_check_token68(chlng->data); + str_append_c(out, ' '); + str_append(out, chlng->data); + + } else { + /* SP #auth-param */ + str_append_c(out, ' '); + http_auth_create_params(out, &chlng->params); + } +} + +void http_auth_create_challenges(string_t *out, + const ARRAY_TYPE(http_auth_challenge) *chlngs) +{ + const struct http_auth_challenge *chlgs; + unsigned int count, i; + + /* WWW-Authenticate = 1#challenge + Proxy-Authenticate = 1#challenge + */ + chlgs = array_get(chlngs, &count); + for (i = 0; i < count; i++) { + if (i > 0) + str_append(out, ", "); + http_auth_create_challenge(out, &chlgs[i]); + } +} + +void http_auth_create_credentials(string_t *out, + const struct http_auth_credentials *crdts) +{ + /* Authorization = credentials + Proxy-Authorization = credentials + + credentials = auth-scheme [ 1*SP ( token68 / #auth-param ) ] + auth-scheme = token + */ + + /* auth-scheme */ + str_append(out, crdts->scheme); + + if (crdts->data != NULL) { + /* SP token68 */ + http_auth_check_token68(crdts->data); + str_append_c(out, ' '); + str_append(out, crdts->data); + + } else { + /* SP #auth-param */ + str_append_c(out, ' '); + http_auth_create_params(out, &crdts->params); + } +} + +/* + * Manipulation + */ + +static void +http_auth_params_clone(pool_t pool, + ARRAY_TYPE(http_auth_param) *dst, + const ARRAY_TYPE(http_auth_param) *src) +{ + const struct http_auth_param *sparam; + + if (!array_is_created(src)) + return; + + p_array_init(dst, pool, 4); + array_foreach(src, sparam) { + struct http_auth_param nparam; + + i_zero(&nparam); + nparam.name = p_strdup(pool, sparam->name); + nparam.value = p_strdup(pool, sparam->value); + + array_push_back(dst, &nparam); + } +} + +void http_auth_challenge_copy(pool_t pool, + struct http_auth_challenge *dst, + const struct http_auth_challenge *src) +{ + dst->scheme = p_strdup(pool, src->scheme); + if (src->data != NULL) + dst->data = p_strdup(pool, src->data); + else + http_auth_params_clone(pool, &dst->params, &src->params); +} + +struct http_auth_challenge * +http_auth_challenge_clone(pool_t pool, + const struct http_auth_challenge *src) +{ + struct http_auth_challenge *new; + + new = p_new(pool, struct http_auth_challenge, 1); + http_auth_challenge_copy(pool, new, src); + + return new; +} + +void http_auth_credentials_copy(pool_t pool, + struct http_auth_credentials *dst, + const struct http_auth_credentials *src) +{ + dst->scheme = p_strdup(pool, src->scheme); + if (src->data != NULL) + dst->data = p_strdup(pool, src->data); + else + http_auth_params_clone(pool, &dst->params, &src->params); +} + +struct http_auth_credentials * +http_auth_credentials_clone(pool_t pool, + const struct http_auth_credentials *src) +{ + struct http_auth_credentials *new; + + new = p_new(pool, struct http_auth_credentials, 1); + http_auth_credentials_copy(pool, new, src); + + return new; +} + +/* + * Simple schemes + */ + +void http_auth_basic_challenge_init(struct http_auth_challenge *chlng, + const char *realm) +{ + i_zero(chlng); + chlng->scheme = "Basic"; + if (realm != NULL) { + struct http_auth_param param; + + i_zero(¶m); + param.name = "realm"; + param.value = t_strdup(realm); + + t_array_init(&chlng->params, 1); + array_push_back(&chlng->params, ¶m); + } +} + +void http_auth_basic_credentials_init(struct http_auth_credentials *crdts, + const char *username, const char *password) +{ + const char *auth; + string_t *data; + + i_assert(username != NULL && *username != '\0'); + i_assert(strchr(username, ':') == NULL); + + data = t_str_new(64); + auth = t_strconcat(username, ":", password, NULL); + base64_encode(auth, strlen(auth), data); + + i_zero(crdts); + crdts->scheme = "Basic"; + crdts->data = str_c(data); +} diff --git a/src/lib-http/http-auth.h b/src/lib-http/http-auth.h new file mode 100644 index 0000000..061f9ce --- /dev/null +++ b/src/lib-http/http-auth.h @@ -0,0 +1,79 @@ +#ifndef HTTP_AUTH_H +#define HTTP_AUTH_H + +#include "array-decl.h" + +struct http_auth_param; +struct http_auth_challenge; +struct http_auth_credentials; + +ARRAY_DEFINE_TYPE(http_auth_param, struct http_auth_param); +ARRAY_DEFINE_TYPE(http_auth_challenge, struct http_auth_challenge); + +struct http_auth_param { + const char *name; + const char *value; +}; + +struct http_auth_challenge { + const char *scheme; + const char *data; + ARRAY_TYPE(http_auth_param) params; +}; + +struct http_auth_credentials { + const char *scheme; + const char *data; + ARRAY_TYPE(http_auth_param) params; +}; + +/* + * Parsing + */ + +int http_auth_parse_challenges(const unsigned char *data, size_t size, + ARRAY_TYPE(http_auth_challenge) *chlngs); +int http_auth_parse_credentials(const unsigned char *data, size_t size, + struct http_auth_credentials *crdts); + +/* + * Construction + */ + +void http_auth_create_challenge(string_t *out, + const struct http_auth_challenge *chlng); +void http_auth_create_challenges(string_t *out, + const ARRAY_TYPE(http_auth_challenge) *chlngs); + +void http_auth_create_credentials(string_t *out, + const struct http_auth_credentials *crdts); + +/* + * Manipulation + */ + +void http_auth_challenge_copy(pool_t pool, + struct http_auth_challenge *dst, + const struct http_auth_challenge *src); +struct http_auth_challenge * +http_auth_challenge_clone(pool_t pool, + const struct http_auth_challenge *src); + +void http_auth_credentials_copy(pool_t pool, + struct http_auth_credentials *dst, + const struct http_auth_credentials *src); +struct http_auth_credentials * +http_auth_credentials_clone(pool_t pool, + const struct http_auth_credentials *src); + +/* + * Simple schemes + */ + +void http_auth_basic_challenge_init(struct http_auth_challenge *chlng, + const char *realm); +void http_auth_basic_credentials_init(struct http_auth_credentials *crdts, + const char *username, const char *password); + +#endif + diff --git a/src/lib-http/http-client-connection.c b/src/lib-http/http-client-connection.c new file mode 100644 index 0000000..45dadac --- /dev/null +++ b/src/lib-http/http-client-connection.c @@ -0,0 +1,1954 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "net.h" +#include "str.h" +#include "hash.h" +#include "llist.h" +#include "array.h" +#include "ioloop.h" +#include "istream.h" +#include "istream-timeout.h" +#include "ostream.h" +#include "time-util.h" +#include "file-lock.h" +#include "iostream-rawlog.h" +#include "iostream-ssl.h" +#include "http-response-parser.h" + +#include "http-client-private.h" + +/* + * Connection + */ + +static void http_client_connection_ready(struct http_client_connection *conn); +static void http_client_connection_input(struct connection *_conn); +static void +http_client_connection_disconnect(struct http_client_connection *conn); + +static inline const struct http_client_settings * +http_client_connection_get_settings(struct http_client_connection *conn) +{ + if (conn->peer != NULL) + return &conn->peer->client->set; + return &conn->ppool->peer->cctx->set; +} + +static inline void +http_client_connection_ref_request(struct http_client_connection *conn, + struct http_client_request *req) +{ + i_assert(req->conn == NULL); + req->conn = conn; + http_client_request_ref(req); +} + +static inline bool +http_client_connection_unref_request(struct http_client_connection *conn, + struct http_client_request **_req) +{ + struct http_client_request *req = *_req; + + i_assert(req->conn == conn); + req->conn = NULL; + return http_client_request_unref(_req); +} + +static void +http_client_connection_unlist_pending(struct http_client_connection *conn) +{ + struct http_client_peer *peer = conn->peer; + struct http_client_peer_pool *ppool = conn->ppool; + ARRAY_TYPE(http_client_connection) *conn_arr; + struct http_client_connection *const *conn_idx; + + /* Remove from pending lists */ + + conn_arr = &ppool->pending_conns; + array_foreach(conn_arr, conn_idx) { + if (*conn_idx == conn) { + array_delete(conn_arr, + array_foreach_idx(conn_arr, conn_idx), 1); + break; + } + } + + if (peer == NULL) + return; + + conn_arr = &peer->pending_conns; + array_foreach(conn_arr, conn_idx) { + if (*conn_idx == conn) { + array_delete(conn_arr, + array_foreach_idx(conn_arr, conn_idx), 1); + break; + } + } +} + +static inline void +http_client_connection_failure(struct http_client_connection *conn, + const char *reason) +{ + struct http_client_peer *peer = conn->peer; + + conn->connect_failed = TRUE; + http_client_connection_unlist_pending(conn); + http_client_peer_connection_failure(peer, reason); +} + +unsigned int +http_client_connection_count_pending(struct http_client_connection *conn) +{ + unsigned int pending_count = array_count(&conn->request_wait_list); + + if (conn->in_req_callback || conn->pending_request != NULL) + pending_count++; + return pending_count; +} + +bool http_client_connection_is_idle(struct http_client_connection *conn) +{ + return conn->idle; +} + +bool http_client_connection_is_active(struct http_client_connection *conn) +{ + if (!conn->connected) + return FALSE; + + if (conn->in_req_callback || conn->pending_request != NULL) + return TRUE; + + return (array_is_created(&conn->request_wait_list) && + array_count(&conn->request_wait_list) > 0); +} + +static void +http_client_connection_retry_requests(struct http_client_connection *conn, + unsigned int status, const char *error) +{ + struct http_client_request *req, **req_idx; + + if (!array_is_created(&conn->request_wait_list)) + return; + + e_debug(conn->event, "Retrying pending requests"); + + array_foreach_modifiable(&conn->request_wait_list, req_idx) { + req = *req_idx; + /* Drop reference from connection */ + if (!http_client_connection_unref_request(conn, req_idx)) + continue; + /* Retry the request, which may drop it */ + if (req->state < HTTP_REQUEST_STATE_FINISHED) + http_client_request_retry(req, status, error); + } + array_clear(&conn->request_wait_list); +} + +static void +http_client_connection_server_close(struct http_client_connection **_conn) +{ + struct http_client_connection *conn = *_conn; + struct http_client_peer *peer = conn->peer; + struct http_client_request *req, **req_idx; + + e_debug(conn->event, "Server explicitly closed connection"); + + array_foreach_modifiable(&conn->request_wait_list, req_idx) { + req = *req_idx; + /* Drop reference from connection */ + if (!http_client_connection_unref_request(conn, req_idx)) + continue; + /* Resubmit the request, which may drop it */ + if (req->state < HTTP_REQUEST_STATE_FINISHED) + http_client_request_resubmit(req); + } + array_clear(&conn->request_wait_list); + + if (peer != NULL) { + struct http_client *client = peer->client; + + if (client->waiting) + io_loop_stop(client->ioloop); + } + + http_client_connection_close(_conn); +} + +static void +http_client_connection_abort_error(struct http_client_connection **_conn, + unsigned int status, const char *error) +{ + struct http_client_connection *conn = *_conn; + struct http_client_request *req, **req_idx; + + e_debug(conn->event, "Aborting connection: %s", error); + + array_foreach_modifiable(&conn->request_wait_list, req_idx) { + req = *req_idx; + i_assert(req->submitted); + /* Drop reference from connection */ + if (!http_client_connection_unref_request(conn, req_idx)) + continue; + /* Drop request if not already aborted */ + http_client_request_error(&req, status, error); + } + array_clear(&conn->request_wait_list); + http_client_connection_close(_conn); +} + +static void +http_client_connection_abort_any_requests(struct http_client_connection *conn) +{ + struct http_client_request *req, **req_idx; + + if (array_is_created(&conn->request_wait_list)) { + array_foreach_modifiable(&conn->request_wait_list, req_idx) { + req = *req_idx; + i_assert(req->submitted); + /* Drop reference from connection */ + if (!http_client_connection_unref_request(conn, req_idx)) + continue; + /* Drop request if not already aborted */ + http_client_request_error( + &req, HTTP_CLIENT_REQUEST_ERROR_ABORTED, + "Aborting"); + } + array_clear(&conn->request_wait_list); + } + if (conn->pending_request != NULL) { + req = conn->pending_request; + /* Drop reference from connection */ + if (http_client_connection_unref_request( + conn, &conn->pending_request)) { + /* Drop request if not already aborted */ + http_client_request_error( + &req, HTTP_CLIENT_REQUEST_ERROR_ABORTED, + "Aborting"); + } + } +} + +static const char * +http_client_connection_get_timing_info(struct http_client_connection *conn) +{ + struct http_client_request *const *requestp; + unsigned int connected_msecs; + string_t *str = t_str_new(64); + + if (array_count(&conn->request_wait_list) > 0) { + requestp = array_front(&conn->request_wait_list); + + str_append(str, "Request "); + http_client_request_append_stats_text(*requestp, str); + } else { + str_append(str, "No requests"); + if (conn->conn.last_input != 0) { + str_printfa(str, ", last input %d secs ago", + (int)(ioloop_time - conn->conn.last_input)); + } + } + connected_msecs = timeval_diff_msecs(&ioloop_timeval, + &conn->connected_timestamp); + str_printfa(str, ", connected %u.%03u secs ago", + connected_msecs/1000, connected_msecs%1000); + return str_c(str); +} + +static void +http_client_connection_abort_temp_error(struct http_client_connection **_conn, + unsigned int status, const char *error) +{ + struct http_client_connection *conn = *_conn; + + error = t_strdup_printf("%s (%s)", error, + http_client_connection_get_timing_info(conn)); + + e_debug(conn->event, + "Aborting connection with temporary error: %s", error); + + http_client_connection_disconnect(conn); + http_client_connection_retry_requests(conn, status, error); + http_client_connection_close(_conn); +} + +void http_client_connection_lost(struct http_client_connection **_conn, + const char *error) +{ + struct http_client_connection *conn = *_conn; + const char *sslerr; + + if (error == NULL) + error = "Connection lost"; + else + error = t_strdup_printf("Connection lost: %s", error); + + if (conn->ssl_iostream != NULL) { + sslerr = ssl_iostream_get_last_error(conn->ssl_iostream); + if (sslerr != NULL) { + error = t_strdup_printf("%s (last SSL error: %s)", + error, sslerr); + } + if (ssl_iostream_has_handshake_failed(conn->ssl_iostream)) { + /* This isn't really a "connection lost", but that we + don't trust the remote's SSL certificate. don't + retry. */ + http_client_connection_abort_error( + _conn, + HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE, error); + return; + } + } + + conn->lost_prematurely = + (conn->conn.input != NULL && + conn->conn.input->v_offset == 0 && + i_stream_get_data_size(conn->conn.input) == 0); + http_client_connection_abort_temp_error( + _conn, HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, error); +} + +void http_client_connection_handle_output_error( + struct http_client_connection *conn) +{ + struct ostream *output = conn->conn.output; + + if (output->stream_errno != EPIPE && + output->stream_errno != ECONNRESET) { + http_client_connection_lost( + &conn, + t_strdup_printf("write(%s) failed: %s", + o_stream_get_name(output), + o_stream_get_error(output))); + } else { + http_client_connection_lost(&conn, "Remote disconnected"); + } +} + +int http_client_connection_check_ready(struct http_client_connection *conn) +{ + const struct http_client_settings *set = + http_client_connection_get_settings(conn); + + if (conn->in_req_callback) { + /* This can happen when a nested ioloop is created inside + request callback. we currently don't reuse connections that + are occupied this way, but theoretically we could, although + that would add quite a bit of complexity. + */ + return 0; + } + + if (!conn->connected || conn->output_locked || conn->output_broken || + conn->close_indicated || conn->tunneling || + (http_client_connection_count_pending(conn) >= + set->max_pipelined_requests)) + return 0; + + if (conn->last_ioloop != NULL && conn->last_ioloop != current_ioloop) { + conn->last_ioloop = current_ioloop; + /* Active ioloop is different from what we saw earlier; we may + have missed a disconnection event on this connection. Verify + status by reading from connection. */ + if (i_stream_read(conn->conn.input) == -1) { + int stream_errno = conn->conn.input->stream_errno; + + i_assert(conn->conn.input->stream_errno != 0 || + conn->conn.input->eof); + http_client_connection_lost( + &conn, + t_strdup_printf( + "read(%s) failed: %s", + i_stream_get_name(conn->conn.input), + (stream_errno != 0 ? + i_stream_get_error(conn->conn.input) : + "EOF"))); + return -1; + } + + /* We may have read some data */ + if (i_stream_get_data_size(conn->conn.input) > 0) + i_stream_set_input_pending(conn->conn.input, TRUE); + } + return 1; +} + +static void +http_client_connection_detach_peer(struct http_client_connection *conn) +{ + struct http_client_peer *peer = conn->peer; + struct http_client_connection *const *conn_idx; + ARRAY_TYPE(http_client_connection) *conn_arr; + bool found = FALSE; + + if (peer == NULL) + return; + + http_client_peer_ref(peer); + conn_arr = &peer->conns; + array_foreach(conn_arr, conn_idx) { + if (*conn_idx == conn) { + array_delete(conn_arr, + array_foreach_idx(conn_arr, conn_idx), 1); + found = TRUE; + break; + } + } + i_assert(found); + + conn_arr = &peer->pending_conns; + array_foreach(conn_arr, conn_idx) { + if (*conn_idx == conn) { + array_delete(conn_arr, + array_foreach_idx(conn_arr, conn_idx), 1); + break; + } + } + + conn->peer = NULL; + e_debug(conn->event, "Detached peer"); + + if (conn->connect_succeeded) + http_client_peer_connection_lost(peer, conn->lost_prematurely); + http_client_peer_unref(&peer); +} + +static void +http_client_connection_idle_timeout(struct http_client_connection *conn) +{ + e_debug(conn->event, "Idle connection timed out"); + + /* Cannot get here unless connection was established at some point */ + i_assert(conn->connect_succeeded); + + http_client_connection_close(&conn); +} + +static unsigned int +http_client_connection_start_idle_timeout(struct http_client_connection *conn) +{ + const struct http_client_settings *set = + http_client_connection_get_settings(conn); + struct http_client_peer_pool *ppool = conn->ppool; + struct http_client_peer_shared *pshared = ppool->peer; + unsigned int timeout, count, idle_count, max; + + i_assert(conn->to_idle == NULL); + + if (set->max_idle_time_msecs == 0) + return UINT_MAX; + + count = array_count(&ppool->conns); + idle_count = array_count(&ppool->idle_conns); + max = http_client_peer_shared_max_connections(pshared); + i_assert(count > 0); + i_assert(count >= idle_count + 1); + i_assert(max > 0); + + /* Set timeout for this connection */ + if (idle_count == 0 || max == UINT_MAX) { + /* No idle connections yet or infinite connections allowed; + use the maximum idle time. */ + timeout = set->max_idle_time_msecs; + } else if (count > max || idle_count >= max) { + /* Instant death for (urgent) connections above limit */ + timeout = 0; + } else { + unsigned int idle_slots_avail; + double idle_time_per_slot; + + /* Kill duplicate connections quicker; + linearly based on the number of connections */ + idle_slots_avail = max - idle_count; + idle_time_per_slot = (double)set->max_idle_time_msecs / max; + timeout = (unsigned int)(idle_time_per_slot * idle_slots_avail); + if (timeout < HTTP_CLIENT_MIN_IDLE_TIMEOUT_MSECS) + timeout = HTTP_CLIENT_MIN_IDLE_TIMEOUT_MSECS; + } + + conn->to_idle = timeout_add_short_to( + conn->conn.ioloop, timeout, + http_client_connection_idle_timeout, conn); + return timeout; +} + +static void +http_client_connection_start_idle(struct http_client_connection *conn, + const char *reason) +{ + struct http_client_peer_pool *ppool = conn->ppool; + unsigned int timeout; + + if (conn->idle) { + e_debug(conn->event, "%s; already idle", reason); + return; + } + + timeout = http_client_connection_start_idle_timeout(conn); + if (timeout == UINT_MAX) + e_debug(conn->event, "%s; going idle", reason); + else { + e_debug(conn->event, "%s; going idle (timeout = %u msecs)", + reason, timeout); + } + + conn->idle = TRUE; + array_push_back(&ppool->idle_conns, &conn); +} + +void http_client_connection_lost_peer(struct http_client_connection *conn) +{ + if (!conn->connected) { + http_client_connection_unref(&conn); + return; + } + + i_assert(!conn->in_req_callback); + + http_client_connection_start_idle(conn, "Lost peer"); + http_client_connection_detach_peer(conn); +} + +void http_client_connection_check_idle(struct http_client_connection *conn) +{ + struct http_client_peer *peer; + + peer = conn->peer; + if (peer == NULL) { + i_assert(conn->idle); + return; + } + + if (conn->idle) { + /* Already idle */ + return; + } + + if (conn->connected && !http_client_connection_is_active(conn)) { + struct http_client *client = peer->client; + + i_assert(conn->to_requests == NULL); + + if (client->waiting) + io_loop_stop(client->ioloop); + + http_client_connection_start_idle( + conn, "No more requests queued"); + } +} + +static void +http_client_connection_stop_idle(struct http_client_connection *conn) +{ + struct http_client_connection *const *conn_idx; + ARRAY_TYPE(http_client_connection) *conn_arr; + + timeout_remove(&conn->to_idle); + conn->idle = FALSE; + + conn_arr = &conn->ppool->idle_conns; + array_foreach(conn_arr, conn_idx) { + if (*conn_idx == conn) { + array_delete(conn_arr, + array_foreach_idx(conn_arr, conn_idx), 1); + break; + } + } +} + +void http_client_connection_claim_idle(struct http_client_connection *conn, + struct http_client_peer *peer) +{ + e_debug(conn->event, "Claimed as idle"); + + i_assert(peer->ppool == conn->ppool); + http_client_connection_stop_idle(conn); + + if (conn->peer == NULL || conn->peer != peer) { + http_client_connection_detach_peer(conn); + + conn->peer = peer; + conn->debug = peer->client->set.debug; + array_push_back(&peer->conns, &conn); + } +} + +static void +http_client_connection_request_timeout(struct http_client_connection *conn) +{ + conn->conn.input->stream_errno = ETIMEDOUT; + http_client_connection_abort_temp_error( + &conn, HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT, + "Request timed out"); +} + +void http_client_connection_start_request_timeout( + struct http_client_connection *conn) +{ + struct http_client_request *const *requestp; + unsigned int timeout_msecs; + + if (conn->pending_request != NULL) + return; + + i_assert(array_is_created(&conn->request_wait_list)); + i_assert(array_count(&conn->request_wait_list) > 0); + requestp = array_front(&conn->request_wait_list); + timeout_msecs = (*requestp)->attempt_timeout_msecs; + + if (timeout_msecs == 0) + ; + else if (conn->to_requests != NULL) + timeout_reset(conn->to_requests); + else { + conn->to_requests = timeout_add_to( + conn->conn.ioloop, timeout_msecs, + http_client_connection_request_timeout, conn); + } +} + +void http_client_connection_reset_request_timeout( + struct http_client_connection *conn) +{ + if (conn->to_requests != NULL) + timeout_reset(conn->to_requests); +} + +void http_client_connection_stop_request_timeout( + struct http_client_connection *conn) +{ + timeout_remove(&conn->to_requests); +} + +static void +http_client_connection_continue_timeout(struct http_client_connection *conn) +{ + struct http_client_peer_shared *pshared = conn->ppool->peer; + struct http_client_request *const *wait_reqs; + struct http_client_request *req; + unsigned int wait_count; + + i_assert(conn->pending_request == NULL); + + timeout_remove(&conn->to_response); + pshared->no_payload_sync = TRUE; + + e_debug(conn->event, + "Expected 100-continue response timed out; " + "sending payload anyway"); + + wait_reqs = array_get(&conn->request_wait_list, &wait_count); + i_assert(wait_count == 1); + req = wait_reqs[wait_count-1]; + + req->payload_sync_continue = TRUE; + if (conn->conn.output != NULL) + o_stream_set_flush_pending(conn->conn.output, TRUE); +} + +int http_client_connection_next_request(struct http_client_connection *conn) +{ + struct http_client_connection *tmp_conn; + struct http_client_peer *peer = conn->peer; + struct http_client_peer_shared *pshared = conn->ppool->peer; + struct http_client_request *req = NULL; + bool pipelined; + int ret; + + if ((ret = http_client_connection_check_ready(conn)) <= 0) { + if (ret == 0) + e_debug(conn->event, "Not ready for next request"); + return ret; + } + + /* Claim request, but no urgent request can be second in line */ + pipelined = (array_count(&conn->request_wait_list) > 0 || + conn->pending_request != NULL); + req = http_client_peer_claim_request(peer, pipelined); + if (req == NULL) + return 0; + + i_assert(req->state == HTTP_REQUEST_STATE_QUEUED); + + http_client_connection_stop_idle(conn); + + req->payload_sync_continue = FALSE; + if (pshared->no_payload_sync) + req->payload_sync = FALSE; + + /* Add request to wait list and add a reference */ + array_push_back(&conn->request_wait_list, &req); + http_client_connection_ref_request(conn, req); + + e_debug(conn->event, "Claimed request %s", + http_client_request_label(req)); + + tmp_conn = conn; + http_client_connection_ref(tmp_conn); + ret = http_client_request_send(req, pipelined); + if (ret == 0 && conn->conn.output != NULL) + o_stream_set_flush_pending(conn->conn.output, TRUE); + if (!http_client_connection_unref(&tmp_conn) || ret < 0) + return -1; + + if (req->connect_tunnel) + conn->tunneling = TRUE; + + /* RFC 7231, Section 5.1.1: Expect + + o A client that sends a 100-continue expectation is not required to + wait for any specific length of time; such a client MAY proceed + to send the message body even if it has not yet received a + response. Furthermore, since 100 (Continue) responses cannot be + sent through an HTTP/1.0 intermediary, such a client SHOULD NOT + wait for an indefinite period before sending the message body. + */ + if (req->payload_sync && !pshared->seen_100_response) { + i_assert(!pipelined); + i_assert(req->payload_chunked || req->payload_size > 0); + i_assert(conn->to_response == NULL); + conn->to_response = timeout_add_to( + conn->conn.ioloop, HTTP_CLIENT_CONTINUE_TIMEOUT_MSECS, + http_client_connection_continue_timeout, conn); + } + + return 1; +} + +static void http_client_connection_destroy(struct connection *_conn) +{ + struct http_client_connection *conn = + (struct http_client_connection *)_conn; + const char *error; + unsigned int msecs; + + switch (_conn->disconnect_reason) { + case CONNECTION_DISCONNECT_CONNECT_TIMEOUT: + if (conn->connected_timestamp.tv_sec == 0) { + msecs = timeval_diff_msecs( + &ioloop_timeval, + &conn->connect_start_timestamp); + error = t_strdup_printf( + "connect(%s) failed: " + "Connection timed out in %u.%03u secs", + _conn->name, msecs/1000, msecs%1000); + } else { + msecs = timeval_diff_msecs(&ioloop_timeval, + &conn->connected_timestamp); + error = t_strdup_printf( + "SSL handshaking with %s failed: " + "Connection timed out in %u.%03u secs", + _conn->name, msecs/1000, msecs%1000); + } + e_debug(conn->event, "%s", error); + http_client_connection_failure(conn, error); + break; + case CONNECTION_DISCONNECT_CONN_CLOSED: + if (conn->connect_failed) { + i_assert(!array_is_created(&conn->request_wait_list) || + array_count(&conn->request_wait_list) == 0); + break; + } + http_client_connection_lost( + &conn, (_conn->input == NULL ? + NULL : i_stream_get_error(_conn->input))); + return; + default: + break; + } + + http_client_connection_close(&conn); +} + +static void http_client_payload_finished(struct http_client_connection *conn) +{ + timeout_remove(&conn->to_input); + connection_input_resume(&conn->conn); + if (array_count(&conn->request_wait_list) > 0) + http_client_connection_start_request_timeout(conn); + else + http_client_connection_stop_request_timeout(conn); +} + +static void +http_client_payload_destroyed_timeout(struct http_client_connection *conn) +{ + if (conn->close_indicated) { + http_client_connection_server_close(&conn); + return; + } + http_client_connection_input(&conn->conn); +} + +static void http_client_payload_destroyed(struct http_client_request *req) +{ + struct http_client_connection *conn = req->conn; + + i_assert(conn != NULL); + i_assert(conn->pending_request == req); + i_assert(conn->incoming_payload != NULL); + i_assert(conn->conn.io == NULL); + + e_debug(conn->event, + "Response payload stream destroyed " + "(%u ms after initial response)", + timeval_diff_msecs(&ioloop_timeval, &req->response_time)); + + /* Caller is allowed to change the socket fd to blocking while reading + the payload. make sure here that it's switched back. */ + net_set_nonblock(conn->conn.fd_in, TRUE); + + i_assert(req->response_offset < conn->conn.input->v_offset); + req->bytes_in = conn->conn.input->v_offset - req->response_offset; + + /* Drop reference from connection */ + if (http_client_connection_unref_request( + conn, &conn->pending_request)) { + /* Finish request if not already aborted */ + http_client_request_finish(req); + } + + conn->incoming_payload = NULL; + + /* Input stream may have pending input. make sure input handler + gets called (but don't do it directly, since we get get here + somewhere from the API user's code, which we can't really know what + state it is in). this call also triggers sending a new request if + necessary. */ + if (!conn->disconnected) { + conn->to_input = timeout_add_short_to( + conn->conn.ioloop, 0, + http_client_payload_destroyed_timeout, conn); + } + + /* Room for new requests */ + if (http_client_connection_check_ready(conn) > 0) + http_client_peer_trigger_request_handler(conn->peer); +} + +void http_client_connection_request_destroyed( + struct http_client_connection *conn, struct http_client_request *req) +{ + struct istream *payload; + + i_assert(req->conn == conn); + if (conn->pending_request != req) + return; + + e_debug(conn->event, "Pending request destroyed prematurely"); + + payload = conn->incoming_payload; + if (payload == NULL) { + /* Payload already gone */ + return; + } + + /* Destroy the payload, so that the timeout istream is closed */ + i_stream_ref(payload); + i_stream_destroy(&payload); + + payload = conn->incoming_payload; + if (payload == NULL) { + /* Not going to happen, but check for it anyway */ + return; + } + + /* The application still holds a reference to the payload stream, but it + is closed and we don't care about it anymore, so act as though it is + destroyed. */ + i_stream_remove_destroy_callback(payload, + http_client_payload_destroyed); + http_client_payload_destroyed(req); +} + +static bool +http_client_connection_return_response(struct http_client_connection *conn, + struct http_client_request *req, + struct http_response *response) +{ + struct http_client_peer_shared *pshared = conn->ppool->peer; + struct istream *payload; + bool retrying; + + i_assert(!conn->in_req_callback); + i_assert(conn->incoming_payload == NULL); + i_assert(conn->pending_request == NULL); + + http_client_connection_ref(conn); + http_client_connection_ref_request(conn, req); + req->state = HTTP_REQUEST_STATE_GOT_RESPONSE; + + if (response->payload != NULL) { + /* Wrap the stream to capture the destroy event without + destroying the actual payload stream. we are already expected + to be on the correct ioloop, so there should be no need to + switch the stream's ioloop here. */ + conn->incoming_payload = response->payload = + i_stream_create_timeout(response->payload, + req->attempt_timeout_msecs); + i_stream_add_destroy_callback(response->payload, + http_client_payload_destroyed, + req); + /* The callback may add its own I/O, so we need to remove + our one before calling it */ + connection_input_halt(&conn->conn); + /* We've received the request itself, and we can't reset the + timeout during the payload reading. */ + http_client_connection_stop_request_timeout(conn); + } + + conn->in_req_callback = TRUE; + retrying = !http_client_request_callback(req, response); + if (conn->disconnected) { + /* The callback managed to get this connection disconnected */ + if (!retrying) + http_client_request_finish(req); + http_client_connection_unref_request(conn, &req); + http_client_connection_unref(&conn); + return FALSE; + } + conn->in_req_callback = FALSE; + + if (retrying) { + /* Retrying, don't destroy the request */ + if (response->payload != NULL) { + i_stream_remove_destroy_callback( + conn->incoming_payload, + http_client_payload_destroyed); + i_stream_unref(&conn->incoming_payload); + connection_input_resume(&conn->conn); + } + http_client_connection_unref_request(conn, &req); + return http_client_connection_unref(&conn); + } + + if (response->payload != NULL) { + req->state = HTTP_REQUEST_STATE_PAYLOAD_IN; + payload = response->payload; + response->payload = NULL; + + /* Maintain request reference while payload is pending */ + conn->pending_request = req; + + /* Request is dereferenced in payload destroy callback */ + i_stream_unref(&payload); + + if (conn->to_input != NULL && conn->conn.input != NULL) { + /* Already finished reading the payload */ + http_client_payload_finished(conn); + } + } else { + http_client_request_finish(req); + http_client_connection_unref_request(conn, &req); + } + + if (conn->incoming_payload == NULL && conn->conn.input != NULL) { + i_assert(conn->conn.io != NULL || + pshared->addr.type == HTTP_CLIENT_PEER_ADDR_RAW); + return http_client_connection_unref(&conn); + } + http_client_connection_unref(&conn); + return FALSE; +} + +static const char * +http_client_request_add_event_headers(struct http_client_request *req, + const struct http_response *response) +{ + if (req->event_headers == NULL) + return ""; + + string_t *str = t_str_new(128); + for (unsigned int i = 0; req->event_headers[i] != NULL; i++) { + const char *hdr_name = req->event_headers[i]; + const char *value = + http_response_header_get(response, hdr_name); + + if (value == NULL) + continue; + + str_append(str, str_len(str) == 0 ? " (" : ", "); + event_add_str(req->event, + t_strconcat("http_hdr_", hdr_name, NULL), value); + str_printfa(str, "%s:%s", hdr_name, value); + } + if (str_len(str) > 0) + str_append_c(str, ')'); + return str_c(str); +} + +static void http_client_connection_input(struct connection *_conn) +{ + struct http_client_connection *conn = + (struct http_client_connection *)_conn; + struct http_client_peer *peer = conn->peer; + struct http_client_peer_shared *pshared = conn->ppool->peer; + struct http_response response; + struct http_client_request *const *reqs; + struct http_client_request *req = NULL, *req_ref; + enum http_response_payload_type payload_type; + unsigned int count; + int finished = 0, ret; + const char *error; + + i_assert(conn->incoming_payload == NULL); + + _conn->last_input = ioloop_time; + + if (conn->ssl_iostream != NULL && + !ssl_iostream_is_handshaked(conn->ssl_iostream)) { + /* Finish SSL negotiation by reading from input stream */ + while ((ret = i_stream_read(conn->conn.input)) > 0 || + ret == -2) { + if (ssl_iostream_is_handshaked(conn->ssl_iostream)) + break; + } + if (ret < 0) { + int stream_errno = conn->conn.input->stream_errno; + + /* Failed somehow */ + i_assert(ret != -2); + error = t_strdup_printf( + "SSL handshaking with %s failed: " + "read(%s) failed: %s", + _conn->name, + i_stream_get_name(conn->conn.input), + (stream_errno != 0 ? + i_stream_get_error(conn->conn.input) : "EOF")); + http_client_connection_failure(conn, error); + e_debug(conn->event, "%s", error); + http_client_connection_close(&conn); + return; + } + + if (!ssl_iostream_is_handshaked(conn->ssl_iostream)) { + /* Not finished */ + i_assert(ret == 0); + return; + } + } + + if (!conn->connect_succeeded) { + /* Just got ready for first request */ + http_client_connection_ready(conn); + } + + if (conn->to_input != NULL) { + /* We came here from a timeout added by + http_client_payload_destroyed(). The IO couldn't be added + back immediately in there, because the HTTP API user may + still have had its own IO pointed to the same fd. It should + be removed by now, so we can add it back. */ + http_client_payload_finished(conn); + finished++; + } + + /* We've seen activity from the server; reset request timeout */ + http_client_connection_reset_request_timeout(conn); + + /* Get first waiting request */ + reqs = array_get(&conn->request_wait_list, &count); + if (count > 0) { + req = reqs[0]; + + /* Determine whether to expect a response payload */ + payload_type = http_client_request_get_payload_type(req); + } else { + req = NULL; + payload_type = HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED; + i_assert(conn->to_requests == NULL); + } + + /* Drop connection with broken output if last possible input was + received */ + if (conn->output_broken && (count == 0 || + (count == 1 && req->state == HTTP_REQUEST_STATE_ABORTED))) { + http_client_connection_server_close(&conn); + return; + } + + while ((ret = http_response_parse_next(conn->http_parser, payload_type, + &response, &error)) > 0) { + bool aborted, early = FALSE; + + if (req == NULL) { + /* Server sent response without any requests in the wait + list */ + if (response.status == 408) { + e_debug(conn->event, + "Server explicitly closed connection: " + "408 %s", response.reason); + } else { + e_debug(conn->event, + "Got unexpected input from server: " + "%u %s", response.status, + response.reason); + } + http_client_connection_close(&conn); + return; + } + + req->response_time = ioloop_timeval; + req->response_offset = + http_response_parser_get_last_offset(conn->http_parser); + i_assert(req->response_offset != UOFF_T_MAX); + i_assert(req->response_offset < conn->conn.input->v_offset); + req->bytes_in = conn->conn.input->v_offset - + req->response_offset; + + /* Got some response; cancel response timeout */ + timeout_remove(&conn->to_response); + + /* RFC 7231, Section 6.2: + + A client MUST be able to parse one or more 1xx responses + received prior to a final response, even if the client does + not expect one. A user agent MAY ignore unexpected 1xx + responses. + */ + if (req->payload_sync && response.status == 100) { + if (req->payload_sync_continue) { + e_debug(conn->event, + "Got 100-continue response after timeout"); + continue; + } + + pshared->no_payload_sync = FALSE; + pshared->seen_100_response = TRUE; + req->payload_sync_continue = TRUE; + + e_debug(conn->event, + "Got expected 100-continue response"); + + if (req->state == HTTP_REQUEST_STATE_ABORTED) { + e_debug(conn->event, + "Request aborted before sending payload was complete."); + http_client_connection_close(&conn); + return; + } + + if (conn->conn.output != NULL) + o_stream_set_flush_pending(conn->conn.output, TRUE); + return; + } else if (response.status / 100 == 1) { + /* Ignore other 1xx for now */ + e_debug(conn->event, + "Got unexpected %u response; ignoring", + response.status); + continue; + } else if (!req->payload_sync && !req->payload_finished && + req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT) { + /* Got early response from server while we're still + sending request payload. we cannot recover from this + reliably, so we stop sending payload and close the + connection once the response is processed */ + e_debug(conn->event, + "Got early input from server; " + "request payload not completely sent " + "(will close connection)"); + o_stream_unset_flush_callback(conn->conn.output); + conn->output_broken = early = TRUE; + } + + const char *suffix = + http_client_request_add_event_headers(req, &response); + e_debug(conn->event, + "Got %u response for request %s: %s%s " + "(took %u ms + %u ms in queue)", + response.status, http_client_request_label(req), + response.reason, suffix, + timeval_diff_msecs(&req->response_time, &req->sent_time), + timeval_diff_msecs(&req->sent_time, &req->submit_time)); + + /* Make sure connection output is unlocked if 100-continue + failed */ + if (req->payload_sync && !req->payload_sync_continue) { + e_debug(conn->event, "Unlocked output"); + conn->output_locked = FALSE; + } + + /* Remove request from queue */ + array_pop_front(&conn->request_wait_list); + aborted = (req->state == HTTP_REQUEST_STATE_ABORTED); + req_ref = req; + if (!http_client_connection_unref_request(conn, &req_ref)) { + i_assert(aborted); + req = NULL; + } + + conn->close_indicated = response.connection_close; + + if (!aborted) { + bool handled = FALSE; + + /* Response cannot be 2xx if request payload was not + completely sent */ + if (early && response.status / 100 == 2) { + http_client_request_error( + &req, HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE, + "Server responded with success response " + "before all payload was sent"); + http_client_connection_close(&conn); + return; + } + + /* Don't redirect/retry if we're sending data in small + blocks via http_client_request_send_payload() + and we're not waiting for 100 continue */ + if (!req->payload_wait || + (req->payload_sync && !req->payload_sync_continue)) { + /* Failed Expect: */ + if (response.status == 417 && req->payload_sync) { + /* Drop Expect: continue */ + req->payload_sync = FALSE; + conn->output_locked = FALSE; + pshared->no_payload_sync = TRUE; + if (http_client_request_try_retry(req)) + handled = TRUE; + /* Redirection */ + } else if (!req->client->set.no_auto_redirect && + response.status / 100 == 3 && + response.status != 304 && + response.location != NULL) { + /* Redirect (possibly after delay) */ + if (http_client_request_delay_from_response( + req, &response) >= 0) { + http_client_request_redirect( + req, response.status, + response.location); + handled = TRUE; + } + /* Service unavailable */ + } else if (response.status == 503) { + /* Automatically retry after delay if + indicated */ + if (response.retry_after != (time_t)-1 && + http_client_request_delay_from_response( + req, &response) > 0 && + http_client_request_try_retry(req)) + handled = TRUE; + /* Request timeout (by server) */ + } else if (response.status == 408) { + /* Automatically retry */ + if (http_client_request_try_retry(req)) + handled = TRUE; + /* Connection close is implicit, + although server should indicate that + explicitly */ + conn->close_indicated = TRUE; + } + } + + if (!handled) { + /* Response for application */ + if (!http_client_connection_return_response( + conn, req, &response)) + return; + } + } + + finished++; + + /* Server closing connection? */ + if (conn->close_indicated) { + http_client_connection_server_close(&conn); + return; + } + + /* Get next waiting request */ + reqs = array_get(&conn->request_wait_list, &count); + if (count > 0) { + req = reqs[0]; + + /* Determine whether to expect a response payload */ + payload_type = http_client_request_get_payload_type(req); + } else { + /* No more requests waiting for the connection */ + req = NULL; + payload_type = HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED; + http_client_connection_stop_request_timeout(conn); + } + + /* Drop connection with broken output if last possible input was + received */ + if (conn->output_broken && (count == 0 || + (count == 1 && req->state == HTTP_REQUEST_STATE_ABORTED))) { + http_client_connection_server_close(&conn); + return; + } + } + + if (ret <= 0 && + (conn->conn.input->eof || conn->conn.input->stream_errno != 0)) { + int stream_errno = conn->conn.input->stream_errno; + + http_client_connection_lost( + &conn, + t_strdup_printf("read(%s) failed: %s", + i_stream_get_name(conn->conn.input), + (stream_errno != 0 ? + i_stream_get_error(conn->conn.input) : + "EOF"))); + return; + } + + if (ret < 0) { + http_client_connection_abort_error( + &conn, HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE, error); + return; + } + + if (finished > 0) { + /* Connection still alive after (at least one) request; + we can pipeline -> mark for subsequent connections */ + pshared->allows_pipelining = TRUE; + + /* Room for new requests */ + if (peer != NULL && + http_client_connection_check_ready(conn) > 0) + http_client_peer_trigger_request_handler(peer); + } +} + +static int +http_client_connection_continue_request(struct http_client_connection *conn) +{ + struct http_client_connection *tmp_conn; + struct http_client_request *const *reqs; + unsigned int count; + struct http_client_request *req; + bool pipelined; + int ret; + + reqs = array_get(&conn->request_wait_list, &count); + i_assert(count > 0 || conn->to_requests == NULL); + if (count == 0 || !conn->output_locked) + return 1; + + req = reqs[count-1]; + pipelined = (count > 1 || conn->pending_request != NULL); + + if (req->state == HTTP_REQUEST_STATE_ABORTED) { + e_debug(conn->event, + "Request aborted before sending payload was complete."); + if (count == 1) { + http_client_connection_close(&conn); + return -1; + } + o_stream_unset_flush_callback(conn->conn.output); + conn->output_broken = TRUE; + return -1; + } + + if (req->payload_sync && !req->payload_sync_continue) + return 1; + + tmp_conn = conn; + http_client_connection_ref(tmp_conn); + ret = http_client_request_send_more(req, pipelined); + if (!http_client_connection_unref(&tmp_conn) || ret < 0) + return -1; + + if (!conn->output_locked) { + /* Room for new requests */ + if (http_client_connection_check_ready(conn) > 0) + http_client_peer_trigger_request_handler(conn->peer); + } + return ret; +} + +int http_client_connection_output(struct http_client_connection *conn) +{ + struct ostream *output = conn->conn.output; + int ret; + + /* We've seen activity from the server; reset request timeout */ + http_client_connection_reset_request_timeout(conn); + + if ((ret = o_stream_flush(output)) <= 0) { + if (ret < 0) + http_client_connection_handle_output_error(conn); + return ret; + } + + i_assert(!conn->output_broken); + + if (conn->ssl_iostream != NULL && + !ssl_iostream_is_handshaked(conn->ssl_iostream)) + return 1; + + return http_client_connection_continue_request(conn); +} + +void http_client_connection_start_tunnel(struct http_client_connection **_conn, + struct http_client_tunnel *tunnel) +{ + struct http_client_connection *conn = *_conn; + + i_assert(conn->tunneling); + + /* Claim connection streams */ + i_zero(tunnel); + tunnel->input = conn->conn.input; + tunnel->output = conn->conn.output; + tunnel->fd_in = conn->conn.fd_in; + tunnel->fd_out = conn->conn.fd_out; + + /* Detach from connection */ + conn->conn.input = NULL; + conn->conn.output = NULL; + conn->conn.fd_in = -1; + conn->conn.fd_out = -1; + conn->closing = TRUE; + conn->connected = FALSE; + connection_disconnect(&conn->conn); + + http_client_connection_unref(_conn); +} + +static void http_client_connection_ready(struct http_client_connection *conn) +{ + struct http_client_peer *peer = conn->peer; + struct http_client_peer_pool *ppool = conn->ppool; + struct http_client_peer_shared *pshared = ppool->peer; + const struct http_client_settings *set = + http_client_connection_get_settings(conn); + + e_debug(conn->event, "Ready for requests"); + i_assert(!conn->connect_succeeded); + + /* Connected */ + conn->connected = TRUE; + conn->last_ioloop = current_ioloop; + timeout_remove(&conn->to_connect); + + /* Indicate connection success */ + conn->connect_succeeded = TRUE; + http_client_connection_unlist_pending(conn); + http_client_peer_connection_success(peer); + + /* Start raw log */ + if (ppool->rawlog_dir != NULL) { + iostream_rawlog_create(ppool->rawlog_dir, + &conn->conn.input, &conn->conn.output); + } + + /* Direct tunneling connections handle connect requests just by + providing a raw connection */ + if (pshared->addr.type == HTTP_CLIENT_PEER_ADDR_RAW) { + struct http_client_request *req; + + req = http_client_peer_claim_request(conn->peer, FALSE); + if (req != NULL) { + struct http_response response; + + conn->tunneling = TRUE; + + i_zero(&response); + response.status = 200; + response.reason = "OK"; + + (void)http_client_connection_return_response(conn, req, + &response); + return; + } + + e_debug(conn->event, + "No raw connect requests pending; " + "closing useless connection"); + http_client_connection_close(&conn); + return; + } + + /* Start protocol I/O */ + conn->http_parser = http_response_parser_init( + conn->conn.input, &set->response_hdr_limits, 0); + o_stream_set_finish_via_child(conn->conn.output, FALSE); + o_stream_set_flush_callback(conn->conn.output, + http_client_connection_output, conn); +} + +static int +http_client_connection_ssl_handshaked(const char **error_r, void *context) +{ + struct http_client_connection *conn = context; + struct http_client_peer_shared *pshared = conn->ppool->peer; + const struct http_client_settings *set = + http_client_connection_get_settings(conn); + const char *error, *host = pshared->addr.a.tcp.https_name; + + if (ssl_iostream_check_cert_validity(conn->ssl_iostream, + host, &error) == 0) + e_debug(conn->event, "SSL handshake successful"); + else if (set->ssl->allow_invalid_cert) { + e_debug(conn->event, "SSL handshake successful, " + "ignoring invalid certificate: %s", error); + } else { + *error_r = error; + return -1; + } + return 0; +} + +static int +http_client_connection_ssl_init(struct http_client_connection *conn, + const char **error_r) +{ + struct http_client_peer_pool *ppool = conn->ppool; + struct http_client_peer_shared *pshared = ppool->peer; + const struct http_client_settings *set = + http_client_connection_get_settings(conn); + struct ssl_iostream_settings ssl_set; + struct ssl_iostream_context *ssl_ctx = ppool->ssl_ctx; + const char *error; + + i_assert(ssl_ctx != NULL); + + ssl_set = *set->ssl; + if (!set->ssl->allow_invalid_cert) + ssl_set.verbose_invalid_cert = TRUE; + + e_debug(conn->event, "Starting SSL handshake"); + + connection_input_halt(&conn->conn); + if (io_stream_create_ssl_client(ssl_ctx, pshared->addr.a.tcp.https_name, + &ssl_set, + &conn->conn.input, &conn->conn.output, + &conn->ssl_iostream, &error) < 0) { + *error_r = t_strdup_printf( + "Couldn't initialize SSL client for %s: %s", + conn->conn.name, error); + return -1; + } + connection_input_resume(&conn->conn); + ssl_iostream_set_handshake_callback( + conn->ssl_iostream, + http_client_connection_ssl_handshaked, conn); + if (ssl_iostream_handshake(conn->ssl_iostream) < 0) { + *error_r = t_strdup_printf( + "SSL handshake to %s failed: %s", conn->conn.name, + ssl_iostream_get_last_error(conn->ssl_iostream)); + return -1; + } + + if (ssl_iostream_is_handshaked(conn->ssl_iostream)) { + http_client_connection_ready(conn); + } else { + /* Wait for handshake to complete; connection input handler does + the rest by reading from the input stream */ + o_stream_set_flush_callback( + conn->conn.output, http_client_connection_output, conn); + } + return 0; +} + +static void +http_client_connection_connected(struct connection *_conn, bool success) +{ + struct http_client_connection *conn = + (struct http_client_connection *)_conn; + struct http_client_peer_shared *pshared = conn->ppool->peer; + const struct http_client_settings *set = + http_client_connection_get_settings(conn); + const char *error; + + if (!success) { + http_client_connection_failure( + conn, t_strdup_printf("connect(%s) failed: %m", + _conn->name)); + } else { + conn->connected_timestamp = ioloop_timeval; + e_debug(conn->event, "Connected"); + + (void)net_set_tcp_nodelay(_conn->fd_out, TRUE); + if (set->socket_send_buffer_size > 0 && + net_set_send_buffer_size( + _conn->fd_out, set->socket_send_buffer_size) < 0) { + i_error("net_set_send_buffer_size(%zu) failed: %m", + set->socket_send_buffer_size); + } + if (set->socket_recv_buffer_size > 0 && + net_set_recv_buffer_size( + _conn->fd_in, set->socket_recv_buffer_size) < 0) { + i_error("net_set_recv_buffer_size(%zu) failed: %m", + set->socket_recv_buffer_size); + } + + if (http_client_peer_addr_is_https(&pshared->addr)) { + if (http_client_connection_ssl_init(conn, &error) < 0) { + e_debug(conn->event, "%s", error); + http_client_connection_failure(conn, error); + http_client_connection_close(&conn); + } + return; + } + http_client_connection_ready(conn); + } +} + +static const struct connection_settings http_client_connection_set = { + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .client = TRUE, + .delayed_unix_client_connected_callback = TRUE, + .log_connection_id = TRUE, +}; + +static const struct connection_vfuncs http_client_connection_vfuncs = { + .destroy = http_client_connection_destroy, + .input = http_client_connection_input, + .client_connected = http_client_connection_connected, +}; + +struct connection_list *http_client_connection_list_init(void) +{ + return connection_list_init(&http_client_connection_set, + &http_client_connection_vfuncs); +} + +static void +http_client_connection_delayed_connect_error( + struct http_client_connection *conn) +{ + timeout_remove(&conn->to_input); + errno = conn->connect_errno; + http_client_connection_connected(&conn->conn, FALSE); + http_client_connection_close(&conn); +} + +static void http_client_connect_timeout(struct http_client_connection *conn) +{ + conn->conn.disconnect_reason = CONNECTION_DISCONNECT_CONNECT_TIMEOUT; + http_client_connection_destroy(&conn->conn); +} + +static void +http_client_connection_connect(struct http_client_connection *conn, + unsigned int timeout_msecs) +{ + struct http_client_context *cctx = conn->ppool->peer->cctx; + + conn->connect_start_timestamp = ioloop_timeval; + if (connection_client_connect(&conn->conn) < 0) { + conn->connect_errno = errno; + e_debug(conn->event, "Connect failed: %m"); + conn->to_input = timeout_add_short_to( + conn->conn.ioloop, 0, + http_client_connection_delayed_connect_error, conn); + return; + } + + /* Don't use connection.h timeout because we want this timeout + to include also the SSL handshake */ + if (timeout_msecs > 0) { + conn->to_connect = timeout_add_to( + cctx->ioloop, timeout_msecs, + http_client_connect_timeout, conn); + } +} + +static void +http_client_connect_tunnel_timeout(struct http_client_connection *conn) +{ + struct http_client_peer_shared *pshared = conn->ppool->peer; + const char *error, *name = http_client_peer_addr2str(&pshared->addr); + unsigned int msecs; + + msecs = timeval_diff_msecs(&ioloop_timeval, + &conn->connect_start_timestamp); + error = t_strdup_printf("Tunnel connect(%s) failed: " + "Connection timed out in %u.%03u secs", + name, msecs/1000, msecs%1000); + + e_debug(conn->event, "%s", error); + http_client_connection_failure(conn, error); + http_client_connection_close(&conn); +} + +static void +http_client_connection_tunnel_response(const struct http_response *response, + struct http_client_connection *conn) +{ + struct http_client_peer_shared *pshared = conn->ppool->peer; + struct http_client_context *cctx = pshared->cctx; + struct http_client_tunnel tunnel; + const char *name = http_client_peer_addr2str(&pshared->addr); + struct http_client_request *req = conn->connect_request; + + conn->connect_request = NULL; + + if (response->status != 200) { + http_client_connection_failure( + conn, + t_strdup_printf("Tunnel connect(%s) failed: %s", name, + http_response_get_message(response))); + return; + } + + http_client_request_start_tunnel(req, &tunnel); + + conn->conn.event_parent = conn->event; + connection_init_from_streams(cctx->conn_list, &conn->conn, + name, tunnel.input, tunnel.output); + connection_switch_ioloop_to(&conn->conn, cctx->ioloop); + i_stream_unref(&tunnel.input); + o_stream_unref(&tunnel.output); +} + +static void +http_client_connection_connect_tunnel(struct http_client_connection *conn, + const struct ip_addr *ip, in_port_t port, + unsigned int timeout_msecs) +{ + struct http_client_context *cctx = conn->ppool->peer->cctx; + struct http_client *client = conn->peer->client; + + conn->connect_start_timestamp = ioloop_timeval; + + conn->connect_request = http_client_request_connect_ip( + client, ip, port, http_client_connection_tunnel_response, conn); + http_client_request_set_urgent(conn->connect_request); + http_client_request_submit(conn->connect_request); + + /* Don't use connection.h timeout because we want this timeout + to include also the SSL handshake */ + if (timeout_msecs > 0) { + conn->to_connect = timeout_add_to( + cctx->ioloop, timeout_msecs, + http_client_connect_tunnel_timeout, conn); + } +} + +struct http_client_connection * +http_client_connection_create(struct http_client_peer *peer) +{ + struct http_client_peer_shared *pshared = peer->shared; + struct http_client_peer_pool *ppool = peer->ppool; + struct http_client_context *cctx = pshared->cctx; + struct http_client *client = peer->client; + const struct http_client_settings *set = &client->set; + struct http_client_connection *conn; + const struct http_client_peer_addr *addr = &pshared->addr; + const char *conn_type = "UNKNOWN"; + unsigned int timeout_msecs; + + switch (pshared->addr.type) { + case HTTP_CLIENT_PEER_ADDR_HTTP: + conn_type = "HTTP"; + break; + case HTTP_CLIENT_PEER_ADDR_HTTPS: + conn_type = "HTTPS"; + break; + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + conn_type = "Tunneled HTTPS"; + break; + case HTTP_CLIENT_PEER_ADDR_RAW: + conn_type = "Raw"; + break; + case HTTP_CLIENT_PEER_ADDR_UNIX: + conn_type = "Unix"; + break; + } + + timeout_msecs = set->connect_timeout_msecs; + if (timeout_msecs == 0) + timeout_msecs = set->request_timeout_msecs; + + conn = i_new(struct http_client_connection, 1); + conn->refcount = 1; + conn->ppool = ppool; + conn->peer = peer; + conn->debug = client->set.debug; + if (pshared->addr.type != HTTP_CLIENT_PEER_ADDR_RAW) + i_array_init(&conn->request_wait_list, 16); + conn->io_wait_timer = io_wait_timer_add_to(cctx->ioloop); + + conn->conn.event_parent = ppool->peer->cctx->event; + connection_init(cctx->conn_list, &conn->conn, + http_client_peer_shared_label(pshared)); + conn->event = conn->conn.event; + + switch (pshared->addr.type) { + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + http_client_connection_connect_tunnel( + conn, &addr->a.tcp.ip, addr->a.tcp.port, timeout_msecs); + break; + case HTTP_CLIENT_PEER_ADDR_UNIX: + connection_init_client_unix(cctx->conn_list, &conn->conn, + addr->a.un.path); + connection_switch_ioloop_to(&conn->conn, cctx->ioloop); + http_client_connection_connect(conn, timeout_msecs); + break; + default: + connection_init_client_ip(cctx->conn_list, &conn->conn, NULL, + &addr->a.tcp.ip, addr->a.tcp.port); + connection_switch_ioloop_to(&conn->conn, cctx->ioloop); + http_client_connection_connect(conn, timeout_msecs); + } + + array_push_back(&ppool->pending_conns, &conn); + array_push_back(&ppool->conns, &conn); + array_push_back(&peer->pending_conns, &conn); + array_push_back(&peer->conns, &conn); + + http_client_peer_pool_ref(ppool); + + e_debug(conn->event, + "%s connection created (%d parallel connections exist)%s", + conn_type, array_count(&ppool->conns), + (conn->to_input == NULL ? "" : " [broken]")); + return conn; +} + +void http_client_connection_ref(struct http_client_connection *conn) +{ + i_assert(conn->refcount > 0); + conn->refcount++; +} + +static void +http_client_connection_disconnect(struct http_client_connection *conn) +{ + struct http_client_peer_pool *ppool = conn->ppool; + ARRAY_TYPE(http_client_connection) *conn_arr; + struct http_client_connection *const *conn_idx; + + if (conn->disconnected) + return; + conn->disconnected = TRUE; + + e_debug(conn->event, "Connection disconnect"); + + conn->closing = TRUE; + conn->connected = FALSE; + + http_client_request_abort(&conn->connect_request); + + if (conn->incoming_payload != NULL) { + /* The stream is still accessed by lib-http caller. */ + i_stream_remove_destroy_callback(conn->incoming_payload, + http_client_payload_destroyed); + conn->incoming_payload = NULL; + } + + if (conn->http_parser != NULL) + http_response_parser_deinit(&conn->http_parser); + + connection_disconnect(&conn->conn); + + io_remove(&conn->io_req_payload); + timeout_remove(&conn->to_requests); + timeout_remove(&conn->to_connect); + timeout_remove(&conn->to_input); + timeout_remove(&conn->to_response); + + /* Remove this connection from the lists */ + conn_arr = &ppool->conns; + array_foreach(conn_arr, conn_idx) { + if (*conn_idx == conn) { + array_delete(conn_arr, + array_foreach_idx(conn_arr, conn_idx), 1); + break; + } + } + conn_arr = &ppool->pending_conns; + array_foreach(conn_arr, conn_idx) { + if (*conn_idx == conn) { + array_delete(conn_arr, + array_foreach_idx(conn_arr, conn_idx), 1); + break; + } + } + + http_client_connection_detach_peer(conn); + + http_client_connection_stop_idle(conn); // FIXME: needed? +} + +bool http_client_connection_unref(struct http_client_connection **_conn) +{ + struct http_client_connection *conn = *_conn; + struct http_client_peer_pool *ppool = conn->ppool; + + i_assert(conn->refcount > 0); + + *_conn = NULL; + + if (--conn->refcount > 0) + return TRUE; + + e_debug(conn->event, "Connection destroy"); + + http_client_connection_disconnect(conn); + http_client_connection_abort_any_requests(conn); + + i_assert(conn->io_req_payload == NULL); + i_assert(conn->to_requests == NULL); + i_assert(conn->to_connect == NULL); + i_assert(conn->to_input == NULL); + i_assert(conn->to_idle == NULL); + i_assert(conn->to_response == NULL); + + if (array_is_created(&conn->request_wait_list)) + array_free(&conn->request_wait_list); + + ssl_iostream_destroy(&conn->ssl_iostream); + connection_deinit(&conn->conn); + io_wait_timer_remove(&conn->io_wait_timer); + + i_free(conn); + + http_client_peer_pool_unref(&ppool); + return FALSE; +} + +void http_client_connection_close(struct http_client_connection **_conn) +{ + struct http_client_connection *conn = *_conn; + + e_debug(conn->event, "Connection close"); + + http_client_connection_disconnect(conn); + http_client_connection_abort_any_requests(conn); + http_client_connection_unref(_conn); +} + +void http_client_connection_switch_ioloop(struct http_client_connection *conn) +{ + struct http_client_peer_shared *pshared = conn->ppool->peer; + struct http_client_context *cctx = pshared->cctx; + struct ioloop *ioloop = cctx->ioloop; + + connection_switch_ioloop_to(&conn->conn, ioloop); + if (conn->io_req_payload != NULL) { + conn->io_req_payload = + io_loop_move_io_to(ioloop, &conn->io_req_payload); + } + if (conn->to_requests != NULL) { + conn->to_requests = + io_loop_move_timeout_to(ioloop, &conn->to_requests); + } + if (conn->to_connect != NULL) { + conn->to_connect = + io_loop_move_timeout_to(ioloop, &conn->to_connect); + } + if (conn->to_input != NULL) { + conn->to_input = + io_loop_move_timeout_to(ioloop, &conn->to_input); + } + if (conn->to_idle != NULL) { + conn->to_idle = + io_loop_move_timeout_to(ioloop, &conn->to_idle); + } + if (conn->to_response != NULL) { + conn->to_response = + io_loop_move_timeout_to(ioloop, &conn->to_response); + } + if (conn->incoming_payload != NULL) + i_stream_switch_ioloop_to(conn->incoming_payload, ioloop); + conn->io_wait_timer = + io_wait_timer_move_to(&conn->io_wait_timer, ioloop); +} diff --git a/src/lib-http/http-client-host.c b/src/lib-http/http-client-host.c new file mode 100644 index 0000000..647ab66 --- /dev/null +++ b/src/lib-http/http-client-host.c @@ -0,0 +1,500 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "net.h" +#include "str.h" +#include "hash.h" +#include "array.h" +#include "llist.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "time-util.h" +#include "dns-lookup.h" +#include "http-response-parser.h" + +#include "http-client-private.h" + +#define HTTP_CLIENT_HOST_MINIMUM_IDLE_TIMEOUT_MSECS 100 + +static void http_client_host_lookup_done(struct http_client_host *host); +static void +http_client_host_lookup_failure(struct http_client_host *host, + const char *error); +static bool http_client_host_is_idle(struct http_client_host *host); +static void http_client_host_free_shared(struct http_client_host **_host); + +/* + * Host (shared) + */ + +static void +http_client_host_shared_idle_timeout(struct http_client_host_shared *hshared) +{ + e_debug(hshared->event, "Idle host timed out"); + http_client_host_shared_free(&hshared); +} + +static void +http_client_host_shared_check_idle(struct http_client_host_shared *hshared) +{ + struct http_client_host *host; + int timeout = 0; + + if (hshared->destroyed) + return; + if (hshared->to_idle != NULL) + return; + + host = hshared->hosts_list; + while (host != NULL) { + if (!http_client_host_is_idle(host)) + return; + host = host->shared_next; + } + + if (!hshared->unix_local && !hshared->explicit_ip && + hshared->ips_timeout.tv_sec > 0) { + timeout = timeval_diff_msecs(&hshared->ips_timeout, + &ioloop_timeval); + } + + if (timeout <= HTTP_CLIENT_HOST_MINIMUM_IDLE_TIMEOUT_MSECS) + timeout = HTTP_CLIENT_HOST_MINIMUM_IDLE_TIMEOUT_MSECS; + + hshared->to_idle = timeout_add_to(hshared->cctx->ioloop, timeout, + http_client_host_shared_idle_timeout, + hshared); + + e_debug(hshared->event, "Host is idle (timeout = %u msecs)", timeout); +} + +static void +http_client_host_shared_lookup_failure(struct http_client_host_shared *hshared, + const char *error) +{ + struct http_client_host *host; + + e_debug(hshared->event, "DNS lookup failed: %s", error); + + error = t_strdup_printf("Failed to lookup host %s: %s", + hshared->name, error); + + host = hshared->hosts_list; + while (host != NULL) { + http_client_host_lookup_failure(host, error); + host = host->shared_next; + } + + http_client_host_shared_check_idle(hshared); +} + +static void +http_client_host_shared_lookup_success(struct http_client_host_shared *hshared, + const struct ip_addr *ips, + unsigned int ips_count) +{ + struct http_client_context *cctx = hshared->cctx; + + i_assert(ips_count > 0); + + e_debug(hshared->event, + "DNS lookup successful; got %d IPs", ips_count); + + hshared->ips = i_realloc_type(hshared->ips, struct ip_addr, + hshared->ips_count, ips_count); + hshared->ips_count = ips_count; + memcpy(hshared->ips, ips, sizeof(struct ip_addr) * ips_count); + + hshared->ips_timeout = ioloop_timeval; + i_assert(cctx->dns_ttl_msecs > 0); + timeval_add_msecs(&hshared->ips_timeout, cctx->dns_ttl_msecs); +} + +static void +http_client_host_shared_dns_callback(const struct dns_lookup_result *result, + struct http_client_host_shared *hshared) +{ + struct http_client_host *host; + + hshared->dns_lookup = NULL; + + if (result->ret != 0) { + /* Lookup failed */ + http_client_host_shared_lookup_failure(hshared, result->error); + return; + } + + http_client_host_shared_lookup_success(hshared, result->ips, + result->ips_count); + + /* Notify all sessions */ + host = hshared->hosts_list; + while (host != NULL) { + http_client_host_lookup_done(host); + host = host->shared_next; + } +} + +static void +http_client_host_shared_lookup(struct http_client_host_shared *hshared) +{ + struct http_client_context *cctx = hshared->cctx; + struct dns_lookup_settings dns_set; + int ret; + + i_assert(!hshared->explicit_ip); + i_assert(hshared->dns_lookup == NULL); + + if (cctx->dns_client != NULL) { + e_debug(hshared->event, "Performing asynchronous DNS lookup"); + (void)dns_client_lookup(cctx->dns_client, hshared->name, + http_client_host_shared_dns_callback, + hshared, &hshared->dns_lookup); + } else if (cctx->dns_client_socket_path != NULL) { + i_assert(cctx->dns_lookup_timeout_msecs > 0); + e_debug(hshared->event, "Performing asynchronous DNS lookup"); + i_zero(&dns_set); + dns_set.dns_client_socket_path = cctx->dns_client_socket_path; + dns_set.timeout_msecs = cctx->dns_lookup_timeout_msecs; + dns_set.ioloop = cctx->ioloop; + dns_set.event_parent = hshared->event; + (void)dns_lookup(hshared->name, &dns_set, + http_client_host_shared_dns_callback, + hshared, &hshared->dns_lookup); + } else { + struct ip_addr *ips; + unsigned int ips_count; + + ret = net_gethostbyname(hshared->name, &ips, &ips_count); + if (ret != 0) { + http_client_host_shared_lookup_failure( + hshared, net_gethosterror(ret)); + return; + } + + http_client_host_shared_lookup_success(hshared, ips, ips_count); + } +} + +static int +http_client_host_shared_refresh(struct http_client_host_shared *hshared) +{ + if (hshared->unix_local) + return 0; + if (hshared->explicit_ip) + return 0; + + if (hshared->dns_lookup != NULL) + return -1; + + if (hshared->ips_count == 0) { + e_debug(hshared->event, "Need to perform DNS lookup"); + } else { + if (timeval_cmp(&hshared->ips_timeout, &ioloop_timeval) > 0) + return 0; + + e_debug(hshared->event, "IPs have expired; " + "need to refresh DNS lookup"); + } + + http_client_host_shared_lookup(hshared); + if (hshared->dns_lookup != NULL) + return -1; + return (hshared->ips_count > 0 ? 1 : -1); +} + +static struct http_client_host_shared * +http_client_host_shared_create(struct http_client_context *cctx, + const char *name) +{ + struct http_client_host_shared *hshared; + + // FIXME: limit the maximum number of inactive cached hosts + hshared = i_new(struct http_client_host_shared, 1); + hshared->cctx = cctx; + hshared->name = i_strdup(name); + hshared->event = event_create(cctx->event); + event_set_append_log_prefix(hshared->event, + t_strdup_printf("host %s: ", name)); + DLLIST_PREPEND(&cctx->hosts_list, hshared); + + return hshared; +} + +static struct http_client_host_shared * +http_client_host_shared_get(struct http_client_context *cctx, + const struct http_url *host_url) +{ + struct http_client_host_shared *hshared; + + if (host_url == NULL) { + hshared = cctx->unix_host; + if (hshared == NULL) { + hshared = http_client_host_shared_create( + cctx, "[unix]"); + hshared->name = i_strdup("[unix]"); + hshared->unix_local = TRUE; + + cctx->unix_host = hshared; + + e_debug(hshared->event, "Unix host created"); + } + + } else { + const char *hostname = host_url->host.name; + struct ip_addr ip = host_url->host.ip; + + hshared = hash_table_lookup(cctx->hosts, hostname); + if (hshared == NULL) { + hshared = http_client_host_shared_create( + cctx, hostname); + hostname = hshared->name; + hash_table_insert(cctx->hosts, hostname, hshared); + + if (ip.family != 0 || + net_addr2ip(hshared->name, &ip) == 0) { + hshared->ips_count = 1; + hshared->ips = i_new(struct ip_addr, + hshared->ips_count); + hshared->ips[0] = ip; + hshared->explicit_ip = TRUE; + } + + e_debug(hshared->event, "Host created"); + } + } + return hshared; +} + +void http_client_host_shared_free(struct http_client_host_shared **_hshared) +{ + struct http_client_host_shared *hshared = *_hshared; + struct http_client_context *cctx = hshared->cctx; + struct http_client_host *host; + const char *hostname = hshared->name; + + if (hshared->destroyed) + return; + hshared->destroyed = TRUE; + + e_debug(hshared->event, "Host destroy"); + + timeout_remove(&hshared->to_idle); + + DLLIST_REMOVE(&cctx->hosts_list, hshared); + if (hshared == cctx->unix_host) + cctx->unix_host = NULL; + else + hash_table_remove(cctx->hosts, hostname); + + if (hshared->dns_lookup != NULL) + dns_lookup_abort(&hshared->dns_lookup); + + /* Drop client sessions */ + while (hshared->hosts_list != NULL) { + host = hshared->hosts_list; + http_client_host_free_shared(&host); + } + + event_unref(&hshared->event); + i_free(hshared->ips); + i_free(hshared->name); + i_free(hshared); + + *_hshared = NULL; +} + +static void +http_client_host_shared_request_submitted( + struct http_client_host_shared *hshared) +{ + /* Cancel host idle timeout */ + timeout_remove(&hshared->to_idle); +} + +void http_client_host_shared_switch_ioloop( + struct http_client_host_shared *hshared) +{ + struct http_client_context *cctx = hshared->cctx; + + if (hshared->dns_lookup != NULL && cctx->dns_client == NULL) + dns_lookup_switch_ioloop(hshared->dns_lookup); + if (hshared->to_idle != NULL) + hshared->to_idle = io_loop_move_timeout(&hshared->to_idle); +} + +/* + * Host + */ + +struct http_client_host * +http_client_host_get(struct http_client *client, + const struct http_url *host_url) +{ + struct http_client_host_shared *hshared; + struct http_client_host *host; + + hshared = http_client_host_shared_get(client->cctx, host_url); + + host = hshared->hosts_list; + while (host != NULL) { + if (host->client == client) + break; + host = host->shared_next; + } + + if (host == NULL) { + host = i_new(struct http_client_host, 1); + host->client = client; + host->shared = hshared; + i_array_init(&host->queues, 4); + DLLIST_PREPEND_FULL(&hshared->hosts_list, + host, shared_prev, shared_next); + DLLIST_PREPEND_FULL(&client->hosts_list, host, + client_prev, client_next); + + e_debug(hshared->event, "Host session created"); + } + + return host; +} + +static void http_client_host_free_shared(struct http_client_host **_host) +{ + struct http_client_host *host = *_host; + struct http_client *client = host->client; + struct http_client_host_shared *hshared = host->shared; + struct http_client_queue *queue; + ARRAY_TYPE(http_client_queue) queues; + + *_host = NULL; + + e_debug(hshared->event, "Host session destroy"); + + DLLIST_REMOVE_FULL(&hshared->hosts_list, host, + shared_prev, shared_next); + DLLIST_REMOVE_FULL(&client->hosts_list, host, + client_prev, client_next); + + /* Drop request queues */ + t_array_init(&queues, array_count(&host->queues)); + array_copy(&queues.arr, 0, &host->queues.arr, 0, + array_count(&host->queues)); + array_clear(&host->queues); + array_foreach_elem(&queues, queue) + http_client_queue_free(queue); + array_free(&host->queues); + + i_free(host); +} + +void http_client_host_free(struct http_client_host **_host) +{ + struct http_client_host *host = *_host; + struct http_client_host_shared *hshared = host->shared; + + http_client_host_free_shared(_host); + + http_client_host_shared_check_idle(hshared); +} + +static void http_client_host_lookup_done(struct http_client_host *host) +{ + struct http_client *client = host->client; + struct http_client_queue *queue; + unsigned int requests = 0; + + /* Notify all queues */ + array_foreach_elem(&host->queues, queue) + requests += http_client_queue_host_lookup_done(queue); + + if (requests == 0 && client->waiting) + io_loop_stop(client->ioloop); +} + +static void +http_client_host_lookup_failure(struct http_client_host *host, + const char *error) +{ + struct http_client_queue *queue; + + array_foreach_elem(&host->queues, queue) + http_client_queue_host_lookup_failure(queue, error); +} + +void http_client_host_submit_request(struct http_client_host *host, + struct http_client_request *req) +{ + struct http_client *client = req->client; + struct http_client_queue *queue; + struct http_client_peer_addr addr; + const char *error; + + req->host = host; + + http_client_request_get_peer_addr(req, &addr); + if (http_client_peer_addr_is_https(&addr) && + client->ssl_ctx == NULL) { + if (http_client_init_ssl_ctx(client, &error) < 0) { + http_client_request_error( + &req, HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, + error); + return; + } + } + + /* Add request to queue */ + queue = http_client_queue_get(host, &addr); + http_client_queue_submit_request(queue, req); + + /* Update shared host object (idle timeout) */ + http_client_host_shared_request_submitted(host->shared); + + /* Queue will trigger host lookup once the request is activated + (may be delayed) */ +} + +static bool http_client_host_is_idle(struct http_client_host *host) +{ + struct http_client_queue *queue; + unsigned int requests = 0; + + array_foreach_elem(&host->queues, queue) + requests += http_client_queue_requests_active(queue); + + return (requests == 0); +} + +void http_client_host_check_idle(struct http_client_host *host) +{ + http_client_host_shared_check_idle(host->shared); +} + +int http_client_host_refresh(struct http_client_host *host) +{ + return http_client_host_shared_refresh(host->shared); +} + +bool http_client_host_get_ip_idx(struct http_client_host *host, + const struct ip_addr *ip, unsigned int *idx_r) +{ + struct http_client_host_shared *hshared = host->shared; + unsigned int i; + + for (i = 0; i < hshared->ips_count; i++) { + if (net_ip_compare(&hshared->ips[i], ip)) { + *idx_r = i; + return TRUE; + } + } + return FALSE; +} + +void http_client_host_switch_ioloop(struct http_client_host *host) +{ + struct http_client_queue *queue; + + array_foreach_elem(&host->queues, queue) + http_client_queue_switch_ioloop(queue); +} diff --git a/src/lib-http/http-client-peer.c b/src/lib-http/http-client-peer.c new file mode 100644 index 0000000..eff40b4 --- /dev/null +++ b/src/lib-http/http-client-peer.c @@ -0,0 +1,1383 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "net.h" +#include "time-util.h" +#include "str.h" +#include "hash.h" +#include "array.h" +#include "llist.h" +#include "istream.h" +#include "ostream.h" +#include "iostream-ssl.h" +#include "http-response-parser.h" + +#include "http-client-private.h" + +static void +http_client_peer_connect_backoff(struct http_client_peer *peer); + +static void +http_client_peer_shared_connection_success( + struct http_client_peer_shared *pshared); +static void +http_client_peer_shared_connection_failure( + struct http_client_peer_shared *pshared); +static void +http_client_peer_connection_succeeded_pool(struct http_client_peer *peer); +static void +http_client_peer_connection_failed_pool(struct http_client_peer *peer, + const char *reason); + +/* + * Peer address + */ + +unsigned int ATTR_NO_SANITIZE_INTEGER +http_client_peer_addr_hash(const struct http_client_peer_addr *peer) +{ + unsigned int hash = (unsigned int)peer->type; + + switch (peer->type) { + case HTTP_CLIENT_PEER_ADDR_HTTPS: + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + if (peer->a.tcp.https_name != NULL) + hash += str_hash(peer->a.tcp.https_name); + /* fall through */ + case HTTP_CLIENT_PEER_ADDR_RAW: + case HTTP_CLIENT_PEER_ADDR_HTTP: + if (peer->a.tcp.ip.family != 0) + hash += net_ip_hash(&peer->a.tcp.ip); + hash += peer->a.tcp.port; + break; + case HTTP_CLIENT_PEER_ADDR_UNIX: + hash += str_hash(peer->a.un.path); + break; + } + + return hash; +} + +int http_client_peer_addr_cmp(const struct http_client_peer_addr *peer1, + const struct http_client_peer_addr *peer2) +{ + int ret; + + if (peer1->type != peer2->type) + return (peer1->type > peer2->type ? 1 : -1); + switch (peer1->type) { + case HTTP_CLIENT_PEER_ADDR_RAW: + case HTTP_CLIENT_PEER_ADDR_HTTP: + case HTTP_CLIENT_PEER_ADDR_HTTPS: + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + /* Queues are created with peer addresses that have an + uninitialized IP value, because that is assigned later when + the host lookup completes. In all other other contexts, the + IP is always initialized, so we do not compare IPs when one + of them is unassigned. */ + if (peer1->a.tcp.ip.family != 0 && + peer2->a.tcp.ip.family != 0 && + (ret = net_ip_cmp(&peer1->a.tcp.ip, &peer2->a.tcp.ip)) != 0) + return ret; + if (peer1->a.tcp.port != peer2->a.tcp.port) + return (peer1->a.tcp.port > peer2->a.tcp.port ? 1 : -1); + if (peer1->type != HTTP_CLIENT_PEER_ADDR_HTTPS && + peer1->type != HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL) + return 0; + return null_strcmp(peer1->a.tcp.https_name, + peer2->a.tcp.https_name); + case HTTP_CLIENT_PEER_ADDR_UNIX: + return null_strcmp(peer1->a.un.path, peer2->a.un.path); + } + i_unreached(); +} + +/* + * Peer pool + */ + +static struct http_client_peer_pool * +http_client_peer_pool_create(struct http_client_peer_shared *pshared, + struct ssl_iostream_context *ssl_ctx, + const char *rawlog_dir) +{ + struct http_client_peer_pool *ppool; + + ppool = i_new(struct http_client_peer_pool, 1); + ppool->refcount = 1; + ppool->peer = pshared; + ppool->event = event_create(pshared->cctx->event); + event_set_append_log_prefix( + ppool->event, + t_strdup_printf("peer %s: ", + http_client_peer_shared_label(pshared))); + + http_client_peer_shared_ref(pshared); + + i_array_init(&ppool->conns, 16); + i_array_init(&ppool->pending_conns, 16); + i_array_init(&ppool->idle_conns, 16); + + DLLIST_PREPEND(&pshared->pools_list, ppool); + + ppool->ssl_ctx = ssl_ctx; + ppool->rawlog_dir = i_strdup(rawlog_dir); + + e_debug(ppool->event, "Peer pool created"); + + return ppool; +} + +void http_client_peer_pool_ref(struct http_client_peer_pool *ppool) +{ + if (ppool->destroyed) + return; + ppool->refcount++; +} + +void http_client_peer_pool_close(struct http_client_peer_pool **_ppool) +{ + struct http_client_peer_pool *ppool = *_ppool; + struct http_client_connection *conn; + ARRAY_TYPE(http_client_connection) conns; + + http_client_peer_pool_ref(ppool); + + /* Make a copy of the connection array; freed connections modify it */ + t_array_init(&conns, array_count(&ppool->conns)); + array_copy(&conns.arr, 0, &ppool->conns.arr, 0, + array_count(&ppool->conns)); + array_foreach_elem(&conns, conn) + http_client_connection_unref(&conn); + i_assert(array_count(&ppool->idle_conns) == 0); + i_assert(array_count(&ppool->pending_conns) == 0); + i_assert(array_count(&ppool->conns) == 0); + + http_client_peer_pool_unref(_ppool); +} + +void http_client_peer_pool_unref(struct http_client_peer_pool **_ppool) +{ + struct http_client_peer_pool *ppool = *_ppool; + struct http_client_peer_shared *pshared = ppool->peer; + + *_ppool = NULL; + + if (ppool->destroyed) + return; + + i_assert(ppool->refcount > 0); + if (--ppool->refcount > 0) + return; + + e_debug(ppool->event, "Peer pool destroy"); + ppool->destroyed = TRUE; + + i_assert(array_count(&ppool->idle_conns) == 0); + i_assert(array_count(&ppool->conns) == 0); + array_free(&ppool->idle_conns); + array_free(&ppool->pending_conns); + array_free(&ppool->conns); + + DLLIST_REMOVE(&pshared->pools_list, ppool); + + event_unref(&ppool->event); + i_free(ppool->rawlog_dir); + i_free(ppool); + http_client_peer_shared_unref(&pshared); +} + +static struct http_client_peer_pool * +http_client_peer_pool_get(struct http_client_peer_shared *pshared, + struct http_client *client) +{ + struct http_client_peer_pool *ppool; + struct ssl_iostream_context *ssl_ctx = client->ssl_ctx; + const char *rawlog_dir = client->set.rawlog_dir; + + i_assert(!http_client_peer_addr_is_https(&pshared->addr) || + ssl_ctx != NULL); + + ppool = pshared->pools_list; + while (ppool != NULL) { + if (ppool->ssl_ctx == ssl_ctx && + null_strcmp(ppool->rawlog_dir, rawlog_dir) == 0) + break; + ppool = ppool->next; + } + + if (ppool == NULL) { + ppool = http_client_peer_pool_create(pshared, ssl_ctx, + rawlog_dir); + } else { + e_debug(ppool->event, "Peer pool reused"); + http_client_peer_pool_ref(ppool); + } + + return ppool; +} + +static void +http_client_peer_pool_connection_success(struct http_client_peer_pool *ppool) +{ + e_debug(ppool->event, "Successfully connected " + "(%u connections exist, %u pending)", + array_count(&ppool->conns), array_count(&ppool->pending_conns)); + + http_client_peer_shared_connection_success(ppool->peer); + + if (array_count(&ppool->pending_conns) > 0) { + /* If there are other connections attempting to connect, wait + for them before notifying other peer objects about the + success (which may be premature). */ + } else { + struct http_client_peer *peer; + + /* This was the only/last connection and connecting to it + succeeded. notify all interested peers in this pool about the + success */ + peer = ppool->peer->peers_list; + while (peer != NULL) { + struct http_client_peer *peer_next = peer->shared_next; + if (peer->ppool == ppool) + http_client_peer_connection_succeeded_pool(peer); + peer = peer_next; + } + } +} + +static void +http_client_peer_pool_connection_failure(struct http_client_peer_pool *ppool, + const char *reason) +{ + e_debug(ppool->event, + "Failed to make connection " + "(%u connections exist, %u pending)", + array_count(&ppool->conns), array_count(&ppool->pending_conns)); + + http_client_peer_shared_connection_failure(ppool->peer); + + if (array_count(&ppool->pending_conns) > 0) { + /* If there are other connections attempting to connect, wait + for them before failing the requests. remember that we had + trouble with connecting so in future we don't try to create + more than one connection until connects work again. */ + } else { + struct http_client_peer *peer; + + /* This was the only/last connection and connecting to it + failed. notify all interested peers in this pool about the + failure */ + peer = ppool->peer->peers_list; + while (peer != NULL) { + struct http_client_peer *peer_next = peer->shared_next; + if (peer->ppool == ppool) + http_client_peer_connection_failed_pool(peer, reason); + peer = peer_next; + } + } +} + +/* + * Peer (shared) + */ + +static struct http_client_peer_shared * +http_client_peer_shared_create(struct http_client_context *cctx, + const struct http_client_peer_addr *addr) +{ + struct http_client_peer_shared *pshared; + + pshared = i_new(struct http_client_peer_shared, 1); + pshared->refcount = 1; + pshared->cctx = cctx; + + pshared->addr = *addr; + switch (addr->type) { + case HTTP_CLIENT_PEER_ADDR_RAW: + case HTTP_CLIENT_PEER_ADDR_HTTP: + i_assert(pshared->addr.a.tcp.ip.family != 0); + break; + case HTTP_CLIENT_PEER_ADDR_HTTPS: + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + i_assert(pshared->addr.a.tcp.ip.family != 0); + pshared->addr_name = i_strdup(addr->a.tcp.https_name); + pshared->addr.a.tcp.https_name = pshared->addr_name; + break; + case HTTP_CLIENT_PEER_ADDR_UNIX: + pshared->addr_name = i_strdup(addr->a.un.path); + pshared->addr.a.un.path = pshared->addr_name; + break; + default: + break; + } + pshared->event = event_create(cctx->event); + event_set_append_log_prefix( + pshared->event, + t_strdup_printf("peer %s (shared): ", + http_client_peer_shared_label(pshared))); + + hash_table_insert(cctx->peers, + (const struct http_client_peer_addr *)&pshared->addr, + pshared); + DLLIST_PREPEND(&cctx->peers_list, pshared); + + pshared->backoff_initial_time_msecs = + cctx->set.connect_backoff_time_msecs; + pshared->backoff_max_time_msecs = + cctx->set.connect_backoff_max_time_msecs; + + e_debug(pshared->event, "Peer created"); + return pshared; +} + +void http_client_peer_shared_ref(struct http_client_peer_shared *pshared) +{ + pshared->refcount++; +} + +void http_client_peer_shared_unref(struct http_client_peer_shared **_pshared) +{ + struct http_client_peer_shared *pshared = *_pshared; + + *_pshared = NULL; + + i_assert(pshared->refcount > 0); + if (--pshared->refcount > 0) + return; + + e_debug(pshared->event, "Peer destroy"); + + i_assert(pshared->pools_list == NULL); + + /* Unlist in client */ + hash_table_remove(pshared->cctx->peers, + (const struct http_client_peer_addr *)&pshared->addr); + DLLIST_REMOVE(&pshared->cctx->peers_list, pshared); + + timeout_remove(&pshared->to_backoff); + + event_unref(&pshared->event); + i_free(pshared->addr_name); + i_free(pshared->label); + i_free(pshared); +} + +static struct http_client_peer_shared * +http_client_peer_shared_get(struct http_client_context *cctx, + const struct http_client_peer_addr *addr) +{ + struct http_client_peer_shared *pshared; + + pshared = hash_table_lookup(cctx->peers, addr); + if (pshared == NULL) { + pshared = http_client_peer_shared_create(cctx, addr); + } else { + e_debug(pshared->event, "Peer reused"); + http_client_peer_shared_ref(pshared); + } + + return pshared; +} + +void http_client_peer_shared_close(struct http_client_peer_shared **_pshared) +{ + struct http_client_peer_shared *pshared = *_pshared; + struct http_client_peer_pool *pool, *next; + + http_client_peer_shared_ref(pshared); + pool = pshared->pools_list; + while (pool != NULL) { + next = pool->next; + http_client_peer_pool_close(&pool); + pool = next; + } + http_client_peer_shared_unref(_pshared); +} + +const char * +http_client_peer_shared_label(struct http_client_peer_shared *pshared) +{ + if (pshared->label == NULL) { + switch (pshared->addr.type) { + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + pshared->label = i_strconcat( + http_client_peer_addr2str(&pshared->addr), + " (tunnel)", NULL); + break; + default: + pshared->label = i_strdup( + http_client_peer_addr2str(&pshared->addr)); + } + } + return pshared->label; +} + +static void +http_client_peer_shared_connect_backoff(struct http_client_peer_shared *pshared) +{ + struct http_client_peer *peer; + + i_assert(pshared->to_backoff != NULL); + + e_debug(pshared->event, "Backoff timer expired"); + + timeout_remove(&pshared->to_backoff); + + peer = pshared->peers_list; + while (peer != NULL) { + struct http_client_peer *peer_next = peer->shared_next; + + http_client_peer_connect_backoff(peer); + peer = peer_next; + } +} + +static bool +http_client_peer_shared_start_backoff_timer( + struct http_client_peer_shared *pshared) +{ + if (pshared->to_backoff != NULL) + return TRUE; + + if (pshared->last_failure.tv_sec > 0) { + int backoff_time_spent = + timeval_diff_msecs(&ioloop_timeval, + &pshared->last_failure); + + if (backoff_time_spent < (int)pshared->backoff_current_time_msecs) { + unsigned int new_time = (unsigned int) + (pshared->backoff_current_time_msecs - + backoff_time_spent); + e_debug(pshared->event, + "Starting backoff timer for %d msecs", new_time); + pshared->to_backoff = timeout_add_to( + pshared->cctx->ioloop, new_time, + http_client_peer_shared_connect_backoff, pshared); + return TRUE; + } + + e_debug(pshared->event, + "Backoff time already exceeded by %d msecs", + (backoff_time_spent - + pshared->backoff_current_time_msecs)); + } + return FALSE; +} + +static void +http_client_peer_shared_increase_backoff_timer( + struct http_client_peer_shared *pshared) +{ + if (pshared->backoff_current_time_msecs == 0) { + pshared->backoff_current_time_msecs = + pshared->backoff_initial_time_msecs; + } else { + pshared->backoff_current_time_msecs *= 2; + } + if (pshared->backoff_current_time_msecs > + pshared->backoff_max_time_msecs) { + pshared->backoff_current_time_msecs = + pshared->backoff_max_time_msecs; + } +} + +static void +http_client_peer_shared_reset_backoff_timer( + struct http_client_peer_shared *pshared) +{ + pshared->backoff_current_time_msecs = 0; + + timeout_remove(&pshared->to_backoff); +} + +static void +http_client_peer_shared_connection_success( + struct http_client_peer_shared *pshared) +{ + pshared->last_failure.tv_sec = pshared->last_failure.tv_usec = 0; + http_client_peer_shared_reset_backoff_timer(pshared); +} + +static void +http_client_peer_shared_connection_failure( + struct http_client_peer_shared *pshared) +{ + struct http_client_peer_pool *ppool; + unsigned int pending = 0; + + /* Determine the number of connections still pending */ + ppool = pshared->pools_list; + while (ppool != NULL) { + pending += array_count(&ppool->pending_conns); + ppool = ppool->next; + } + + pshared->last_failure = ioloop_timeval; + + /* Manage backoff timer only when this was the only attempt */ + if (pending == 0) + http_client_peer_shared_increase_backoff_timer(pshared); +} + +static void +http_client_peer_shared_connection_lost(struct http_client_peer_shared *pshared, + bool premature) +{ + /* Update backoff timer if the connection was lost prematurely. this + prevents reconnecting immediately to a server that is misbehaving by + disconnecting before sending a response. + */ + if (premature) { + pshared->last_failure = ioloop_timeval; + http_client_peer_shared_increase_backoff_timer(pshared); + } +} + +void http_client_peer_shared_switch_ioloop( + struct http_client_peer_shared *pshared) +{ + if (pshared->to_backoff != NULL) { + pshared->to_backoff = + io_loop_move_timeout(&pshared->to_backoff); + } +} + +unsigned int +http_client_peer_shared_max_connections(struct http_client_peer_shared *pshared) +{ + struct http_client_peer *peer; + unsigned int max_conns = 0; + + peer = pshared->peers_list; + while (peer != NULL) { + const struct http_client_settings *set = &peer->client->set; + unsigned int client_max_conns = set->max_parallel_connections; + + if ((UINT_MAX - max_conns) <= client_max_conns) + return UINT_MAX; + max_conns += client_max_conns; + peer = peer->shared_next; + } + + return max_conns; +} + +/* + * Peer + */ + +static void http_client_peer_drop(struct http_client_peer **_peer); + +static struct http_client_peer * +http_client_peer_create(struct http_client *client, + struct http_client_peer_shared *pshared) +{ + struct http_client_peer *peer; + + peer = i_new(struct http_client_peer, 1); + peer->refcount = 1; + peer->client = client; + peer->shared = pshared; + + peer->event = event_create(client->event); + event_set_append_log_prefix(peer->event, t_strdup_printf( + "peer %s: ", http_client_peer_shared_label(pshared))); + + i_array_init(&peer->queues, 16); + i_array_init(&peer->conns, 16); + i_array_init(&peer->pending_conns, 16); + + DLLIST_PREPEND_FULL(&client->peers_list, peer, + client_prev, client_next); + DLLIST_PREPEND_FULL(&pshared->peers_list, peer, + shared_prev, shared_next); + pshared->peers_count++; + + http_client_peer_shared_ref(pshared); + peer->ppool = http_client_peer_pool_get(pshared, client); + + /* Choose backoff times */ + if (pshared->peers_list == NULL || + (client->set.connect_backoff_time_msecs < + pshared->backoff_initial_time_msecs)) { + pshared->backoff_initial_time_msecs = + client->set.connect_backoff_time_msecs; + } + if (pshared->peers_list == NULL || + (client->set.connect_backoff_max_time_msecs > + pshared->backoff_max_time_msecs)) { + pshared->backoff_max_time_msecs = + client->set.connect_backoff_max_time_msecs; + } + + e_debug(peer->event, "Peer created"); + return peer; +} + +void http_client_peer_ref(struct http_client_peer *peer) +{ + peer->refcount++; +} + +static void http_client_peer_disconnect(struct http_client_peer *peer) +{ + struct http_client_queue *queue; + struct http_client *client = peer->client; + struct http_client_peer_shared *pshared = peer->shared; + struct http_client_connection *conn; + ARRAY_TYPE(http_client_connection) conns; + + if (peer->disconnected) + return; + peer->disconnected = TRUE; + + e_debug(peer->event, "Peer disconnect"); + + /* Make a copy of the connection array; freed connections modify it */ + t_array_init(&conns, array_count(&peer->conns)); + array_copy(&conns.arr, 0, &peer->conns.arr, 0, + array_count(&peer->conns)); + array_foreach_elem(&conns, conn) + http_client_connection_lost_peer(conn); + i_assert(array_count(&peer->conns) == 0); + array_clear(&peer->pending_conns); + + timeout_remove(&peer->to_req_handling); + + /* Unlist in client */ + DLLIST_REMOVE_FULL(&client->peers_list, peer, + client_prev, client_next); + /* Unlist in peer */ + DLLIST_REMOVE_FULL(&pshared->peers_list, peer, + shared_prev, shared_next); + pshared->peers_count--; + + /* Unlink all queues */ + array_foreach_elem(&peer->queues, queue) + http_client_queue_peer_disconnected(queue, peer); + array_clear(&peer->queues); +} + +bool http_client_peer_unref(struct http_client_peer **_peer) +{ + struct http_client_peer *peer = *_peer; + struct http_client_peer_pool *ppool = peer->ppool; + struct http_client_peer_shared *pshared = peer->shared; + + *_peer = NULL; + + i_assert(peer->refcount > 0); + if (--peer->refcount > 0) + return TRUE; + + e_debug(peer->event, "Peer destroy"); + + http_client_peer_disconnect(peer); + + i_assert(array_count(&peer->queues) == 0); + + event_unref(&peer->event); + array_free(&peer->conns); + array_free(&peer->pending_conns); + array_free(&peer->queues); + i_free(peer); + + /* Choose new backoff times */ + peer = pshared->peers_list; + while (peer != NULL) { + struct http_client *client = peer->client; + + if (client->set.connect_backoff_time_msecs < + pshared->backoff_initial_time_msecs) { + pshared->backoff_initial_time_msecs = + client->set.connect_backoff_time_msecs; + } + if (client->set.connect_backoff_max_time_msecs > + pshared->backoff_max_time_msecs) { + pshared->backoff_max_time_msecs = + client->set.connect_backoff_max_time_msecs; + } + peer = peer->shared_next; + } + + http_client_peer_pool_unref(&ppool); + http_client_peer_shared_unref(&pshared); + return FALSE; +} + +void http_client_peer_close(struct http_client_peer **_peer) +{ + struct http_client_peer *peer = *_peer; + + e_debug(peer->event, "Peer close"); + + http_client_peer_disconnect(peer); + + (void)http_client_peer_unref(_peer); +} + +static void http_client_peer_drop(struct http_client_peer **_peer) +{ + struct http_client_peer *peer = *_peer; + struct http_client_peer_shared *pshared = peer->shared; + unsigned int conns_active = + http_client_peer_active_connections(peer); + + if (conns_active > 0) { + e_debug(peer->event, + "Not dropping peer (%d connections active)", + conns_active); + return; + } + + if (pshared->to_backoff != NULL) + return; + + if (http_client_peer_shared_start_backoff_timer(pshared)) { + e_debug(peer->event, + "Dropping peer (waiting for backof timeout)"); + + /* Will disconnect any pending connections */ + http_client_peer_trigger_request_handler(peer); + } else { + e_debug(peer->event, "Dropping peer now"); + /* Drop peer immediately */ + http_client_peer_close(_peer); + } +} + +struct http_client_peer * +http_client_peer_get(struct http_client *client, + const struct http_client_peer_addr *addr) +{ + struct http_client_peer *peer; + struct http_client_peer_shared *pshared; + + pshared = http_client_peer_shared_get(client->cctx, addr); + + peer = pshared->peers_list; + while (peer != NULL) { + if (peer->client == client) + break; + peer = peer->shared_next; + } + + if (peer == NULL) + peer = http_client_peer_create(client, pshared); + + http_client_peer_shared_unref(&pshared); + return peer; +} + +static void +http_client_peer_do_connect(struct http_client_peer *peer, unsigned int count) +{ + struct http_client_peer_pool *ppool = peer->ppool; + struct http_client_connection *const *idle_conns; + unsigned int i, idle_count; + bool claimed_existing = FALSE; + + if (count == 0) + return; + + idle_conns = array_get(&ppool->idle_conns, &idle_count); + for (i = 0; i < count && i < idle_count; i++) { + http_client_connection_claim_idle(idle_conns[i], peer); + claimed_existing = TRUE; + + e_debug(peer->event, + "Claimed idle connection " + "(%u connections exist, %u pending)", + array_count(&peer->conns), + array_count(&peer->pending_conns)); + } + + for (; i < count; i++) { + e_debug(peer->event, + "Making new connection %u of %u " + "(%u connections exist, %u pending)", + i+1, count, array_count(&peer->conns), + array_count(&peer->pending_conns)); + + (void)http_client_connection_create(peer); + } + + if (claimed_existing) + http_client_peer_connection_success(peer); +} + +static void http_client_peer_connect_backoff(struct http_client_peer *peer) +{ + if (peer->connect_backoff && array_count(&peer->queues) == 0) { + http_client_peer_close(&peer); + return; + } + + http_client_peer_do_connect(peer, 1); + peer->connect_backoff = FALSE; +} + +static void +http_client_peer_connect(struct http_client_peer *peer, unsigned int count) +{ + if (http_client_peer_shared_start_backoff_timer(peer->shared)) { + peer->connect_backoff = TRUE; + return; + } + + http_client_peer_do_connect(peer, count); +} + +bool http_client_peer_is_connected(struct http_client_peer *peer) +{ + struct http_client_connection *conn; + + if (array_count(&peer->ppool->idle_conns) > 0) + return TRUE; + + array_foreach_elem(&peer->conns, conn) { + if (conn->connected) + return TRUE; + } + + return FALSE; +} + +static void +http_client_peer_cancel(struct http_client_peer *peer) +{ + struct http_client_connection *conn; + ARRAY_TYPE(http_client_connection) conns; + + e_debug(peer->event, "Peer cancel"); + + /* Make a copy of the connection array; freed connections modify it */ + t_array_init(&conns, array_count(&peer->conns)); + array_copy(&conns.arr, 0, &peer->conns.arr, 0, + array_count(&peer->conns)); + array_foreach_elem(&conns, conn) { + if (!http_client_connection_is_active(conn)) + http_client_connection_close(&conn); + } + i_assert(array_count(&peer->pending_conns) == 0); +} + +static unsigned int +http_client_peer_requests_pending(struct http_client_peer *peer, + unsigned int *num_urgent_r) +{ + struct http_client_queue *queue; + unsigned int num_requests = 0, num_urgent = 0, requests, urgent; + + array_foreach_elem(&peer->queues, queue) { + requests = http_client_queue_requests_pending(queue, &urgent); + + num_requests += requests; + num_urgent += urgent; + } + *num_urgent_r = num_urgent; + return num_requests; +} + +static void http_client_peer_check_idle(struct http_client_peer *peer) +{ + struct http_client_connection *conn; + unsigned int num_urgent = 0; + + if (array_count(&peer->conns) == 0 && + http_client_peer_requests_pending(peer, &num_urgent) == 0) { + /* No connections or pending requests; disconnect immediately */ + http_client_peer_drop(&peer); + return; + } + + /* Check all connections for idle status */ + array_foreach_elem(&peer->conns, conn) + http_client_connection_check_idle(conn); +} + +static void +http_client_peer_handle_requests_real(struct http_client_peer *peer) +{ + struct _conn_available { + struct http_client_connection *conn; + unsigned int pending_requests; + }; + struct http_client_connection *const *conn_idx; + ARRAY(struct _conn_available) conns_avail; + struct _conn_available *conn_avail_idx; + struct http_client_peer_shared *pshared = peer->shared; + unsigned int connecting, closing, idle; + unsigned int num_pending, num_urgent, new_connections; + unsigned int working_conn_count; + struct http_client_peer *tmp_peer; + bool statistics_dirty = TRUE; + + /* FIXME: limit the number of requests handled in one run to prevent + I/O starvation. */ + + /* Disconnect pending connections if we're not linked to any queue + anymore */ + if (array_count(&peer->queues) == 0) { + if (array_count(&peer->conns) == 0 && + pshared->to_backoff == NULL) { + /* Peer is completely unused and inactive; drop it + immediately */ + http_client_peer_drop(&peer); + return; + } + e_debug(peer->event, + "Peer no longer used; will now cancel pending connections " + "(%u connections exist, %u pending)", + array_count(&peer->conns), + array_count(&peer->pending_conns)); + + http_client_peer_cancel(peer); + return; + } + + /* Don't do anything unless we have pending requests */ + num_pending = http_client_peer_requests_pending(peer, &num_urgent); + if (num_pending == 0) { + e_debug(peer->event, + "No requests to service for this peer " + "(%u connections exist, %u pending)", + array_count(&peer->conns), + array_count(&peer->pending_conns)); + http_client_peer_check_idle(peer); + return; + } + + http_client_peer_ref(peer); + peer->handling_requests = TRUE; + t_array_init(&conns_avail, array_count(&peer->conns)); + do { + bool conn_lost = FALSE; + + array_clear(&conns_avail); + closing = idle = 0; + + /* Gather connection statistics */ + array_foreach(&peer->conns, conn_idx) { + struct http_client_connection *conn = *conn_idx; + int ret; + + ret = http_client_connection_check_ready(conn); + if (ret < 0) { + conn_lost = TRUE; + break; + } else if (ret > 0) { + struct _conn_available *conn_avail; + unsigned int insert_idx, pending_requests; + + /* Compile sorted availability list */ + pending_requests = http_client_connection_count_pending(conn); + if (array_count(&conns_avail) == 0) { + insert_idx = 0; + } else { + insert_idx = array_count(&conns_avail); + array_foreach_modifiable(&conns_avail, conn_avail_idx) { + if (conn_avail_idx->pending_requests > pending_requests) { + insert_idx = array_foreach_idx(&conns_avail, conn_avail_idx); + break; + } + } + } + conn_avail = array_insert_space(&conns_avail, insert_idx); + conn_avail->conn = conn; + conn_avail->pending_requests = pending_requests; + if (pending_requests == 0) + idle++; + } + /* Count the number of connecting and closing connections */ + if (conn->closing) + closing++; + } + + if (conn_lost) { + /* Connection array changed while iterating; retry */ + continue; + } + + working_conn_count = array_count(&peer->conns) - closing; + statistics_dirty = FALSE; + + /* Use idle connections right away */ + if (idle > 0) { + e_debug(peer->event, + "Using %u idle connections to handle %u requests " + "(%u total connections ready)", + idle, num_pending > idle ? idle : num_pending, + array_count(&conns_avail)); + + array_foreach_modifiable(&conns_avail, conn_avail_idx) { + if (num_pending == 0 || + conn_avail_idx->pending_requests > 0) + break; + idle--; + if (http_client_connection_next_request(conn_avail_idx->conn) <= 0) { + /* No longer available (probably connection error/closed) */ + statistics_dirty = TRUE; + conn_avail_idx->conn = NULL; + } else { + /* Update statistics */ + conn_avail_idx->pending_requests++; + if (num_urgent > 0) + num_urgent--; + num_pending--; + } + } + } + + /* Don't continue unless we have more pending requests */ + num_pending = http_client_peer_requests_pending(peer, &num_urgent); + if (num_pending == 0) { + e_debug(peer->event, + "No more requests to service for this peer " + "(%u connections exist, %u pending)", + array_count(&peer->conns), + array_count(&peer->pending_conns)); + http_client_peer_check_idle(peer); + break; + } + } while (statistics_dirty); + + tmp_peer = peer; + if (!http_client_peer_unref(&tmp_peer)) + return; + peer->handling_requests = FALSE; + + if (num_pending == 0) + return; + + i_assert(idle == 0); + connecting = array_count(&peer->pending_conns); + + /* Determine how many new connections we can set up */ + if (pshared->last_failure.tv_sec > 0 && working_conn_count > 0 && + working_conn_count == connecting) { + /* Don't create new connections until the existing ones have + finished connecting successfully. */ + new_connections = 0; + } else { + if ((working_conn_count - connecting + num_urgent) >= + peer->client->set.max_parallel_connections) { + /* Only create connections for urgent requests */ + new_connections = (num_urgent > connecting ? + num_urgent - connecting : 0); + } else if (num_pending <= connecting) { + /* There are already enough connections being made */ + new_connections = 0; + } else if (working_conn_count == connecting) { + /* No connections succeeded so far, don't hammer the + server with more than one connection attempt unless + its urgent */ + if (num_urgent > 0) { + new_connections = (num_urgent > connecting ? + num_urgent - connecting : 0); + } else { + new_connections = (connecting == 0 ? 1 : 0); + } + } else if ((num_pending - connecting) > + (peer->client->set.max_parallel_connections - + working_conn_count)) { + /* Create maximum allowed connections */ + new_connections = + (peer->client->set.max_parallel_connections - + working_conn_count); + } else { + /* Create as many connections as we need */ + new_connections = num_pending - connecting; + } + } + + /* Create connections */ + if (new_connections > 0) { + e_debug(peer->event, + "Creating %u new connections to handle requests " + "(already %u usable, connecting to %u, closing %u)", + new_connections, working_conn_count - connecting, + connecting, closing); + http_client_peer_connect(peer, new_connections); + return; + } + + /* Cannot create new connections for normal request; attempt pipelining + */ + if ((working_conn_count - connecting) >= + peer->client->set.max_parallel_connections) { + unsigned int pipeline_level = 0, total_handled = 0, handled; + + if (!pshared->allows_pipelining) { + e_debug(peer->event, + "Will not pipeline until peer has shown support"); + return; + } + + /* Fill pipelines */ + do { + handled = 0; + /* Fill smallest pipelines first, + until all pipelines are filled to the same level */ + array_foreach_modifiable(&conns_avail, conn_avail_idx) { + if (conn_avail_idx->conn == NULL) + continue; + if (pipeline_level == 0) { + pipeline_level = conn_avail_idx->pending_requests; + } else if (conn_avail_idx->pending_requests > pipeline_level) { + pipeline_level = conn_avail_idx->pending_requests; + break; /* restart from least busy connection */ + } + /* Pipeline it */ + if (http_client_connection_next_request(conn_avail_idx->conn) <= 0) { + /* connection now unavailable */ + conn_avail_idx->conn = NULL; + } else { + /* Successfully pipelined */ + conn_avail_idx->pending_requests++; + num_pending--; + handled++; + } + } + + total_handled += handled; + } while (num_pending > num_urgent && handled > 0); + + e_debug(peer->event, + "Pipelined %u requests " + "(filled pipelines up to %u requests)", + total_handled, pipeline_level); + return; + } + + /* Still waiting for connections to finish */ + e_debug(peer->event, "No request handled; " + "waiting for pending connections " + "(%u connections exist, %u pending)", + array_count(&peer->conns), connecting); + return; +} + +static void http_client_peer_handle_requests(struct http_client_peer *peer) +{ + timeout_remove(&peer->to_req_handling); + + T_BEGIN { + http_client_peer_handle_requests_real(peer); + } T_END; +} + +void http_client_peer_trigger_request_handler(struct http_client_peer *peer) +{ + /* Trigger request handling through timeout */ + if (peer->to_req_handling == NULL) { + peer->to_req_handling = timeout_add_short_to( + peer->client->ioloop, 0, + http_client_peer_handle_requests, peer); + } +} + +bool http_client_peer_have_queue(struct http_client_peer *peer, + struct http_client_queue *queue) +{ + struct http_client_queue *const *queue_idx; + + array_foreach(&peer->queues, queue_idx) { + if (*queue_idx == queue) + return TRUE; + } + return FALSE; +} + +void http_client_peer_link_queue(struct http_client_peer *peer, + struct http_client_queue *queue) +{ + if (!http_client_peer_have_queue(peer, queue)) { + array_push_back(&peer->queues, &queue); + + e_debug(peer->event, "Linked queue %s (%d queues linked)", + queue->name, array_count(&peer->queues)); + } +} + +void http_client_peer_unlink_queue(struct http_client_peer *peer, + struct http_client_queue *queue) +{ + struct http_client_queue *const *queue_idx; + + array_foreach(&peer->queues, queue_idx) { + if (*queue_idx == queue) { + array_delete(&peer->queues, + array_foreach_idx(&peer->queues, queue_idx), 1); + + e_debug(peer->event, + "Unlinked queue %s (%d queues linked)", + queue->name, array_count(&peer->queues)); + + if (array_count(&peer->queues) == 0) + http_client_peer_check_idle(peer); + return; + } + } +} + +struct http_client_request * +http_client_peer_claim_request(struct http_client_peer *peer, bool no_urgent) +{ + struct http_client_queue *queue; + struct http_client_request *req; + + array_foreach_elem(&peer->queues, queue) { + req = http_client_queue_claim_request( + queue, &peer->shared->addr, no_urgent); + if (req != NULL) { + req->peer = peer; + return req; + } + } + + return NULL; +} + +void http_client_peer_connection_success(struct http_client_peer *peer) +{ + struct http_client_peer_pool *ppool = peer->ppool; + struct http_client_queue *queue; + + e_debug(peer->event, "Successfully connected " + "(%u connections exist, %u pending)", + array_count(&peer->conns), array_count(&peer->pending_conns)); + + http_client_peer_pool_connection_success(ppool); + + array_foreach_elem(&peer->queues, queue) + http_client_queue_connection_success(queue, peer); + + http_client_peer_trigger_request_handler(peer); +} + +void http_client_peer_connection_failure(struct http_client_peer *peer, + const char *reason) +{ + struct http_client_peer_pool *ppool = peer->ppool; + + e_debug(peer->event, "Connection failed " + "(%u connections exist, %u pending)", + array_count(&peer->conns), array_count(&peer->pending_conns)); + + http_client_peer_pool_connection_failure(ppool, reason); + + peer->connect_failed = TRUE; +} + +static void +http_client_peer_connection_succeeded_pool(struct http_client_peer *peer) +{ + if (!peer->connect_failed) + return; + peer->connect_failed = FALSE; + + e_debug(peer->event, + "A connection succeeded within our peer pool, " + "so this peer can retry connecting as well if needed " + "(%u connections exist, %u pending)", + array_count(&peer->conns), array_count(&peer->pending_conns)); + + /* If there are pending requests for this peer, try creating a new + connection for them. if not, this peer will wind itself down. */ + http_client_peer_trigger_request_handler(peer); +} + +static void +http_client_peer_connection_failed_pool(struct http_client_peer *peer, + const char *reason) +{ + struct http_client_queue *queue; + ARRAY_TYPE(http_client_queue) queues; + + e_debug(peer->event, + "Failed to establish any connection within our peer pool: %s " + "(%u connections exist, %u pending)", reason, + array_count(&peer->conns), array_count(&peer->pending_conns)); + + peer->connect_failed = TRUE; + + /* Make a copy of the queue array; queues get linked/unlinged while the + connection failure is handled */ + t_array_init(&queues, array_count(&peer->queues)); + array_copy(&queues.arr, 0, &peer->queues.arr, 0, + array_count(&peer->queues)); + + /* Failed to make any connection. a second connect will probably also + fail, so just try another IP for the hosts(s) or abort all requests + if this was the only/last option. */ + array_foreach_elem(&queues, queue) + http_client_queue_connection_failure(queue, peer, reason); +} + +void http_client_peer_connection_lost(struct http_client_peer *peer, + bool premature) +{ + unsigned int num_pending, num_urgent; + + /* We get here when an already connected connection fails. if the + connect itself fails, http_client_peer_shared_connection_failure() is + called instead. */ + + if (peer->disconnected) + return; + + http_client_peer_shared_connection_lost(peer->shared, premature); + + num_pending = http_client_peer_requests_pending(peer, &num_urgent); + + e_debug(peer->event, + "Lost a connection%s " + "(%u queues linked, %u connections left, " + "%u connections pending, %u requests pending, " + "%u requests urgent)", + (premature ? " prematurely" : ""), + array_count(&peer->queues), array_count(&peer->conns), + array_count(&peer->pending_conns), num_pending, num_urgent); + + if (peer->handling_requests) { + /* We got here from the request handler loop */ + e_debug(peer->event, + "Lost a connection while handling requests"); + return; + } + + /* If there are pending requests for this peer, create a new connection + for them. if not, this peer will wind itself down. */ + http_client_peer_trigger_request_handler(peer); +} + +unsigned int +http_client_peer_active_connections(struct http_client_peer *peer) +{ + struct http_client_connection *conn; + unsigned int active = 0; + + /* Find idle connections */ + array_foreach_elem(&peer->conns, conn) { + if (http_client_connection_is_active(conn)) + active++; + } + + return active; +} + +unsigned int +http_client_peer_pending_connections(struct http_client_peer *peer) +{ + return array_count(&peer->pending_conns); +} + +void http_client_peer_switch_ioloop(struct http_client_peer *peer) +{ + if (peer->to_req_handling != NULL) { + peer->to_req_handling = + io_loop_move_timeout(&peer->to_req_handling); + } +} diff --git a/src/lib-http/http-client-private.h b/src/lib-http/http-client-private.h new file mode 100644 index 0000000..86b1d96 --- /dev/null +++ b/src/lib-http/http-client-private.h @@ -0,0 +1,718 @@ +#ifndef HTTP_CLIENT_PRIVATE_H +#define HTTP_CLIENT_PRIVATE_H + +#include "connection.h" + +#include "http-url.h" +#include "http-client.h" + +/* + * Defaults + */ + +#define HTTP_CLIENT_CONTINUE_TIMEOUT_MSECS (1000*2) +#define HTTP_CLIENT_DEFAULT_REQUEST_TIMEOUT_MSECS (1000*60*1) +#define HTTP_CLIENT_DEFAULT_DNS_LOOKUP_TIMEOUT_MSECS (1000*10) +#define HTTP_CLIENT_DEFAULT_BACKOFF_TIME_MSECS (100) +#define HTTP_CLIENT_DEFAULT_BACKOFF_MAX_TIME_MSECS (1000*60) +#define HTTP_CLIENT_DEFAULT_DNS_TTL_MSECS (1000*60*30) +#define HTTP_CLIENT_MIN_IDLE_TIMEOUT_MSECS 50 + +/* + * Types + */ + +struct http_client_connection; +struct http_client_peer_pool; +struct http_client_peer_shared; +struct http_client_peer; +struct http_client_queue; +struct http_client_host_shared; +struct http_client_host; + +ARRAY_DEFINE_TYPE(http_client_request, struct http_client_request *); +ARRAY_DEFINE_TYPE(http_client_connection, struct http_client_connection *); +ARRAY_DEFINE_TYPE(http_client_peer, struct http_client_peer *); +ARRAY_DEFINE_TYPE(http_client_peer_shared, struct http_client_peer_shared *); +ARRAY_DEFINE_TYPE(http_client_peer_pool, struct http_client_peer_pool *); +ARRAY_DEFINE_TYPE(http_client_queue, struct http_client_queue *); +ARRAY_DEFINE_TYPE(http_client_host, struct http_client_host_shared *); + +HASH_TABLE_DEFINE_TYPE(http_client_peer_shared, + const struct http_client_peer_addr *, + struct http_client_peer_shared *); +HASH_TABLE_DEFINE_TYPE(http_client_host_shared, const char *, + struct http_client_host_shared *); + +enum http_client_peer_addr_type { + HTTP_CLIENT_PEER_ADDR_HTTP = 0, + HTTP_CLIENT_PEER_ADDR_HTTPS, + HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL, + HTTP_CLIENT_PEER_ADDR_RAW, + HTTP_CLIENT_PEER_ADDR_UNIX, +}; + +struct http_client_peer_addr { + enum http_client_peer_addr_type type; + union { + struct { + const char *https_name; /* TLS SNI */ + struct ip_addr ip; + in_port_t port; + } tcp; + struct { + const char *path; + } un; + } a; +}; + +/* + * Objects + */ + +struct http_client_request { + pool_t pool; + unsigned int refcount; + const char *label; + unsigned int id; + + struct http_client_request *prev, *next; + + const char *method, *target; + struct http_url origin_url; + const char *username, *password; + + const char *host_socket; + const struct http_url *host_url; + const char *authority; + + struct http_client *client; + struct http_client_host *host; + struct http_client_queue *queue; + struct http_client_peer *peer; + struct http_client_connection *conn; + + struct event *event; + const char *const *event_headers; + unsigned int last_status; + + string_t *headers; + time_t date; + + struct istream *payload_input; + uoff_t payload_size, payload_offset; + struct ostream *payload_output; + + /* Time when request can be sent the next time. This is set by + http_client_request_delay*(). Default is 0 = immediately. Retries + can update this. */ + struct timeval release_time; + /* Time when http_client_request_submit() was called. */ + struct timeval submit_time; + /* Time when the request was first sent to the server. The HTTP + connection already exists at this time. */ + struct timeval first_sent_time; + /* Time when the request was last sent to the server (if it was + retried). */ + struct timeval sent_time; + /* Time when the HTTP response header was last received. */ + struct timeval response_time; + /* Time when the request will be aborted. Set by + http_client_request_set_timeout(). */ + struct timeval timeout_time; + unsigned int timeout_msecs; + unsigned int attempt_timeout_msecs; + unsigned int max_attempts; + + uoff_t response_offset, request_offset; + uoff_t bytes_in, bytes_out; + + unsigned int attempts, send_attempts; + unsigned int redirects; + uint64_t sent_global_ioloop_usecs; + uint64_t sent_http_ioloop_usecs; + uint64_t sent_lock_usecs; + + unsigned int delayed_error_status; + const char *delayed_error; + + http_client_request_callback_t *callback; + void *context; + + void (*destroy_callback)(void *); + void *destroy_context; + + enum http_request_state state; + + bool have_hdr_authorization:1; + bool have_hdr_body_spec:1; + bool have_hdr_connection:1; + bool have_hdr_date:1; + bool have_hdr_expect:1; + bool have_hdr_host:1; + bool have_hdr_user_agent:1; + + bool payload_sync:1; + bool payload_sync_continue:1; + bool payload_chunked:1; + bool payload_wait:1; + bool payload_finished:1; + bool payload_empty:1; + bool urgent:1; + bool submitted:1; + bool listed:1; + bool connect_tunnel:1; + bool connect_direct:1; + bool ssl_tunnel:1; + bool preserve_exact_reason:1; +}; + +struct http_client_connection { + struct connection conn; + struct event *event; + unsigned int refcount; + + struct http_client_peer_pool *ppool; + struct http_client_peer *peer; + + int connect_errno; + struct timeval connect_start_timestamp; + struct timeval connected_timestamp; + struct http_client_request *connect_request; + + struct ssl_iostream *ssl_iostream; + struct http_response_parser *http_parser; + struct timeout *to_connect, *to_input, *to_idle, *to_response; + struct timeout *to_requests; + + struct http_client_request *pending_request; + struct istream *incoming_payload; + struct io *io_req_payload; + struct ioloop *last_ioloop; + struct io_wait_timer *io_wait_timer; + + /* Requests that have been sent, waiting for response */ + ARRAY_TYPE(http_client_request) request_wait_list; + + bool connected:1; /* Connection is connected */ + bool idle:1; /* Connection is idle */ + bool tunneling:1; /* Last sent request turns this + connection into tunnel */ + bool connect_succeeded:1; /* Connection succeeded including SSL */ + bool connect_failed:1; /* Connection failed */ + bool lost_prematurely:1; /* Lost connection before receiving any data */ + bool closing:1; + bool disconnected:1; + bool close_indicated:1; + bool output_locked:1; /* Output is locked; no pipelining */ + bool output_broken:1; /* Output is broken; no more requests */ + bool in_req_callback:1; /* Performing request callback (busy) */ + bool debug:1; +}; + +struct http_client_peer_shared { + unsigned int refcount; + struct http_client_peer_addr addr; + char *addr_name; + struct event *event; + + char *label; + + struct http_client_context *cctx; + struct http_client_peer_shared *prev, *next; + + struct http_client_peer_pool *pools_list; + + struct http_client_peer *peers_list; + unsigned int peers_count; + + /* Connection retry */ + struct timeval last_failure; + struct timeout *to_backoff; + unsigned int backoff_initial_time_msecs; + unsigned int backoff_current_time_msecs; + unsigned int backoff_max_time_msecs; + + bool no_payload_sync:1; /* Expect: 100-continue failed before */ + bool seen_100_response:1; /* Expect: 100-continue succeeded before */ + bool allows_pipelining:1; /* Peer is known to allow persistent + connections */ +}; + +struct http_client_peer_pool { + unsigned int refcount; + struct http_client_peer_shared *peer; + struct http_client_peer_pool *prev, *next; + struct event *event; + + /* All connections to this peer */ + ARRAY_TYPE(http_client_connection) conns; + + /* Pending connections (not ready connecting) */ + ARRAY_TYPE(http_client_connection) pending_conns; + + /* Available connections to this peer */ + ARRAY_TYPE(http_client_connection) idle_conns; + + /* Distinguishing settings for these connections */ + struct ssl_iostream_context *ssl_ctx; + char *rawlog_dir; + struct pcap_output *pcap_output; + + bool destroyed:1; /* Peer pool is being destroyed */ +}; + +struct http_client_peer { + unsigned int refcount; + struct http_client_peer_shared *shared; + struct http_client_peer *shared_prev, *shared_next; + + struct http_client *client; + struct http_client_peer *client_prev, *client_next; + + struct http_client_peer_pool *ppool; + struct event *event; + + /* Queues using this peer */ + ARRAY_TYPE(http_client_queue) queues; + + /* Active connections to this peer */ + ARRAY_TYPE(http_client_connection) conns; + /* Pending connections (not ready connecting) */ + ARRAY_TYPE(http_client_connection) pending_conns; + + /* Zero time-out for consolidating request handling */ + struct timeout *to_req_handling; + + bool connect_failed:1; /* Last connection attempt failed */ + bool connect_backoff:1; /* Peer is waiting for backoff timout*/ + bool disconnected:1; /* Peer is already disconnected */ + bool handling_requests:1; /* Currently running request handler */ +}; + +struct http_client_queue { + struct http_client *client; + struct http_client_queue *prev, *next; + + struct http_client_host *host; + char *name; + struct event *event; + + struct http_client_peer_addr addr; + char *addr_name; + + /* Current index in host->ips */ + unsigned int ips_connect_idx; + /* The first IP that started the current round of connection attempts. + initially 0, and later set to the ip index of the last successful + connected IP */ + unsigned int ips_connect_start_idx; + + struct timeval first_connect_time; + unsigned int connect_attempts; + + /* Peers we are trying to connect to; + this can be more than one when soft connect timeouts are enabled */ + ARRAY_TYPE(http_client_peer) pending_peers; + + /* Currently active peer */ + struct http_client_peer *cur_peer; + + /* All requests associated to this queue + (ordered by earliest timeout first) */ + ARRAY_TYPE(http_client_request) requests; + + /* Delayed requests waiting to be released after delay */ + ARRAY_TYPE(http_client_request) delayed_requests; + + /* Requests pending in queue to be picked up by connections */ + ARRAY_TYPE(http_client_request) queued_requests, queued_urgent_requests; + + struct timeout *to_connect, *to_request, *to_delayed; +}; + +struct http_client_host_shared { + struct http_client_host_shared *prev, *next; + + struct http_client_context *cctx; + char *name; + struct event *event; + + /* The ip addresses DNS returned for this host */ + unsigned int ips_count; + struct ip_addr *ips; + struct timeval ips_timeout; + + /* Private instance for each client that uses this host */ + struct http_client_host *hosts_list; + + /* Active DNS lookup */ + struct dns_lookup *dns_lookup; + + /* Timeouts */ + struct timeout *to_idle; + + bool destroyed:1; /* Shared host object is being destroyed */ + bool unix_local:1; + bool explicit_ip:1; +}; + +struct http_client_host { + struct http_client_host_shared *shared; + struct http_client_host *shared_prev, *shared_next; + + struct http_client *client; + struct http_client_host *client_prev, *client_next; + + /* Separate queue for each host port */ + ARRAY_TYPE(http_client_queue) queues; +}; + +struct http_client { + pool_t pool; + struct http_client_context *cctx; + struct http_client_settings set; + + struct http_client *prev, *next; + + struct event *event; + struct ioloop *ioloop; + struct ssl_iostream_context *ssl_ctx; + + /* List of failed requests that are waiting for ioloop */ + ARRAY(struct http_client_request *) delayed_failing_requests; + struct timeout *to_failing_requests; + + struct http_client_host *hosts_list; + struct http_client_peer *peers_list; + + struct http_client_request *requests_list; + unsigned int requests_count; + + bool waiting:1; +}; + +struct http_client_context { + pool_t pool; + unsigned int refcount; + struct event *event; + struct ioloop *ioloop; + + struct http_client_settings set; + + struct dns_client *dns_client; + const char *dns_client_socket_path; + unsigned int dns_ttl_msecs; + unsigned int dns_lookup_timeout_msecs; + + struct http_client *clients_list; + struct connection_list *conn_list; + + HASH_TABLE_TYPE(http_client_peer_shared) peers; + struct http_client_peer_shared *peers_list; + HASH_TABLE_TYPE(http_client_host_shared) hosts; + struct http_client_host_shared *unix_host; + struct http_client_host_shared *hosts_list; +}; + +/* + * Peer address + */ + +static inline bool +http_client_peer_addr_is_https(const struct http_client_peer_addr *addr) +{ + switch (addr->type) { + case HTTP_CLIENT_PEER_ADDR_HTTPS: + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + return TRUE; + default: + break; + } + return FALSE; +} + +static inline const char * +http_client_peer_addr_get_https_name(const struct http_client_peer_addr *addr) +{ + switch (addr->type) { + case HTTP_CLIENT_PEER_ADDR_HTTPS: + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + return addr->a.tcp.https_name; + default: + break; + } + return NULL; +} + +static inline const char * +http_client_peer_addr2str(const struct http_client_peer_addr *addr) +{ + switch (addr->type) { + case HTTP_CLIENT_PEER_ADDR_HTTP: + case HTTP_CLIENT_PEER_ADDR_HTTPS: + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + case HTTP_CLIENT_PEER_ADDR_RAW: + if (addr->a.tcp.ip.family == AF_INET6) { + return t_strdup_printf("[%s]:%u", + net_ip2addr(&addr->a.tcp.ip), + addr->a.tcp.port); + } + return t_strdup_printf("%s:%u", + net_ip2addr(&addr->a.tcp.ip), + addr->a.tcp.port); + case HTTP_CLIENT_PEER_ADDR_UNIX: + return t_strdup_printf("unix:%s", addr->a.un.path); + default: + break; + } + i_unreached(); + return ""; +} + +/* + * Request + */ + +static inline bool +http_client_request_to_proxy(const struct http_client_request *req) +{ + return (req->host_url != &req->origin_url); +} + +const char *http_client_request_label(struct http_client_request *req); + +void http_client_request_ref(struct http_client_request *req); +/* Returns FALSE if unrefing destroyed the request entirely */ +bool http_client_request_unref(struct http_client_request **_req); +void http_client_request_destroy(struct http_client_request **_req); + +void http_client_request_get_peer_addr(const struct http_client_request *req, + struct http_client_peer_addr *addr); +enum http_response_payload_type +http_client_request_get_payload_type(struct http_client_request *req); +int http_client_request_send(struct http_client_request *req, bool pipelined); +int http_client_request_send_more(struct http_client_request *req, + bool pipelined); + +bool http_client_request_callback(struct http_client_request *req, + struct http_response *response); +void http_client_request_connect_callback(struct http_client_request *req, + const struct http_client_tunnel *tunnel, + struct http_response *response); + +void http_client_request_resubmit(struct http_client_request *req); +void http_client_request_retry(struct http_client_request *req, + unsigned int status, const char *error); +void http_client_request_error_delayed(struct http_client_request **_req); +void http_client_request_error(struct http_client_request **req, + unsigned int status, const char *error); +void http_client_request_redirect(struct http_client_request *req, + unsigned int status, const char *location); +void http_client_request_finish(struct http_client_request *req); + +/* + * Connection + */ + +struct connection_list *http_client_connection_list_init(void); + +struct http_client_connection * +http_client_connection_create(struct http_client_peer *peer); +void http_client_connection_ref(struct http_client_connection *conn); +/* Returns FALSE if unrefing destroyed the connection entirely */ +bool http_client_connection_unref(struct http_client_connection **_conn); +void http_client_connection_close(struct http_client_connection **_conn); + +void http_client_connection_lost(struct http_client_connection **_conn, + const char *error) ATTR_NULL(2); + +void http_client_connection_peer_closed(struct http_client_connection **_conn); +void http_client_connection_request_destroyed( + struct http_client_connection *conn, struct http_client_request *req); + +void http_client_connection_handle_output_error( + struct http_client_connection *conn); +int http_client_connection_output(struct http_client_connection *conn); +void http_client_connection_start_request_timeout( + struct http_client_connection *conn); +void http_client_connection_reset_request_timeout( + struct http_client_connection *conn); +void http_client_connection_stop_request_timeout( + struct http_client_connection *conn); +unsigned int +http_client_connection_count_pending(struct http_client_connection *conn); +int http_client_connection_check_ready(struct http_client_connection *conn); +bool http_client_connection_is_idle(struct http_client_connection *conn); +bool http_client_connection_is_active(struct http_client_connection *conn); +int http_client_connection_next_request(struct http_client_connection *conn); +void http_client_connection_check_idle(struct http_client_connection *conn); +void http_client_connection_switch_ioloop(struct http_client_connection *conn); +void http_client_connection_start_tunnel(struct http_client_connection **_conn, + struct http_client_tunnel *tunnel); +void http_client_connection_lost_peer(struct http_client_connection *conn); +void http_client_connection_claim_idle(struct http_client_connection *conn, + struct http_client_peer *peer); + +/* + * Peer + */ + +/* address */ + +unsigned int +http_client_peer_addr_hash(const struct http_client_peer_addr *peer) ATTR_PURE; +int http_client_peer_addr_cmp(const struct http_client_peer_addr *peer1, + const struct http_client_peer_addr *peer2) + ATTR_PURE; + +/* connection pool */ + +void http_client_peer_pool_ref(struct http_client_peer_pool *ppool); +void http_client_peer_pool_unref(struct http_client_peer_pool **_ppool); + +void http_client_peer_pool_close(struct http_client_peer_pool **_ppool); + +/* peer (shared) */ + +const char * +http_client_peer_shared_label(struct http_client_peer_shared *pshared); + +void http_client_peer_shared_ref(struct http_client_peer_shared *pshared); +void http_client_peer_shared_unref(struct http_client_peer_shared **_pshared); +void http_client_peer_shared_close(struct http_client_peer_shared **_pshared); + +void http_client_peer_shared_switch_ioloop( + struct http_client_peer_shared *pshared); + +unsigned int +http_client_peer_shared_max_connections( + struct http_client_peer_shared *pshared); + +/* peer */ + +struct http_client_peer * +http_client_peer_get(struct http_client *client, + const struct http_client_peer_addr *addr); +void http_client_peer_ref(struct http_client_peer *peer); +bool http_client_peer_unref(struct http_client_peer **_peer); +void http_client_peer_close(struct http_client_peer **_peer); + +bool http_client_peer_have_queue(struct http_client_peer *peer, + struct http_client_queue *queue); +void http_client_peer_link_queue(struct http_client_peer *peer, + struct http_client_queue *queue); +void http_client_peer_unlink_queue(struct http_client_peer *peer, + struct http_client_queue *queue); +struct http_client_request * +http_client_peer_claim_request(struct http_client_peer *peer, bool no_urgent); +void http_client_peer_trigger_request_handler(struct http_client_peer *peer); +void http_client_peer_connection_success(struct http_client_peer *peer); +void http_client_peer_connection_failure(struct http_client_peer *peer, + const char *reason); +void http_client_peer_connection_lost(struct http_client_peer *peer, + bool premature); +bool http_client_peer_is_connected(struct http_client_peer *peer); +unsigned int +http_client_peer_idle_connections(struct http_client_peer *peer); +unsigned int +http_client_peer_active_connections(struct http_client_peer *peer); +unsigned int +http_client_peer_pending_connections(struct http_client_peer *peer); +void http_client_peer_switch_ioloop(struct http_client_peer *peer); + +/* + * Queue + */ + +struct http_client_queue * +http_client_queue_get(struct http_client_host *host, + const struct http_client_peer_addr *addr); +void http_client_queue_free(struct http_client_queue *queue); +void http_client_queue_connection_setup(struct http_client_queue *queue); +unsigned int +http_client_queue_host_lookup_done(struct http_client_queue *queue); +void http_client_queue_host_lookup_failure(struct http_client_queue *queue, + const char *error); +void http_client_queue_submit_request(struct http_client_queue *queue, + struct http_client_request *req); +void http_client_queue_drop_request(struct http_client_queue *queue, + struct http_client_request *req); +struct http_client_request * +http_client_queue_claim_request(struct http_client_queue *queue, + const struct http_client_peer_addr *addr, + bool no_urgent); +unsigned int +http_client_queue_requests_pending(struct http_client_queue *queue, + unsigned int *num_urgent_r) ATTR_NULL(2); +unsigned int http_client_queue_requests_active(struct http_client_queue *queue); +void http_client_queue_connection_success(struct http_client_queue *queue, + struct http_client_peer *peer); +void http_client_queue_connection_failure(struct http_client_queue *queue, + struct http_client_peer *peer, + const char *reason); +void http_client_queue_peer_disconnected(struct http_client_queue *queue, + struct http_client_peer *peer); +void http_client_queue_switch_ioloop(struct http_client_queue *queue); + +/* + * Host + */ + +/* host (shared) */ + +void http_client_host_shared_free(struct http_client_host_shared **_hshared); +void http_client_host_shared_switch_ioloop( + struct http_client_host_shared *hshared); + +/* host */ + +static inline unsigned int +http_client_host_get_ips_count(struct http_client_host *host) +{ + return host->shared->ips_count; +} + +static inline const struct ip_addr * +http_client_host_get_ip(struct http_client_host *host, unsigned int idx) +{ + i_assert(idx < host->shared->ips_count); + return &host->shared->ips[idx]; +} + +static inline bool +http_client_host_ready(struct http_client_host *host) +{ + return host->shared->dns_lookup == NULL; +} + +struct http_client_host * +http_client_host_get(struct http_client *client, + const struct http_url *host_url); +void http_client_host_free(struct http_client_host **_host); +void http_client_host_submit_request(struct http_client_host *host, + struct http_client_request *req); +void http_client_host_switch_ioloop(struct http_client_host *host); +void http_client_host_check_idle(struct http_client_host *host); +int http_client_host_refresh(struct http_client_host *host); +bool http_client_host_get_ip_idx(struct http_client_host *host, + const struct ip_addr *ip, unsigned int *idx_r); + +/* + * Client + */ + +int http_client_init_ssl_ctx(struct http_client *client, const char **error_r); + +void http_client_delay_request_error(struct http_client *client, + struct http_client_request *req); +void http_client_remove_request_error(struct http_client *client, + struct http_client_request *req); + +/* + * Client shared context + */ + +void http_client_context_switch_ioloop(struct http_client_context *cctx); + +#endif diff --git a/src/lib-http/http-client-queue.c b/src/lib-http/http-client-queue.c new file mode 100644 index 0000000..5c7915a --- /dev/null +++ b/src/lib-http/http-client-queue.c @@ -0,0 +1,1065 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "net.h" +#include "str.h" +#include "str-sanitize.h" +#include "hash.h" +#include "array.h" +#include "bsearch-insert-pos.h" +#include "llist.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "time-util.h" +#include "dns-lookup.h" +#include "http-response-parser.h" + +#include "http-client-private.h" + +#define TIMEOUT_CMP_MARGIN_USECS 2000 + +static void +http_client_queue_fail_full(struct http_client_queue *queue, + unsigned int status, const char *error, bool all); +static void +http_client_queue_set_delay_timer(struct http_client_queue *queue, + struct timeval time); +static void +http_client_queue_set_request_timer(struct http_client_queue *queue, + const struct timeval *time); + +/* + * Queue object + */ + +static struct http_client_queue * +http_client_queue_find(struct http_client_host *host, + const struct http_client_peer_addr *addr) +{ + struct http_client_queue *queue; + + array_foreach_elem(&host->queues, queue) { + if (http_client_peer_addr_cmp(&queue->addr, addr) == 0) + return queue; + } + + return NULL; +} + +static struct http_client_queue * +http_client_queue_create(struct http_client_host *host, + const struct http_client_peer_addr *addr) +{ + const char *hostname = host->shared->name; + struct http_client_queue *queue; + + queue = i_new(struct http_client_queue, 1); + queue->client = host->client; + queue->host = host; + queue->addr = *addr; + + switch (addr->type) { + case HTTP_CLIENT_PEER_ADDR_RAW: + queue->name = i_strdup_printf("raw://%s:%u", + hostname, addr->a.tcp.port); + queue->addr.a.tcp.https_name = NULL; + break; + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + case HTTP_CLIENT_PEER_ADDR_HTTPS: + queue->name = i_strdup_printf("https://%s:%u", + hostname, addr->a.tcp.port); + queue->addr_name = i_strdup(addr->a.tcp.https_name); + queue->addr.a.tcp.https_name = queue->addr_name; + break; + case HTTP_CLIENT_PEER_ADDR_HTTP: + queue->name = i_strdup_printf("http://%s:%u", + hostname, addr->a.tcp.port); + queue->addr.a.tcp.https_name = NULL; + break; + case HTTP_CLIENT_PEER_ADDR_UNIX: + queue->name = i_strdup_printf("unix:%s", addr->a.un.path); + queue->addr_name = i_strdup(addr->a.un.path); + queue->addr.a.un.path = queue->addr_name; + break; + default: + i_unreached(); + } + + queue->event = event_create(queue->client->event); + event_set_append_log_prefix(queue->event, + t_strdup_printf("queue %s: ", str_sanitize(queue->name, 256))); + queue->ips_connect_idx = 0; + i_array_init(&queue->pending_peers, 8); + i_array_init(&queue->requests, 16); + i_array_init(&queue->queued_requests, 16); + i_array_init(&queue->queued_urgent_requests, 16); + i_array_init(&queue->delayed_requests, 4); + array_push_back(&host->queues, &queue); + + return queue; +} + +struct http_client_queue * +http_client_queue_get(struct http_client_host *host, + const struct http_client_peer_addr *addr) +{ + struct http_client_queue *queue; + + queue = http_client_queue_find(host, addr); + if (queue == NULL) + queue = http_client_queue_create(host, addr); + + return queue; +} + +void http_client_queue_free(struct http_client_queue *queue) +{ + struct http_client_peer *peer; + ARRAY_TYPE(http_client_peer) peers; + + e_debug(queue->event, "Destroy"); + + /* Currently only called when peer is freed, so there is no need to + unlink from the peer */ + + /* Unlink all peers */ + if (queue->cur_peer != NULL) { + struct http_client_peer *peer = queue->cur_peer; + + queue->cur_peer = NULL; + http_client_peer_unlink_queue(peer, queue); + } + t_array_init(&peers, array_count(&queue->pending_peers)); + array_copy(&peers.arr, 0, &queue->pending_peers.arr, 0, + array_count(&queue->pending_peers)); + array_foreach_elem(&peers, peer) + http_client_peer_unlink_queue(peer, queue); + array_free(&queue->pending_peers); + + /* Abort all requests */ + http_client_queue_fail_full(queue, HTTP_CLIENT_REQUEST_ERROR_ABORTED, + "Aborted", TRUE); + array_free(&queue->requests); + array_free(&queue->queued_requests); + array_free(&queue->queued_urgent_requests); + array_free(&queue->delayed_requests); + + /* Cancel timeouts */ + timeout_remove(&queue->to_connect); + timeout_remove(&queue->to_delayed); + + /* Free */ + event_unref(&queue->event); + i_free(queue->addr_name); + i_free(queue->name); + i_free(queue); +} + +/* + * Error handling + */ + +static void +http_client_queue_fail_full(struct http_client_queue *queue, + unsigned int status, const char *error, bool all) +{ + ARRAY_TYPE(http_client_request) *req_arr, treqs; + struct http_client_request *req; + unsigned int retained = 0; + + /* Abort requests */ + req_arr = &queue->requests; + t_array_init(&treqs, array_count(req_arr)); + array_copy(&treqs.arr, 0, &req_arr->arr, 0, array_count(req_arr)); + array_foreach_elem(&treqs, req) { + i_assert(req->state >= HTTP_REQUEST_STATE_QUEUED); + if (!all && + req->state != HTTP_REQUEST_STATE_QUEUED) + retained++; + else + http_client_request_error(&req, status, error); + } + + /* All queues should be empty now... unless new requests were submitted + from the callback. this invariant captures it all: */ + i_assert((retained + + array_count(&queue->delayed_requests) + + array_count(&queue->queued_requests) + + array_count(&queue->queued_urgent_requests)) == + array_count(&queue->requests)); +} + +static void +http_client_queue_fail(struct http_client_queue *queue, + unsigned int status, const char *error) +{ + http_client_queue_fail_full(queue, status, error, FALSE); +} + +/* + * Connection management + */ + +static bool +http_client_queue_is_last_connect_ip(struct http_client_queue *queue) +{ + const struct http_client_settings *set = + &queue->client->set; + struct http_client_host *host = queue->host; + unsigned int ips_count = http_client_host_get_ips_count(host); + + i_assert(queue->addr.type != HTTP_CLIENT_PEER_ADDR_UNIX); + i_assert(queue->ips_connect_idx < ips_count); + i_assert(queue->ips_connect_start_idx < ips_count); + + /* If a maximum connect attempts > 1 is set, enforce it directly */ + if (set->max_connect_attempts > 1 && + queue->connect_attempts >= set->max_connect_attempts) + return TRUE; + + /* Otherwise, we'll always go through all the IPs. we don't necessarily + start connecting from the first IP, so we'll need to treat the IPs as + a ring buffer where we automatically wrap back to the first IP + when necessary. */ + return ((queue->ips_connect_idx + 1) % ips_count == + queue->ips_connect_start_idx); +} + +static void +http_client_queue_recover_from_lookup(struct http_client_queue *queue) +{ + struct http_client_host *host = queue->host; + unsigned int ip_idx; + + i_assert(queue->addr.type != HTTP_CLIENT_PEER_ADDR_UNIX); + + if (queue->cur_peer == NULL) { + queue->ips_connect_idx = queue->ips_connect_start_idx = 0; + return; + } + + if (http_client_host_get_ip_idx( + host, &queue->cur_peer->shared->addr.a.tcp.ip, &ip_idx)) { + /* Continue with current peer */ + queue->ips_connect_idx = queue->ips_connect_start_idx = ip_idx; + } else { + /* Reset connect attempts */ + queue->ips_connect_idx = queue->ips_connect_start_idx = 0; + } +} + +static void +http_client_queue_soft_connect_timeout(struct http_client_queue *queue) +{ + struct http_client_host *host = queue->host; + const struct http_client_peer_addr *addr = &queue->addr; + unsigned int ips_count = http_client_host_get_ips_count(host); + const char *https_name; + + i_assert(queue->addr.type != HTTP_CLIENT_PEER_ADDR_UNIX); + + timeout_remove(&queue->to_connect); + + if (http_client_queue_is_last_connect_ip(queue)) { + /* No more IPs to try */ + return; + } + + /* If our our previous connection attempt takes longer than the + soft_connect_timeout, we start a connection attempt to the next IP in + parallel */ + https_name = http_client_peer_addr_get_https_name(addr); + e_debug(queue->event, "Connection to %s%s is taking a long time; " + "starting parallel connection attempt to next IP", + http_client_peer_addr2str(addr), + (https_name == NULL ? + "" : t_strdup_printf(" (SSL=%s)", https_name))); + + /* Next IP */ + queue->ips_connect_idx = (queue->ips_connect_idx + 1) % ips_count; + + /* Setup connection to new peer (can start new soft timeout) */ + http_client_queue_connection_setup(queue); +} + +static struct http_client_peer * +http_client_queue_connection_attempt(struct http_client_queue *queue) +{ + struct http_client *client = queue->client; + struct http_client_host *host = queue->host; + struct http_client_peer *peer; + struct http_client_peer_addr *addr = &queue->addr; + unsigned int num_requests = + array_count(&queue->queued_requests) + + array_count(&queue->queued_urgent_requests); + const char *ssl = ""; + int ret; + + if (num_requests == 0) + return NULL; + + /* Check whether host IPs are still up-to-date */ + ret = http_client_host_refresh(host); + if (ret < 0) { + /* Performing asynchronous lookup */ + timeout_remove(&queue->to_connect); + return NULL; + } + if (ret > 0) { + /* New lookup performed */ + http_client_queue_recover_from_lookup(queue); + } + + /* Update our peer address */ + if (queue->addr.type != HTTP_CLIENT_PEER_ADDR_UNIX) { + const struct ip_addr *ip = + http_client_host_get_ip(host, queue->ips_connect_idx); + + queue->addr.a.tcp.ip = *ip; + ssl = http_client_peer_addr_get_https_name(addr); + ssl = (ssl == NULL ? "" : t_strdup_printf(" (SSL=%s)", ssl)); + } + + /* Already got a peer? */ + peer = NULL; + if (queue->cur_peer != NULL) { + i_assert(array_count(&queue->pending_peers) == 0); + + /* Is it still the one we want? */ + if (http_client_peer_addr_cmp( + addr, &queue->cur_peer->shared->addr) == 0) { + /* Is it still connected? */ + if (http_client_peer_is_connected(queue->cur_peer)) { + /* Yes */ + e_debug(queue->event, + "Using existing connection to %s%s " + "(%u requests pending)", + http_client_peer_addr2str(addr), + ssl, num_requests); + + /* Handle requests; */ + http_client_peer_trigger_request_handler( + queue->cur_peer); + return queue->cur_peer; + } + /* No */ + peer = queue->cur_peer; + } else { + /* Peer is not relevant to this queue anymore */ + http_client_peer_unlink_queue(queue->cur_peer, queue); + } + + queue->cur_peer = NULL; + } + + if (peer == NULL) + peer = http_client_peer_get(queue->client, addr); + + e_debug(queue->event, + "Setting up connection to %s%s (%u requests pending)", + http_client_peer_addr2str(addr), ssl, num_requests); + + /* Create provisional link between queue and peer */ + http_client_peer_link_queue(peer, queue); + + /* Handle requests; creates new connections when needed/possible */ + http_client_peer_trigger_request_handler(peer); + + if (http_client_peer_is_connected(peer)) { + /* Drop any pending peers */ + if (array_count(&queue->pending_peers) > 0) { + struct http_client_peer *pending_peer; + + array_foreach_elem(&queue->pending_peers, pending_peer) { + if (pending_peer == peer) { + /* This can happen with shared clients + */ + continue; + } + i_assert(http_client_peer_addr_cmp( + &pending_peer->shared->addr, addr) != 0); + http_client_peer_unlink_queue(pending_peer, queue); + } + array_clear(&queue->pending_peers); + } + queue->cur_peer = peer; + + http_client_peer_trigger_request_handler(queue->cur_peer); + + } else { + struct http_client_peer *pending_peer; + unsigned int msecs; + bool new_peer = TRUE; + + /* Not already connected, wait for connections */ + + /* We may be waiting for this peer already */ + array_foreach_elem(&queue->pending_peers, pending_peer) { + if (http_client_peer_addr_cmp( + &pending_peer->shared->addr, addr) == 0) { + i_assert(pending_peer == peer); + new_peer = FALSE; + break; + } + } + if (new_peer) { + e_debug(queue->event, "Started new connection to %s%s", + http_client_peer_addr2str(addr), ssl); + + array_push_back(&queue->pending_peers, &peer); + if (queue->connect_attempts++ == 0) + queue->first_connect_time = ioloop_timeval; + } + + /* Start soft connect time-out + (but only if we have another IP left) */ + if (queue->addr.type != HTTP_CLIENT_PEER_ADDR_UNIX) { + msecs = client->set.soft_connect_timeout_msecs; + if (!http_client_queue_is_last_connect_ip(queue) && + msecs > 0 && queue->to_connect == NULL) { + queue->to_connect = timeout_add_to( + client->ioloop, msecs, + http_client_queue_soft_connect_timeout, + queue); + } + } + } + + return peer; +} + +void http_client_queue_connection_setup(struct http_client_queue *queue) +{ + (void)http_client_queue_connection_attempt(queue); +} + +unsigned int +http_client_queue_host_lookup_done(struct http_client_queue *queue) +{ + unsigned int reqs_pending = + http_client_queue_requests_pending(queue, NULL); + + http_client_queue_recover_from_lookup(queue); + if (reqs_pending > 0) + http_client_queue_connection_setup(queue); + return reqs_pending; +} + +void http_client_queue_host_lookup_failure( + struct http_client_queue *queue, const char *error) +{ + http_client_queue_fail( + queue, HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED, error); +} + +void http_client_queue_connection_success(struct http_client_queue *queue, + struct http_client_peer *peer) +{ + const struct http_client_peer_addr *addr = &peer->shared->addr; + struct http_client_host *host = queue->host; + + if (http_client_host_ready(host) && + queue->addr.type != HTTP_CLIENT_PEER_ADDR_UNIX) { + /* We achieved at least one connection the the addr->ip */ + if (!http_client_host_get_ip_idx( + host, &addr->a.tcp.ip, &queue->ips_connect_start_idx)) { + /* list of IPs changed during connect */ + queue->ips_connect_start_idx = 0; + } + } + + /* Reset attempt counter */ + queue->connect_attempts = 0; + + /* stop soft connect time-out */ + timeout_remove(&queue->to_connect); + + /* Drop all other attempts to the hport. note that we get here whenever + a connection is successfully created, so pending_peers array + may be empty. */ + if (array_count(&queue->pending_peers) > 0) { + struct http_client_peer *pending_peer; + + array_foreach_elem(&queue->pending_peers, pending_peer) { + if (pending_peer == peer) { + /* Don't drop any connections to the + successfully connected peer, even if some of + the connections are pending. they may be + intended for urgent requests. */ + i_assert(queue->cur_peer == NULL); + queue->cur_peer = pending_peer; + continue; + } + /* Unlink this queue from the peer; if this was the + last/only queue, the peer will be freed, closing all + connections. + */ + http_client_peer_unlink_queue(pending_peer, queue); + } + + array_clear(&queue->pending_peers); + i_assert(queue->cur_peer != NULL); + } +} + +void http_client_queue_connection_failure(struct http_client_queue *queue, + struct http_client_peer *peer, + const char *reason) +{ + const struct http_client_settings *set = + &queue->client->set; + const struct http_client_peer_addr *addr = &peer->shared->addr; + const char *https_name = http_client_peer_addr_get_https_name(addr); + struct http_client_host *host = queue->host; + unsigned int ips_count = http_client_host_get_ips_count(host); + struct http_client_peer *const *peer_idx; + unsigned int num_requests = + array_count(&queue->queued_requests) + + array_count(&queue->queued_urgent_requests); + + e_debug(queue->event, + "Failed to set up connection to %s%s: %s " + "(%u peers pending, %u requests pending)", + http_client_peer_addr2str(addr), + (https_name == NULL ? + "" : t_strdup_printf(" (SSL=%s)", https_name)), + reason, array_count(&queue->pending_peers), num_requests); + + http_client_peer_unlink_queue(peer, queue); + + if (array_count(&queue->pending_peers) == 0) { + i_assert(queue->cur_peer == NULL || queue->cur_peer == peer); + queue->cur_peer = NULL; + } else { + bool found = FALSE; + + i_assert(queue->cur_peer == NULL); + + /* We're still doing the initial connections to this hport. if + we're also doing parallel connections with soft timeouts + (pending_peer_count>1), wait for them to finish first. */ + array_foreach(&queue->pending_peers, peer_idx) { + if (*peer_idx == peer) { + array_delete(&queue->pending_peers, + array_foreach_idx( + &queue->pending_peers, + peer_idx), 1); + found = TRUE; + break; + } + } + i_assert(found); + if (array_count(&queue->pending_peers) > 0) { + e_debug(queue->event, + "Waiting for remaining pending peers."); + return; + } + + /* One of the connections failed. if we're not using soft + timeouts, we need to try to connect to the next IP. if we are + using soft timeouts, we've already tried all of the IPs by + now. */ + timeout_remove(&queue->to_connect); + + if (queue->addr.type == HTTP_CLIENT_PEER_ADDR_UNIX) { + http_client_queue_fail( + queue, HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, + reason); + return; + } + } + + if (http_client_queue_is_last_connect_ip(queue)) { + if (array_count(&queue->pending_peers) > 0) { + /* Other connection attempts still pending */ + return; + } + + /* All IPs failed up until here and we allow no more connect + attempts, but try the next ones on the next request. */ + queue->ips_connect_idx = queue->ips_connect_start_idx = + (queue->ips_connect_idx + 1) % ips_count; + + if (set->max_connect_attempts == 0 || + queue->connect_attempts >= set->max_connect_attempts) { + + e_debug(queue->event, + "Failed to set up any connection; " + "failing all queued requests"); + if (queue->connect_attempts > 1) { + unsigned int total_msecs = + timeval_diff_msecs(&ioloop_timeval, + &queue->first_connect_time); + reason = t_strdup_printf( + "%s (%u attempts in %u.%03u secs)", + reason, queue->connect_attempts, + total_msecs/1000, total_msecs%1000); + } + queue->connect_attempts = 0; + http_client_queue_fail( + queue, HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, + reason); + return; + } + } else { + queue->ips_connect_idx = + (queue->ips_connect_idx + 1) % ips_count; + } + + if (http_client_queue_connection_attempt(queue) != peer) + http_client_peer_unlink_queue(peer, queue); + return; +} + +void http_client_queue_peer_disconnected(struct http_client_queue *queue, + struct http_client_peer *peer) +{ + struct http_client_peer *const *peer_idx; + + if (queue->cur_peer == peer) { + queue->cur_peer = NULL; + return; + } + + array_foreach(&queue->pending_peers, peer_idx) { + if (*peer_idx == peer) { + array_delete(&queue->pending_peers, + array_foreach_idx(&queue->pending_peers, + peer_idx), 1); + break; + } + } +} + +/* + * Main request queue + */ + +void http_client_queue_drop_request(struct http_client_queue *queue, + struct http_client_request *req) +{ + struct http_client_request **reqs; + unsigned int count, i; + + e_debug(queue->event, + "Dropping request %s", http_client_request_label(req)); + + /* Drop from queue */ + if (req->urgent) { + reqs = array_get_modifiable(&queue->queued_urgent_requests, + &count); + for (i = 0; i < count; i++) { + if (reqs[i] == req) { + array_delete(&queue->queued_urgent_requests, + i, 1); + break; + } + } + } else { + reqs = array_get_modifiable(&queue->queued_requests, &count); + for (i = 0; i < count; i++) { + if (reqs[i] == req) { + array_delete(&queue->queued_requests, i, 1); + break; + } + } + } + + /* Drop from delay queue */ + if (req->release_time.tv_sec > 0) { + reqs = array_get_modifiable(&queue->delayed_requests, &count); + for (i = 0; i < count; i++) { + if (reqs[i] == req) + break; + } + if (i < count) { + if (i == 0) { + if (queue->to_delayed != NULL) { + timeout_remove(&queue->to_delayed); + if (count > 1) { + i_assert(reqs[1]->release_time.tv_sec > 0); + http_client_queue_set_delay_timer( + queue, reqs[1]->release_time); + } + } + } + array_delete(&queue->delayed_requests, i, 1); + } + } + + /* Drop from main request list */ + reqs = array_get_modifiable(&queue->requests, &count); + for (i = 0; i < count; i++) { + if (reqs[i] == req) + break; + } + i_assert(i < count); + + if (i == 0) { + if (queue->to_request != NULL) { + timeout_remove(&queue->to_request); + if (count > 1 && reqs[1]->timeout_time.tv_sec > 0) { + http_client_queue_set_request_timer(queue, + &reqs[1]->timeout_time); + } + } + } + req->queue = NULL; + array_delete(&queue->requests, i, 1); + + if (array_count(&queue->requests) == 0) + http_client_host_check_idle(queue->host); + return; +} + +static void http_client_queue_request_timeout(struct http_client_queue *queue) +{ + struct http_client_request *const *reqs; + ARRAY_TYPE(http_client_request) failed_requests; + struct timeval new_to = { 0, 0 }; + string_t *str; + size_t prefix_size; + unsigned int count, i; + + e_debug(queue->event, "Timeout (now: %s.%03lu)", + t_strflocaltime("%Y-%m-%d %H:%M:%S", ioloop_timeval.tv_sec), + ((unsigned long)ioloop_timeval.tv_usec) / 1000); + + timeout_remove(&queue->to_request); + + /* Collect failed requests */ + reqs = array_get(&queue->requests, &count); + i_assert(count > 0); + t_array_init(&failed_requests, count); + for (i = 0; i < count; i++) { + if (reqs[i]->timeout_time.tv_sec > 0 && + timeval_cmp_margin(&reqs[i]->timeout_time, + &ioloop_timeval, + TIMEOUT_CMP_MARGIN_USECS) > 0) { + break; + } + array_push_back(&failed_requests, &reqs[i]); + } + + /* Update timeout */ + if (i < count) + new_to = reqs[i]->timeout_time; + + str = t_str_new(64); + str_append(str, "Request "); + prefix_size = str_len(str); + + /* Abort all failed request */ + reqs = array_get(&failed_requests, &count); + i_assert(count > 0); /* At least one request timed out */ + for (i = 0; i < count; i++) { + struct http_client_request *req = reqs[i]; + + str_truncate(str, prefix_size); + http_client_request_append_stats_text(req, str); + + e_debug(queue->event, + "Absolute timeout expired for request %s (%s)", + http_client_request_label(req), str_c(str)); + http_client_request_error( + &req, HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT, + t_strdup_printf( + "Absolute request timeout expired (%s)", + str_c(str))); + } + + if (new_to.tv_sec > 0) { + e_debug(queue->event, "New timeout"); + http_client_queue_set_request_timer(queue, &new_to); + } +} + +static void +http_client_queue_set_request_timer(struct http_client_queue *queue, + const struct timeval *time) +{ + i_assert(time->tv_sec > 0); + timeout_remove(&queue->to_request); + + e_debug(queue->event, + "Set request timeout to %s.%03lu (now: %s.%03lu)", + t_strflocaltime("%Y-%m-%d %H:%M:%S", time->tv_sec), + ((unsigned long)time->tv_usec) / 1000, + t_strflocaltime("%Y-%m-%d %H:%M:%S", ioloop_timeval.tv_sec), + ((unsigned long)ioloop_timeval.tv_usec) / 1000); + + /* Set timer */ + queue->to_request = timeout_add_absolute_to( + queue->client->ioloop, time, + http_client_queue_request_timeout, queue); +} + +static int +http_client_queue_request_timeout_cmp(struct http_client_request *const *req1, + struct http_client_request *const *req2) +{ + int ret; + + /* 0 means no timeout */ + if ((*req1)->timeout_time.tv_sec == 0) { + if ((*req2)->timeout_time.tv_sec == 0) { + /* sort by age */ + ret = timeval_cmp(&(*req1)->submit_time, + &(*req2)->submit_time); + if (ret != 0) + return ret; + } else { + return 1; + } + } else if ((*req2)->timeout_time.tv_sec == 0) { + return -1; + + /* Sort by timeout */ + } else if ((ret = timeval_cmp(&(*req1)->timeout_time, + &(*req2)->timeout_time)) != 0) { + return ret; + } + + /* Sort by minimum attempts for fairness */ + return ((int)(*req2)->attempts - (int)(*req1)->attempts); +} + +static void +http_client_queue_submit_now(struct http_client_queue *queue, + struct http_client_request *req) +{ + ARRAY_TYPE(http_client_request) *req_queue; + + req->release_time.tv_sec = 0; + req->release_time.tv_usec = 0; + + if (req->urgent) + req_queue = &queue->queued_urgent_requests; + else + req_queue = &queue->queued_requests; + + /* Enqueue */ + if (req->timeout_time.tv_sec == 0) { + /* No timeout; enqueue at end */ + array_push_back(req_queue, &req); + } else if (timeval_diff_msecs(&req->timeout_time, + &ioloop_timeval) <= 1) { + /* Pretty much already timed out; don't bother */ + return; + } else { + unsigned int insert_idx; + + /* Keep transmission queue sorted earliest timeout first */ + (void)array_bsearch_insert_pos( + req_queue, &req, + http_client_queue_request_timeout_cmp, &insert_idx); + array_insert(req_queue, insert_idx, &req, 1); + } + + http_client_queue_connection_setup(queue); +} + +/* + * Delayed request queue + */ + +static void +http_client_queue_delay_timeout(struct http_client_queue *queue) +{ + struct http_client_request *const *reqs; + unsigned int count, i, finished; + + timeout_remove(&queue->to_delayed); + io_loop_time_refresh(); + + finished = 0; + reqs = array_get(&queue->delayed_requests, &count); + for (i = 0; i < count; i++) { + if (timeval_cmp_margin(&reqs[i]->release_time, + &ioloop_timeval, + TIMEOUT_CMP_MARGIN_USECS) > 0) { + break; + } + + e_debug(queue->event, "Activated delayed request %s%s", + http_client_request_label(reqs[i]), + (reqs[i]->urgent ? " (urgent)" : "")); + http_client_queue_submit_now(queue, reqs[i]); + finished++; + } + if (i < count) + http_client_queue_set_delay_timer(queue, reqs[i]->release_time); + array_delete(&queue->delayed_requests, 0, finished); +} + +static void +http_client_queue_set_delay_timer(struct http_client_queue *queue, + struct timeval time) +{ + struct http_client *client = queue->client; + int usecs = timeval_diff_usecs(&time, &ioloop_timeval); + int msecs; + + /* Round up to nearest microsecond */ + msecs = (usecs + 999) / 1000; + + /* Set timer */ + timeout_remove(&queue->to_delayed); + queue->to_delayed = timeout_add_to( + client->ioloop, msecs, + http_client_queue_delay_timeout, queue); +} + +static int +http_client_queue_delayed_cmp(struct http_client_request *const *req1, + struct http_client_request *const *req2) +{ + return timeval_cmp(&(*req1)->release_time, &(*req2)->release_time); +} + +/* + * Request submission + */ + +void http_client_queue_submit_request(struct http_client_queue *queue, + struct http_client_request *req) +{ + unsigned int insert_idx; + + if (req->queue != NULL) + http_client_queue_drop_request(req->queue, req); + req->queue = queue; + + /* Check delay vs timeout */ + if (req->release_time.tv_sec > 0 && req->timeout_time.tv_sec > 0 && + timeval_cmp_margin(&req->release_time, &req->timeout_time, + TIMEOUT_CMP_MARGIN_USECS) >= 0) { + /* Release time is later than absolute timeout */ + req->release_time.tv_sec = 0; + req->release_time.tv_usec = 0; + + /* Timeout rightaway */ + req->timeout_time = ioloop_timeval; + + e_debug(queue->event, + "Delayed request %s%s already timed out", + http_client_request_label(req), + (req->urgent ? " (urgent)" : "")); + } + + /* Add to main request list */ + if (req->timeout_time.tv_sec == 0) { + /* No timeout; just append */ + array_push_back(&queue->requests, &req); + } else { + unsigned int insert_idx; + + /* Keep main request list sorted earliest timeout first */ + (void)array_bsearch_insert_pos( + &queue->requests, &req, + http_client_queue_request_timeout_cmp, &insert_idx); + array_insert(&queue->requests, insert_idx, &req, 1); + + /* Now first in queue; update timer */ + if (insert_idx == 0) { + http_client_queue_set_request_timer(queue, + &req->timeout_time); + } + } + + /* Handle delay */ + if (req->release_time.tv_sec > 0) { + io_loop_time_refresh(); + + if (timeval_cmp_margin(&req->release_time, &ioloop_timeval, + TIMEOUT_CMP_MARGIN_USECS) > 0) { + e_debug(queue->event, + "Delayed request %s%s submitted " + "(time remaining: %d msecs)", + http_client_request_label(req), + (req->urgent ? " (urgent)" : ""), + timeval_diff_msecs(&req->release_time, + &ioloop_timeval)); + + (void)array_bsearch_insert_pos( + &queue->delayed_requests, &req, + http_client_queue_delayed_cmp, &insert_idx); + array_insert(&queue->delayed_requests, insert_idx, + &req, 1); + if (insert_idx == 0) { + http_client_queue_set_delay_timer( + queue, req->release_time); + } + return; + } + } + + http_client_queue_submit_now(queue, req); +} + +/* + * Request retrieval + */ + +struct http_client_request * +http_client_queue_claim_request(struct http_client_queue *queue, + const struct http_client_peer_addr *addr, + bool no_urgent) +{ + struct http_client_request *const *requests; + struct http_client_request *req; + unsigned int i, count; + + count = 0; + if (!no_urgent) + requests = array_get(&queue->queued_urgent_requests, &count); + + if (count == 0) + requests = array_get(&queue->queued_requests, &count); + if (count == 0) + return NULL; + i = 0; + req = requests[i]; + if (req->urgent) + array_delete(&queue->queued_urgent_requests, i, 1); + else + array_delete(&queue->queued_requests, i, 1); + + e_debug(queue->event, + "Connection to peer %s claimed request %s %s", + http_client_peer_addr2str(addr), http_client_request_label(req), + (req->urgent ? "(urgent)" : "")); + + return req; +} + +unsigned int +http_client_queue_requests_pending(struct http_client_queue *queue, + unsigned int *num_urgent_r) +{ + unsigned int urg_count = array_count(&queue->queued_urgent_requests); + + if (num_urgent_r != NULL) + *num_urgent_r = urg_count; + return array_count(&queue->queued_requests) + urg_count; +} + +unsigned int http_client_queue_requests_active(struct http_client_queue *queue) +{ + return array_count(&queue->requests); +} + +/* + * Ioloop + */ + +void http_client_queue_switch_ioloop(struct http_client_queue *queue) +{ + if (queue->to_connect != NULL) + queue->to_connect = io_loop_move_timeout(&queue->to_connect); + if (queue->to_request != NULL) + queue->to_request = io_loop_move_timeout(&queue->to_request); + if (queue->to_delayed != NULL) + queue->to_delayed = io_loop_move_timeout(&queue->to_delayed); +} diff --git a/src/lib-http/http-client-request.c b/src/lib-http/http-client-request.c new file mode 100644 index 0000000..60602e0 --- /dev/null +++ b/src/lib-http/http-client-request.c @@ -0,0 +1,1875 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "net.h" +#include "str.h" +#include "str-sanitize.h" +#include "hash.h" +#include "array.h" +#include "llist.h" +#include "time-util.h" +#include "istream.h" +#include "ostream.h" +#include "file-lock.h" +#include "dns-lookup.h" +#include "http-url.h" +#include "http-date.h" +#include "http-auth.h" +#include "http-response-parser.h" +#include "http-transfer.h" + +#include "http-client-private.h" + +const char *http_request_state_names[] = { + "new", + "queued", + "payload_out", + "waiting", + "got_response", + "payload_in", + "finished", + "aborted" +}; + +/* + * Request + */ + +static bool +http_client_request_send_error(struct http_client_request *req, + unsigned int status, const char *error); + +const char *http_client_request_label(struct http_client_request *req) +{ + if (req->label == NULL) { + req->label = p_strdup_printf( + req->pool, "[Req%u: %s %s%s]", req->id, req->method, + http_url_create_host(&req->origin_url), req->target); + } + return req->label; +} + +static void http_client_request_update_event(struct http_client_request *req) +{ + event_add_str(req->event, "method", req->method); + event_add_str(req->event, "dest_host", req->origin_url.host.name); + event_add_int(req->event, "dest_port", + http_url_get_port(&req->origin_url)); + if (req->target != NULL) + event_add_str(req->event, "target", req->target); + event_set_append_log_prefix( + req->event, t_strdup_printf("request %s: ", + str_sanitize(http_client_request_label(req), 256))); +} + +static struct event_passthrough * +http_client_request_result_event(struct http_client_request *req) +{ + struct http_client_connection *conn = req->conn; + + if (conn != NULL) { + if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT) { + /* Got here prematurely; use bytes written so far */ + i_assert(req->request_offset < + conn->conn.output->offset); + req->bytes_out = conn->conn.output->offset - + req->request_offset; + } + if (conn->incoming_payload != NULL && + (req->state == HTTP_REQUEST_STATE_GOT_RESPONSE || + req->state == HTTP_REQUEST_STATE_PAYLOAD_IN)) { + /* Got here prematurely; use bytes read so far */ + i_assert(conn->in_req_callback || + conn->pending_request == req); + i_assert(req->response_offset < + conn->conn.input->v_offset); + req->bytes_in = conn->conn.input->v_offset - + req->response_offset; + } + } + + struct event_passthrough *e = event_create_passthrough(req->event); + if (req->queue != NULL && + req->queue->addr.type != HTTP_CLIENT_PEER_ADDR_UNIX) + e->add_str("dest_ip", net_ip2addr(&req->queue->addr.a.tcp.ip)); + + return e->add_int("status_code", req->last_status)-> + add_int("attempts", req->attempts)-> + add_int("redirects", req->redirects)-> + add_int("bytes_in", req->bytes_in)-> + add_int("bytes_out", req->bytes_out); +} + +static struct http_client_request * +http_client_request_new(struct http_client *client, const char *method, + http_client_request_callback_t *callback, void *context) +{ + static unsigned int id_counter = 0; + pool_t pool; + struct http_client_request *req; + + pool = pool_alloconly_create("http client request", 2048); + req = p_new(pool, struct http_client_request, 1); + req->pool = pool; + req->refcount = 1; + req->client = client; + req->id = ++id_counter; + req->method = p_strdup(pool, method); + req->callback = callback; + req->context = context; + req->date = (time_t)-1; + req->event = event_create(client->event); + event_strlist_copy_recursive(req->event, event_get_global(), + EVENT_REASON_CODE); + + /* Default to client-wide settings: */ + req->max_attempts = client->set.max_attempts; + req->attempt_timeout_msecs = client->set.request_timeout_msecs; + + req->state = HTTP_REQUEST_STATE_NEW; + return req; +} + +#undef http_client_request +struct http_client_request * +http_client_request(struct http_client *client, + const char *method, const char *host, const char *target, + http_client_request_callback_t *callback, void *context) +{ + struct http_client_request *req; + + req = http_client_request_new(client, method, callback, context); + req->origin_url.host.name = p_strdup(req->pool, host); + req->target = (target == NULL ? "/" : p_strdup(req->pool, target)); + http_client_request_update_event(req); + return req; +} + +#undef http_client_request_url +struct http_client_request * +http_client_request_url(struct http_client *client, + const char *method, const struct http_url *target_url, + http_client_request_callback_t *callback, void *context) +{ + struct http_client_request *req; + + req = http_client_request_new(client, method, callback, context); + http_url_copy_authority(req->pool, &req->origin_url, target_url); + req->target = p_strdup(req->pool, http_url_create_target(target_url)); + if (target_url->user != NULL && *target_url->user != '\0' && + target_url->password != NULL) { + req->username = p_strdup(req->pool, target_url->user); + req->password = p_strdup(req->pool, target_url->password); + } + http_client_request_update_event(req); + return req; +} + +#undef http_client_request_url_str +struct http_client_request * +http_client_request_url_str(struct http_client *client, + const char *method, const char *url_str, + http_client_request_callback_t *callback, + void *context) +{ + struct http_client_request *req, *tmpreq; + struct http_url *target_url; + const char *error; + + req = tmpreq = http_client_request_new(client, method, + callback, context); + + if (http_url_parse(url_str, NULL, HTTP_URL_ALLOW_USERINFO_PART, + req->pool, &target_url, &error) < 0) { + req->label = p_strdup_printf(req->pool, "[Req%u: %s %s]", + req->id, req->method, url_str); + http_client_request_error( + &tmpreq, HTTP_CLIENT_REQUEST_ERROR_INVALID_URL, + t_strdup_printf("Invalid HTTP URL: %s", error)); + http_client_request_update_event(req); + return req; + } + + req->origin_url = *target_url; + req->target = p_strdup(req->pool, http_url_create_target(target_url)); + if (target_url->user != NULL && *target_url->user != '\0' && + target_url->password != NULL) { + req->username = p_strdup(req->pool, target_url->user); + req->password = p_strdup(req->pool, target_url->password); + } + http_client_request_update_event(req); + return req; +} + +#undef http_client_request_connect +struct http_client_request * +http_client_request_connect(struct http_client *client, + const char *host, in_port_t port, + http_client_request_callback_t *callback, + void *context) +{ + struct http_client_request *req; + + req = http_client_request_new(client, "CONNECT", callback, context); + req->origin_url.host.name = p_strdup(req->pool, host); + req->origin_url.port = port; + req->connect_tunnel = TRUE; + req->target = req->origin_url.host.name; + http_client_request_update_event(req); + return req; +} + +#undef http_client_request_connect_ip +struct http_client_request * +http_client_request_connect_ip(struct http_client *client, + const struct ip_addr *ip, in_port_t port, + http_client_request_callback_t *callback, + void *context) +{ + struct http_client_request *req; + const char *hostname; + + i_assert(ip->family != 0); + hostname = net_ip2addr(ip); + + req = http_client_request_connect(client, hostname, port, + callback, context); + req->origin_url.host.ip = *ip; + return req; +} + +void http_client_request_set_event(struct http_client_request *req, + struct event *event) +{ + event_unref(&req->event); + req->event = event_create(event); + event_set_forced_debug(req->event, req->client->set.debug); + event_strlist_copy_recursive(req->event, event_get_global(), + EVENT_REASON_CODE); + http_client_request_update_event(req); +} + +static void http_client_request_add(struct http_client_request *req) +{ + struct http_client *client = req->client; + + DLLIST_PREPEND(&client->requests_list, req); + client->requests_count++; + req->listed = TRUE; +} + +static void http_client_request_remove(struct http_client_request *req) +{ + struct http_client *client = req->client; + + if (client == NULL) { + i_assert(!req->listed); + return; + } + if (req->listed) { + /* Only decrease pending request counter if this request was + submitted */ + DLLIST_REMOVE(&client->requests_list, req); + client->requests_count--; + } + req->listed = FALSE; + + if (client->requests_count == 0 && client->waiting) + io_loop_stop(client->ioloop); +} + +void http_client_request_ref(struct http_client_request *req) +{ + i_assert(req->refcount > 0); + req->refcount++; +} + +bool http_client_request_unref(struct http_client_request **_req) +{ + struct http_client_request *req = *_req; + struct http_client *client = req->client; + + i_assert(req->refcount > 0); + + *_req = NULL; + + if (--req->refcount > 0) + return TRUE; + + if (client == NULL) { + e_debug(req->event, "Free (client already destroyed)"); + } else { + e_debug(req->event, "Free (requests left=%d)", + client->requests_count); + } + + /* Cannot be destroyed while it is still pending */ + i_assert(req->conn == NULL); + + if (req->queue != NULL) + http_client_queue_drop_request(req->queue, req); + + if (req->destroy_callback != NULL) { + req->destroy_callback(req->destroy_context); + req->destroy_callback = NULL; + } + + http_client_request_remove(req); + + if (client != NULL) { + if (client->requests_count == 0 && client->waiting) + io_loop_stop(client->ioloop); + if (req->delayed_error != NULL) + http_client_remove_request_error(req->client, req); + } + i_stream_unref(&req->payload_input); + o_stream_unref(&req->payload_output); + str_free(&req->headers); + event_unref(&req->event); + pool_unref(&req->pool); + return FALSE; +} + +void http_client_request_destroy(struct http_client_request **_req) +{ + struct http_client_request *req = *_req, *tmp_req; + struct http_client *client = req->client; + + *_req = NULL; + + if (client == NULL) { + e_debug(req->event, "Destroy (client already destroyed)"); + } else { + e_debug(req->event, "Destroy (requests left=%d)", + client->requests_count); + } + + + if (req->state < HTTP_REQUEST_STATE_FINISHED) + req->state = HTTP_REQUEST_STATE_ABORTED; + req->callback = NULL; + + if (req->queue != NULL) + http_client_queue_drop_request(req->queue, req); + + if (client != NULL && req->delayed_error != NULL) + http_client_remove_request_error(req->client, req); + req->delayed_error = NULL; + + if (req->destroy_callback != NULL) { + void (*callback)(void *) = req->destroy_callback; + + req->destroy_callback = NULL; + callback(req->destroy_context); + } + + if (req->conn != NULL) + http_client_connection_request_destroyed(req->conn, req); + + tmp_req = req; + http_client_request_remove(req); + if (http_client_request_unref(&tmp_req)) + req->client = NULL; +} + +void http_client_request_set_port(struct http_client_request *req, + in_port_t port) +{ + i_assert(req->state == HTTP_REQUEST_STATE_NEW); + req->origin_url.port = port; + event_add_int(req->event, "port", port); +} + +void http_client_request_set_ssl(struct http_client_request *req, bool ssl) +{ + i_assert(req->state == HTTP_REQUEST_STATE_NEW); + req->origin_url.have_ssl = ssl; +} + +void http_client_request_set_urgent(struct http_client_request *req) +{ + i_assert(req->state == HTTP_REQUEST_STATE_NEW); + req->urgent = TRUE; +} + +void http_client_request_set_preserve_exact_reason( + struct http_client_request *req) +{ + req->preserve_exact_reason = TRUE; +} + +static bool +http_client_request_lookup_header_pos(struct http_client_request *req, + const char *key, size_t *key_pos_r, + size_t *value_pos_r, size_t *next_pos_r) +{ + const unsigned char *data, *p; + size_t size, line_len; + size_t key_len = strlen(key); + + if (req->headers == NULL) + return FALSE; + + data = str_data(req->headers); + size = str_len(req->headers); + while ((p = memchr(data, '\n', size)) != NULL) { + line_len = (p+1) - data; + if (size > key_len && i_memcasecmp(data, key, key_len) == 0 && + data[key_len] == ':' && data[key_len+1] == ' ') { + /* Key was found from header, replace its value */ + *key_pos_r = str_len(req->headers) - size; + *value_pos_r = *key_pos_r + key_len + 2; + *next_pos_r = *key_pos_r + line_len; + return TRUE; + } + size -= line_len; + data += line_len; + } + return FALSE; +} + +static void +http_client_request_add_header_full(struct http_client_request *req, + const char *key, const char *value, + bool replace_existing) +{ + size_t key_pos, value_pos, next_pos; + + i_assert(req->state == HTTP_REQUEST_STATE_NEW || + /* Allow calling for retries */ + req->state == HTTP_REQUEST_STATE_GOT_RESPONSE || + req->state == HTTP_REQUEST_STATE_ABORTED); + /* Make sure key or value can't break HTTP headers entirely */ + i_assert(strpbrk(key, ":\r\n") == NULL); + i_assert(strpbrk(value, "\r\n") == NULL); + + /* Mark presence of special headers */ + switch (key[0]) { + case 'a': case 'A': + if (strcasecmp(key, "Authorization") == 0) + req->have_hdr_authorization = TRUE; + break; + case 'c': case 'C': + if (strcasecmp(key, "Connection") == 0) + req->have_hdr_connection = TRUE; + else if (strcasecmp(key, "Content-Length") == 0) + req->have_hdr_body_spec = TRUE; + break; + case 'd': case 'D': + if (strcasecmp(key, "Date") == 0) + req->have_hdr_date = TRUE; + break; + case 'e': case 'E': + if (strcasecmp(key, "Expect") == 0) + req->have_hdr_expect = TRUE; + break; + case 'h': case 'H': + if (strcasecmp(key, "Host") == 0) + req->have_hdr_host = TRUE; + break; + case 'p': case 'P': + i_assert(strcasecmp(key, "Proxy-Authorization") != 0); + break; + case 't': case 'T': + if (strcasecmp(key, "Transfer-Encoding") == 0) + req->have_hdr_body_spec = TRUE; + break; + case 'u': case 'U': + if (strcasecmp(key, "User-Agent") == 0) + req->have_hdr_user_agent = TRUE; + break; + } + if (req->headers == NULL) + req->headers = str_new(default_pool, 256); + if (!http_client_request_lookup_header_pos(req, key, &key_pos, + &value_pos, &next_pos)) + str_printfa(req->headers, "%s: %s\r\n", key, value); + else if (replace_existing) { + /* Don't delete CRLF */ + size_t old_value_len = next_pos - value_pos - 2; + str_replace(req->headers, value_pos, old_value_len, value); + } +} + +void http_client_request_add_header(struct http_client_request *req, + const char *key, const char *value) +{ + http_client_request_add_header_full(req, key, value, TRUE); +} + +void http_client_request_add_missing_header(struct http_client_request *req, + const char *key, const char *value) +{ + http_client_request_add_header_full(req, key, value, FALSE); +} + +void http_client_request_remove_header(struct http_client_request *req, + const char *key) +{ + size_t key_pos, value_pos, next_pos; + + i_assert(req->state == HTTP_REQUEST_STATE_NEW || + /* Allow calling for retries */ + req->state == HTTP_REQUEST_STATE_GOT_RESPONSE || + req->state == HTTP_REQUEST_STATE_ABORTED); + + if (http_client_request_lookup_header_pos(req, key, &key_pos, + &value_pos, &next_pos)) + str_delete(req->headers, key_pos, next_pos - key_pos); +} + +const char *http_client_request_lookup_header(struct http_client_request *req, + const char *key) +{ + size_t key_pos, value_pos, next_pos; + + if (!http_client_request_lookup_header_pos(req, key, &key_pos, + &value_pos, &next_pos)) + return NULL; + + /* Don't return CRLF */ + return t_strndup(str_data(req->headers) + value_pos, + next_pos - value_pos - 2); +} + +void http_client_request_set_date(struct http_client_request *req, time_t date) +{ + i_assert(req->state == HTTP_REQUEST_STATE_NEW); + req->date = date; +} + +void http_client_request_set_payload(struct http_client_request *req, + struct istream *input, bool sync) +{ + int ret; + + i_assert(req->state == HTTP_REQUEST_STATE_NEW); + i_assert(req->payload_input == NULL); + + i_stream_ref(input); + req->payload_input = input; + if ((ret = i_stream_get_size(input, TRUE, &req->payload_size)) <= 0) { + if (ret < 0) { + i_error("i_stream_get_size(%s) failed: %s", + i_stream_get_name(input), + i_stream_get_error(input)); + } + req->payload_size = 0; + req->payload_chunked = TRUE; + } else { + i_assert(input->v_offset <= req->payload_size); + req->payload_size -= input->v_offset; + } + req->payload_offset = input->v_offset; + + /* Prepare request payload sync using 100 Continue response from server + */ + if ((req->payload_chunked || req->payload_size > 0) && sync) + req->payload_sync = TRUE; +} + +void http_client_request_set_payload_data(struct http_client_request *req, + const unsigned char *data, + size_t size) +{ + struct istream *input; + unsigned char *payload_data; + + if (size == 0) + return; + + payload_data = p_malloc(req->pool, size); + memcpy(payload_data, data, size); + input = i_stream_create_from_data(payload_data, size); + + http_client_request_set_payload(req, input, FALSE); + i_stream_unref(&input); +} + +void http_client_request_set_payload_empty(struct http_client_request *req) +{ + req->payload_empty = TRUE; +} + +void http_client_request_set_timeout_msecs(struct http_client_request *req, + unsigned int msecs) +{ + i_assert(req->state == HTTP_REQUEST_STATE_NEW || + req->state == HTTP_REQUEST_STATE_GOT_RESPONSE); + + req->timeout_msecs = msecs; +} + +void http_client_request_set_timeout(struct http_client_request *req, + const struct timeval *time) +{ + i_assert(req->state == HTTP_REQUEST_STATE_NEW || + req->state == HTTP_REQUEST_STATE_GOT_RESPONSE); + + req->timeout_time = *time; + req->timeout_msecs = 0; +} + +void http_client_request_set_attempt_timeout_msecs( + struct http_client_request *req, unsigned int msecs) +{ + i_assert(req->state == HTTP_REQUEST_STATE_NEW || + req->state == HTTP_REQUEST_STATE_GOT_RESPONSE); + + req->attempt_timeout_msecs = msecs; +} + +void http_client_request_set_max_attempts(struct http_client_request *req, + unsigned int max_attempts) +{ + i_assert(req->state == HTTP_REQUEST_STATE_NEW || + req->state == HTTP_REQUEST_STATE_GOT_RESPONSE); + + req->max_attempts = max_attempts; +} + +void http_client_request_set_event_headers(struct http_client_request *req, + const char *const *headers) +{ + req->event_headers = p_strarray_dup(req->pool, headers); +} + +void http_client_request_set_auth_simple(struct http_client_request *req, + const char *username, + const char *password) +{ + req->username = p_strdup(req->pool, username); + req->password = p_strdup(req->pool, password); +} + +void http_client_request_set_proxy_url(struct http_client_request *req, + const struct http_url *proxy_url) +{ + i_assert(req->state == HTTP_REQUEST_STATE_NEW || + req->state == HTTP_REQUEST_STATE_GOT_RESPONSE); + + req->host_url = http_url_clone_authority(req->pool, proxy_url); + req->host_socket = NULL; +} + +void http_client_request_set_proxy_socket(struct http_client_request *req, + const char *proxy_socket) +{ + i_assert(req->state == HTTP_REQUEST_STATE_NEW || + req->state == HTTP_REQUEST_STATE_GOT_RESPONSE); + + req->host_socket = p_strdup(req->pool, proxy_socket); + req->host_url = NULL; +} + +void http_client_request_delay_until(struct http_client_request *req, + time_t time) +{ + req->release_time.tv_sec = time; + req->release_time.tv_usec = 0; +} + +void http_client_request_delay(struct http_client_request *req, time_t seconds) +{ + req->release_time = ioloop_timeval; + req->release_time.tv_sec += seconds; +} + +void http_client_request_delay_msecs(struct http_client_request *req, + unsigned int msecs) +{ + req->release_time = ioloop_timeval; + timeval_add_msecs(&req->release_time, msecs); +} + +int http_client_request_delay_from_response( + struct http_client_request *req, const struct http_response *response) +{ + time_t retry_after = response->retry_after; + unsigned int max; + + i_assert(req->client != NULL); + + if (retry_after == (time_t)-1) + return 0; /* no delay */ + if (retry_after < ioloop_time) + return 0; /* delay already expired */ + max = (req->client->set.max_auto_retry_delay_secs == 0 ? + req->attempt_timeout_msecs / 1000 : + req->client->set.max_auto_retry_delay_secs); + if ((unsigned int)(retry_after - ioloop_time) > max) + return -1; /* delay too long */ + req->release_time.tv_sec = retry_after; + req->release_time.tv_usec = 0; + return 1; /* valid delay */ +} + +const char * +http_client_request_get_method(const struct http_client_request *req) +{ + return req->method; +} + +const char * +http_client_request_get_target(const struct http_client_request *req) +{ + return req->target; +} + +const struct http_url * +http_client_request_get_origin_url(const struct http_client_request *req) +{ + return &req->origin_url; +} + +enum http_request_state +http_client_request_get_state(const struct http_client_request *req) +{ + return req->state; +} + +unsigned int +http_client_request_get_attempts(const struct http_client_request *req) +{ + return req->attempts; +} + +void http_client_request_get_stats(struct http_client_request *req, + struct http_client_request_stats *stats_r) +{ + struct http_client *client = req->client; + int diff_msecs; + uint64_t wait_usecs; + + i_zero(stats_r); + if (!req->submitted) + return; + + /* Total elapsed time since message was submitted */ + diff_msecs = timeval_diff_msecs(&ioloop_timeval, &req->submit_time); + stats_r->total_msecs = (unsigned int)I_MAX(diff_msecs, 0); + + /* Elapsed time since message was first sent */ + if (req->first_sent_time.tv_sec > 0) { + diff_msecs = timeval_diff_msecs(&ioloop_timeval, + &req->first_sent_time); + stats_r->first_sent_msecs = (unsigned int)I_MAX(diff_msecs, 0); + } + + /* Elapsed time since message was last sent */ + if (req->sent_time.tv_sec > 0) { + diff_msecs = timeval_diff_msecs(&ioloop_timeval, + &req->sent_time); + stats_r->last_sent_msecs = (unsigned int)I_MAX(diff_msecs, 0); + } + + if (req->conn != NULL) { + /* Time spent in other ioloops */ + i_assert(ioloop_global_wait_usecs >= + req->sent_global_ioloop_usecs); + stats_r->other_ioloop_msecs = (unsigned int) + (ioloop_global_wait_usecs - + req->sent_global_ioloop_usecs + 999) / 1000; + + /* Time spent in the http-client's own ioloop */ + if (client != NULL && client->waiting) { + wait_usecs = + io_wait_timer_get_usecs(req->conn->io_wait_timer); + i_assert(wait_usecs >= req->sent_http_ioloop_usecs); + stats_r->http_ioloop_msecs = (unsigned int) + (wait_usecs - + req->sent_http_ioloop_usecs + 999) / 1000; + + i_assert(stats_r->other_ioloop_msecs >= + stats_r->http_ioloop_msecs); + stats_r->other_ioloop_msecs -= stats_r->http_ioloop_msecs; + } + } + + /* Total time spent on waiting for file locks */ + wait_usecs = file_lock_wait_get_total_usecs(); + i_assert(wait_usecs >= req->sent_lock_usecs); + stats_r->lock_msecs = (unsigned int) + (wait_usecs - req->sent_lock_usecs + 999) / 1000; + + /* Number of attempts for this request */ + stats_r->attempts = req->attempts; + /* Number of send attempts for this request */ + stats_r->send_attempts = req->send_attempts; +} + +void http_client_request_append_stats_text(struct http_client_request *req, + string_t *str) +{ + struct http_client_request_stats stats; + + if (!req->submitted) { + str_append(str, "not yet submitted"); + return; + } + + http_client_request_get_stats(req, &stats); + + str_printfa(str, "queued %u.%03u secs ago", + stats.total_msecs/1000, stats.total_msecs%1000); + if (stats.attempts > 0) + str_printfa(str, ", %u times retried", stats.attempts); + + if (stats.send_attempts == 0) { + str_append(str, ", not yet sent"); + } else { + str_printfa(str, ", %u send attempts in %u.%03u secs", + stats.send_attempts, stats.first_sent_msecs/1000, + stats.first_sent_msecs%1000); + if (stats.send_attempts > 1) { + str_printfa(str, ", %u.%03u in last attempt", + stats.last_sent_msecs/1000, + stats.last_sent_msecs%1000); + } + } + + if (stats.http_ioloop_msecs > 0) { + str_printfa(str, ", %u.%03u in http ioloop", + stats.http_ioloop_msecs/1000, + stats.http_ioloop_msecs%1000); + } + str_printfa(str, ", %u.%03u in other ioloops", + stats.other_ioloop_msecs/1000, + stats.other_ioloop_msecs%1000); + + if (stats.lock_msecs > 0) { + str_printfa(str, ", %u.%03u in locks", + stats.lock_msecs/1000, stats.lock_msecs%1000); + } +} + +enum http_response_payload_type +http_client_request_get_payload_type(struct http_client_request *req) +{ + /* RFC 7230, Section 3.3: + + The presence of a message body in a response depends on both the + request method to which it is responding and the response status code + (Section 3.1.2 of [RFC7230]). Responses to the HEAD request method + (Section 4.3.2 of [RFC7231]) never include a message body because the + associated response header fields (e.g., Transfer-Encoding, + Content-Length, etc.), if present, indicate only what their values + would have been if the request method had been GET (Section 4.3.1 of + [RFC7231]). 2xx (Successful) responses to a CONNECT request method + (Section 4.3.6 of [RFC7231]) switch to tunnel mode instead of having + a message body. + */ + if (strcmp(req->method, "HEAD") == 0) + return HTTP_RESPONSE_PAYLOAD_TYPE_NOT_PRESENT; + if (strcmp(req->method, "CONNECT") == 0) + return HTTP_RESPONSE_PAYLOAD_TYPE_ONLY_UNSUCCESSFUL; + return HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED; +} + +static void http_client_request_do_submit(struct http_client_request *req) +{ + struct http_client *client = req->client; + struct http_client_host *host; + const char *proxy_socket_path = client->set.proxy_socket_path; + const struct http_url *proxy_url = client->set.proxy_url; + bool have_proxy = + ((proxy_socket_path != NULL) || (proxy_url != NULL) || + (req->host_socket != NULL) || (req->host_url != NULL)); + const char *authority, *target; + + if (req->state == HTTP_REQUEST_STATE_ABORTED) + return; + i_assert(client != NULL); + i_assert(req->state == HTTP_REQUEST_STATE_NEW); + + authority = http_url_create_authority(&req->origin_url); + if (req->connect_tunnel) { + /* Connect requests require authority form for request target */ + target = authority; + } else { + /* Absolute target URL */ + target = t_strconcat(http_url_create_host(&req->origin_url), + req->target, NULL); + } + + /* Determine what host to contact to submit this request */ + if (have_proxy) { + if (req->host_socket != NULL) { + /* Specific socket proxy */ + req->host_url = NULL; + } else if (req->host_url != NULL) { + /* Specific normal proxy */ + req->host_socket = NULL; + } else if (req->origin_url.have_ssl && + !client->set.no_ssl_tunnel && + !req->connect_tunnel) { + /* Tunnel to origin server */ + req->host_url = &req->origin_url; + req->ssl_tunnel = TRUE; + } else if (proxy_socket_path != NULL) { + /* Proxy on unix socket */ + req->host_socket = proxy_socket_path; + req->host_url = NULL; + } else { + /* Normal proxy server */ + req->host_url = proxy_url; + req->host_socket = NULL; + } + } else { + /* Origin server */ + req->host_url = &req->origin_url; + } + + /* Use submission date if no date is set explicitly */ + if (req->date == (time_t)-1) + req->date = ioloop_time; + + /* Prepare value for Host header */ + req->authority = p_strdup(req->pool, authority); + + /* Debug label */ + req->label = p_strdup_printf(req->pool, "[Req%u: %s %s]", + req->id, req->method, target); + + /* Update request target */ + if (req->connect_tunnel || have_proxy) + req->target = p_strdup(req->pool, target); + + if (!have_proxy) { + /* If we don't have a proxy, CONNECT requests are handled by + creating the requested connection directly */ + req->connect_direct = req->connect_tunnel; + if (req->connect_direct) + req->urgent = TRUE; + } + + if (req->timeout_time.tv_sec == 0) { + if (req->timeout_msecs > 0) { + req->timeout_time = ioloop_timeval; + timeval_add_msecs(&req->timeout_time, + req->timeout_msecs); + } else if (client->set.request_absolute_timeout_msecs > 0) { + req->timeout_time = ioloop_timeval; + timeval_add_msecs(&req->timeout_time, + client->set.request_absolute_timeout_msecs); + } + } + + host = http_client_host_get(client, req->host_url); + req->state = HTTP_REQUEST_STATE_QUEUED; + req->last_status = 0; + + http_client_host_submit_request(host, req); +} + +void http_client_request_submit(struct http_client_request *req) +{ + i_assert(req->client != NULL); + + req->submit_time = ioloop_timeval; + + http_client_request_update_event(req); + http_client_request_do_submit(req); + + req->submitted = TRUE; + http_client_request_add(req); + + e_debug(req->event, "Submitted (requests left=%d)", + req->client->requests_count); +} + +void http_client_request_get_peer_addr(const struct http_client_request *req, + struct http_client_peer_addr *addr) +{ + const char *host_socket = req->host_socket; + const struct http_url *host_url = req->host_url; + + /* The IP address may be unassigned in the returned peer address, since + that is only available at this stage when the target URL has an + explicit IP address. */ + i_zero(addr); + if (host_socket != NULL) { + addr->type = HTTP_CLIENT_PEER_ADDR_UNIX; + addr->a.un.path = host_socket; + } else if (req->connect_direct) { + addr->type = HTTP_CLIENT_PEER_ADDR_RAW; + addr->a.tcp.ip = host_url->host.ip; + addr->a.tcp.port = + http_url_get_port_default(host_url, HTTPS_DEFAULT_PORT); + } else if (host_url->have_ssl) { + if (req->ssl_tunnel) + addr->type = HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL; + else + addr->type = HTTP_CLIENT_PEER_ADDR_HTTPS; + addr->a.tcp.ip = host_url->host.ip; + addr->a.tcp.https_name = host_url->host.name; + addr->a.tcp.port = http_url_get_port(host_url); + } else { + addr->type = HTTP_CLIENT_PEER_ADDR_HTTP; + addr->a.tcp.ip = host_url->host.ip; + addr->a.tcp.port = http_url_get_port(host_url); + } +} + +static int http_client_request_flush_payload(struct http_client_request *req) +{ + struct http_client_connection *conn = req->conn; + int ret; + + if (req->payload_output != conn->conn.output && + (ret = o_stream_finish(req->payload_output)) <= 0) { + if (ret < 0) + http_client_connection_handle_output_error(conn); + return ret; + } + + return 1; +} + +static int +http_client_request_finish_payload_out(struct http_client_request *req) +{ + struct http_client_connection *conn = req->conn; + int ret; + + i_assert(conn != NULL); + req->payload_finished = TRUE; + + /* Drop payload output stream */ + if (req->payload_output != NULL) { + ret = http_client_request_flush_payload(req); + if (ret < 0) + return -1; + if (ret == 0) { + e_debug(req->event, + "Not quite finished sending payload"); + return 0; + } + o_stream_unref(&req->payload_output); + req->payload_output = NULL; + } + + i_assert(req->request_offset < conn->conn.output->offset); + req->bytes_out = conn->conn.output->offset - req->request_offset; + + /* Advance state only when request didn't get aborted in the mean time + */ + if (req->state != HTTP_REQUEST_STATE_ABORTED) { + i_assert(req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT); + + /* we're now waiting for a response from the server */ + req->state = HTTP_REQUEST_STATE_WAITING; + http_client_connection_start_request_timeout(conn); + } + + /* Release connection */ + conn->output_locked = FALSE; + + e_debug(req->event, "Finished sending%s payload", + (req->state == HTTP_REQUEST_STATE_ABORTED ? " aborted" : "")); + return 1; +} + +static int +http_client_request_continue_payload(struct http_client_request **_req, + const unsigned char *data, size_t size) +{ + struct ioloop *prev_ioloop, *client_ioloop, *prev_client_ioloop; + struct http_client_request *req = *_req; + struct http_client_connection *conn = req->conn; + struct http_client *client = req->client; + int ret; + + i_assert(client != NULL); + i_assert(req->state == HTTP_REQUEST_STATE_NEW || + req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT); + i_assert(req->payload_input == NULL); + + if (conn != NULL) + http_client_connection_ref(conn); + http_client_request_ref(req); + req->payload_wait = TRUE; + + if (data == NULL) { + req->payload_input = NULL; + if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT) + (void)http_client_request_finish_payload_out(req); + } else { + req->payload_input = i_stream_create_from_data(data, size); + i_stream_set_name(req->payload_input, "<HTTP request payload>"); + } + req->payload_size = 0; + req->payload_chunked = TRUE; + + if (req->state == HTTP_REQUEST_STATE_NEW) + http_client_request_submit(req); + if (req->state == HTTP_REQUEST_STATE_ABORTED) { + /* Request already failed */ + if (req->delayed_error != NULL) { + struct http_client_request *tmpreq = req; + + /* Handle delayed error outside ioloop; the caller + expects callbacks occurring, so there is no need for + delay. Also, it is very important that any error + triggers a callback before + http_client_request_send_payload() finishes, since + its return value is not always checked. + */ + http_client_remove_request_error(client, req); + http_client_request_error_delayed(&tmpreq); + } + } else { + /* Wait for payload data to be written */ + + prev_ioloop = current_ioloop; + client_ioloop = io_loop_create(); + prev_client_ioloop = http_client_switch_ioloop(client); + if (client->set.dns_client != NULL) + dns_client_switch_ioloop(client->set.dns_client); + + client->waiting = TRUE; + while (req->state < HTTP_REQUEST_STATE_PAYLOAD_IN) { + e_debug(req->event, "Waiting for request to finish"); + + if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT) { + o_stream_set_flush_pending( + req->payload_output, TRUE); + } + + io_loop_run(client_ioloop); + + if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT && + req->payload_input->eof) { + i_stream_unref(&req->payload_input); + req->payload_input = NULL; + break; + } + } + client->waiting = FALSE; + + if (prev_client_ioloop != NULL) + io_loop_set_current(prev_client_ioloop); + else + io_loop_set_current(prev_ioloop); + (void)http_client_switch_ioloop(client); + if (client->set.dns_client != NULL) + dns_client_switch_ioloop(client->set.dns_client); + io_loop_set_current(client_ioloop); + io_loop_destroy(&client_ioloop); + } + + switch (req->state) { + case HTTP_REQUEST_STATE_PAYLOAD_IN: + case HTTP_REQUEST_STATE_FINISHED: + ret = 1; + break; + case HTTP_REQUEST_STATE_ABORTED: + ret = -1; + break; + default: + ret = 0; + break; + } + + req->payload_wait = FALSE; + + /* callback may have messed with our pointer, so unref using local + variable */ + if (!http_client_request_unref(&req)) + *_req = NULL; + + if (conn != NULL) + http_client_connection_unref(&conn); + + return ret; +} + +int http_client_request_send_payload(struct http_client_request **_req, + const unsigned char *data, size_t size) +{ + struct http_client_request *req = *_req; + int ret; + + i_assert(data != NULL); + + ret = http_client_request_continue_payload(&req, data, size); + if (ret < 0) { + /* Failed to send payload */ + *_req = NULL; + } else if (ret > 0) { + /* Premature end of request; + server sent error before all payload could be sent */ + ret = -1; + *_req = NULL; + } else { + /* Not finished sending payload */ + i_assert(req != NULL); + } + return ret; +} + +int http_client_request_finish_payload(struct http_client_request **_req) +{ + struct http_client_request *req = *_req; + int ret; + + *_req = NULL; + ret = http_client_request_continue_payload(&req, NULL, 0); + i_assert(ret != 0); + return ret < 0 ? -1 : 0; +} + +static void http_client_request_payload_input(struct http_client_request *req) +{ + struct http_client_connection *conn = req->conn; + + io_remove(&conn->io_req_payload); + + (void)http_client_connection_output(conn); +} + +int http_client_request_send_more(struct http_client_request *req, + bool pipelined) +{ + struct http_client_connection *conn = req->conn; + struct http_client_context *cctx = conn->ppool->peer->cctx; + struct ostream *output = req->payload_output; + enum ostream_send_istream_result res; + const char *error; + uoff_t offset; + + if (req->payload_finished) + return http_client_request_finish_payload_out(req); + + i_assert(req->payload_input != NULL); + i_assert(req->payload_output != NULL); + + io_remove(&conn->io_req_payload); + + /* Chunked ostream needs to write to the parent stream's buffer */ + offset = req->payload_input->v_offset; + o_stream_set_max_buffer_size(output, IO_BLOCK_SIZE); + res = o_stream_send_istream(output, req->payload_input); + o_stream_set_max_buffer_size(output, SIZE_MAX); + + i_assert(req->payload_input->v_offset >= offset); + e_debug(req->event, "Send more (sent %"PRIuUOFF_T", buffered=%zu)", + (uoff_t)(req->payload_input->v_offset - offset), + o_stream_get_buffer_used_size(output)); + + switch (res) { + case OSTREAM_SEND_ISTREAM_RESULT_FINISHED: + /* Finished sending */ + if (!req->payload_chunked && + (req->payload_input->v_offset - req->payload_offset) != + req->payload_size) { + error = t_strdup_printf( + "BUG: stream '%s' input size changed: " + "%"PRIuUOFF_T"-%"PRIuUOFF_T" != %"PRIuUOFF_T, + i_stream_get_name(req->payload_input), + req->payload_input->v_offset, + req->payload_offset, req->payload_size); + i_error("%s", error); //FIXME: remove? + http_client_connection_lost(&conn, error); + return -1; + } + + if (req->payload_wait) { + /* This chunk of input is finished + (client needs to act; disable timeout) */ + i_assert(!pipelined); + conn->output_locked = TRUE; + http_client_connection_stop_request_timeout(conn); + if (req->client != NULL && req->client->waiting) + io_loop_stop(req->client->ioloop); + return 0; + } + /* Finished sending payload */ + return http_client_request_finish_payload_out(req); + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT: + /* Input is blocking (client needs to act; disable timeout) */ + conn->output_locked = TRUE; + if (!pipelined) + http_client_connection_stop_request_timeout(conn); + conn->io_req_payload = io_add_istream_to( + cctx->ioloop, req->payload_input, + http_client_request_payload_input, req); + return 1; + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT: + /* Output is blocking (server needs to act; enable timeout) */ + conn->output_locked = TRUE; + if (!pipelined) + http_client_connection_start_request_timeout(conn); + e_debug(req->event, "Partially sent payload"); + return 0; + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT: + /* We're in the middle of sending a request, so the connection + will also have to be aborted */ + error = t_strdup_printf("read(%s) failed: %s", + i_stream_get_name(req->payload_input), + i_stream_get_error(req->payload_input)); + + /* The payload stream assigned to this request is broken, fail + this the request immediately */ + http_client_request_error(&req, + HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD, + "Broken payload stream"); + + http_client_connection_lost(&conn, error); + return -1; + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT: + /* Failed to send request */ + http_client_connection_handle_output_error(conn); + return -1; + } + i_unreached(); +} + +static int +http_client_request_send_real(struct http_client_request *req, bool pipelined) +{ + const struct http_client_settings *set = &req->client->set; + struct http_client_connection *conn = req->conn; + string_t *rtext = t_str_new(256); + struct const_iovec iov[3]; + int ret; + + i_assert(!req->conn->output_locked); + i_assert(req->payload_output == NULL); + + /* Create request line */ + str_append(rtext, req->method); + str_append(rtext, " "); + str_append(rtext, req->target); + str_append(rtext, " HTTP/1.1\r\n"); + + /* Create special headers implicitly if not set explicitly using + http_client_request_add_header() */ + if (!req->have_hdr_host) { + str_append(rtext, "Host: "); + str_append(rtext, req->authority); + str_append(rtext, "\r\n"); + } + if (!req->have_hdr_date) { + str_append(rtext, "Date: "); + str_append(rtext, http_date_create(req->date)); + str_append(rtext, "\r\n"); + } + if (!req->have_hdr_authorization && + req->username != NULL && req->password != NULL) { + struct http_auth_credentials auth_creds; + + http_auth_basic_credentials_init(&auth_creds, + req->username, req->password); + + str_append(rtext, "Authorization: "); + http_auth_create_credentials(rtext, &auth_creds); + str_append(rtext, "\r\n"); + } + if (http_client_request_to_proxy(req) && + set->proxy_username != NULL && set->proxy_password != NULL) { + struct http_auth_credentials auth_creds; + + http_auth_basic_credentials_init(&auth_creds, + set->proxy_username, set->proxy_password); + + str_append(rtext, "Proxy-Authorization: "); + http_auth_create_credentials(rtext, &auth_creds); + str_append(rtext, "\r\n"); + } + if (!req->have_hdr_user_agent && req->client->set.user_agent != NULL) { + str_printfa(rtext, "User-Agent: %s\r\n", + req->client->set.user_agent); + } + if (!req->have_hdr_expect && req->payload_sync) { + str_append(rtext, "Expect: 100-continue\r\n"); + } + if (req->payload_input != NULL && req->payload_chunked) { + // FIXME: can't do this for a HTTP/1.0 server + if (!req->have_hdr_body_spec) + str_append(rtext, "Transfer-Encoding: chunked\r\n"); + req->payload_output = + http_transfer_chunked_ostream_create(conn->conn.output); + o_stream_set_finish_also_parent(req->payload_output, FALSE); + } else if (req->payload_input != NULL || + req->payload_empty || + strcasecmp(req->method, "POST") == 0 || + strcasecmp(req->method, "PUT") == 0) { + + /* Send Content-Length if we have specified a payload or when + one is normally expected, even if it's 0 bytes. */ + i_assert(req->payload_input != NULL || req->payload_size == 0); + if (!req->have_hdr_body_spec) { + str_printfa(rtext, "Content-Length: %"PRIuUOFF_T"\r\n", + req->payload_size); + } + if (req->payload_input != NULL) { + req->payload_output = conn->conn.output; + o_stream_ref(conn->conn.output); + } + } + if (!req->have_hdr_connection && + !http_client_request_to_proxy(req)) { + /* RFC 2068, Section 19.7.1: + + A client MUST NOT send the Keep-Alive connection token to a + proxy server as HTTP/1.0 proxy servers do not obey the rules + of HTTP/1.1 for parsing the Connection header field. + */ + str_append(rtext, "Connection: Keep-Alive\r\n"); + } + + /* Request line + implicit headers */ + iov[0].iov_base = str_data(rtext); + iov[0].iov_len = str_len(rtext); + /* Explicit headers */ + if (req->headers != NULL) { + iov[1].iov_base = str_data(req->headers); + iov[1].iov_len = str_len(req->headers); + } else { + iov[1].iov_base = ""; + iov[1].iov_len = 0; + } + /* End of header */ + iov[2].iov_base = "\r\n"; + iov[2].iov_len = 2; + + req->state = HTTP_REQUEST_STATE_PAYLOAD_OUT; + req->payload_finished = FALSE; + + req->send_attempts++; + if (req->first_sent_time.tv_sec == 0) + req->first_sent_time = ioloop_timeval; + req->sent_time = ioloop_timeval; + req->sent_lock_usecs = file_lock_wait_get_total_usecs(); + req->sent_global_ioloop_usecs = ioloop_global_wait_usecs; + req->sent_http_ioloop_usecs = + io_wait_timer_get_usecs(req->conn->io_wait_timer); + + ret = 1; + o_stream_cork(conn->conn.output); + req->request_offset = conn->conn.output->offset; + + if (o_stream_sendv(conn->conn.output, iov, N_ELEMENTS(iov)) < 0) { + http_client_connection_handle_output_error(conn); + return -1; + } + + e_debug(req->event, "Sent header"); + + if (req->payload_output != NULL) { + if (!req->payload_sync) { + ret = http_client_request_send_more(req, pipelined); + if (ret < 0) + return -1; + } else { + e_debug(req->event, "Waiting for 100-continue"); + conn->output_locked = TRUE; + } + } else { + req->state = HTTP_REQUEST_STATE_WAITING; + if (!pipelined) + http_client_connection_start_request_timeout(req->conn); + conn->output_locked = FALSE; + } + if (conn->conn.output != NULL) { + i_assert(req->request_offset < conn->conn.output->offset); + req->bytes_out = conn->conn.output->offset - req->request_offset; + if (o_stream_uncork_flush(conn->conn.output) < 0) { + http_client_connection_handle_output_error(conn); + return -1; + } + } + return ret; +} + +int http_client_request_send(struct http_client_request *req, bool pipelined) +{ + int ret; + + T_BEGIN { + ret = http_client_request_send_real(req, pipelined); + } T_END; + + return ret; +} + +bool http_client_request_callback(struct http_client_request *req, + struct http_response *response) +{ + http_client_request_callback_t *callback = req->callback; + unsigned int orig_attempts = req->attempts; + + req->state = HTTP_REQUEST_STATE_GOT_RESPONSE; + req->last_status = response->status; + + req->callback = NULL; + if (callback != NULL) { + struct http_response response_copy = *response; + + if (req->attempts > 0 && !req->preserve_exact_reason) { + unsigned int total_msecs = + timeval_diff_msecs(&ioloop_timeval, + &req->submit_time); + response_copy.reason = t_strdup_printf( + "%s (%u retries in %u.%03u secs)", + response_copy.reason, req->attempts, + total_msecs/1000, total_msecs%1000); + } + + callback(&response_copy, req->context); + if (req->attempts != orig_attempts) { + /* Retrying */ + req->callback = callback; + http_client_request_resubmit(req); + return FALSE; + } else { + /* Release payload early + (prevents server/client deadlock in proxy) */ + i_stream_unref(&req->payload_input); + } + } + return TRUE; +} + +static bool +http_client_request_send_error(struct http_client_request *req, + unsigned int status, const char *error) +{ + http_client_request_callback_t *callback; + bool sending = (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT); + unsigned int orig_attempts = req->attempts; + + req->state = HTTP_REQUEST_STATE_ABORTED; + + callback = req->callback; + req->callback = NULL; + if (callback != NULL) { + struct http_response response; + + http_response_init(&response, status, error); + (void)callback(&response, req->context); + + if (req->attempts != orig_attempts) { + /* Retrying */ + req->callback = callback; + http_client_request_resubmit(req); + return FALSE; + } else { + /* Release payload early + (prevents server/client deadlock in proxy) */ + if (!sending && req->payload_input != NULL) + i_stream_unref(&req->payload_input); + } + } + if (req->payload_wait) { + i_assert(req->client != NULL); + io_loop_stop(req->client->ioloop); + } + return TRUE; +} + +void http_client_request_error_delayed(struct http_client_request **_req) +{ + struct http_client_request *req = *_req; + const char *error = req->delayed_error; + unsigned int status = req->delayed_error_status; + bool destroy; + + i_assert(req->state == HTTP_REQUEST_STATE_ABORTED); + + *_req = NULL; + req->delayed_error = NULL; + req->delayed_error_status = 0; + + i_assert(error != NULL && status != 0); + destroy = http_client_request_send_error(req, status, error); + if (req->queue != NULL) + http_client_queue_drop_request(req->queue, req); + if (destroy) + http_client_request_destroy(&req); +} + +void http_client_request_error(struct http_client_request **_req, + unsigned int status, const char *error) +{ + struct http_client_request *req = *_req; + + *_req = NULL; + + i_assert(req->delayed_error_status == 0); + i_assert(req->state < HTTP_REQUEST_STATE_FINISHED); + + req->state = HTTP_REQUEST_STATE_ABORTED; + req->last_status = status; + + e_debug(http_client_request_result_event(req)-> + set_name("http_request_finished")->event(), + "Error: %u %s", status, error); + + if (req->queue != NULL) + http_client_queue_drop_request(req->queue, req); + + if (req->client != NULL && + (!req->submitted || + req->state == HTTP_REQUEST_STATE_GOT_RESPONSE)) { + /* We're still in http_client_request_submit() or in the + callback during a retry attempt. delay reporting the error, + so the caller doesn't have to handle immediate or nested + callbacks. */ + req->delayed_error = p_strdup(req->pool, error); + req->delayed_error_status = status; + http_client_delay_request_error(req->client, req); + } else { + if (http_client_request_send_error(req, status, error)) + http_client_request_destroy(&req); + } +} + +void http_client_request_abort(struct http_client_request **_req) +{ + struct http_client_request *req = *_req; + bool sending; + + if (req == NULL) + return; + + sending = (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT); + + *_req = NULL; + + if (req->state >= HTTP_REQUEST_STATE_FINISHED && + req->delayed_error_status == 0) + return; + + req->callback = NULL; + req->state = HTTP_REQUEST_STATE_ABORTED; + if (req->last_status == 0) + req->last_status = HTTP_CLIENT_REQUEST_ERROR_ABORTED; + + if (req->state > HTTP_REQUEST_STATE_NEW && + req->delayed_error_status == 0) { + e_debug(http_client_request_result_event(req)-> + set_name("http_request_finished")->event(), + "Aborted"); + } + + /* Release payload early (prevents server/client deadlock in proxy) */ + if (!sending && req->payload_input != NULL) + i_stream_unref(&req->payload_input); + + if (req->queue != NULL) + http_client_queue_drop_request(req->queue, req); + if (req->payload_wait) { + i_assert(req->client != NULL); + i_assert(req->client->ioloop != NULL); + io_loop_stop(req->client->ioloop); + } + http_client_request_destroy(&req); +} + +void http_client_request_finish(struct http_client_request *req) +{ + if (req->state >= HTTP_REQUEST_STATE_FINISHED) + return; + + i_assert(req->refcount > 0); + + e_debug(http_client_request_result_event(req)-> + set_name("http_request_finished")->event(), + "Finished"); + + req->callback = NULL; + req->state = HTTP_REQUEST_STATE_FINISHED; + + if (req->queue != NULL) + http_client_queue_drop_request(req->queue, req); + if (req->payload_wait) { + i_assert(req->client != NULL); + i_assert(req->client->ioloop != NULL); + io_loop_stop(req->client->ioloop); + } + http_client_request_unref(&req); +} + +static int +http_client_request_reset(struct http_client_request *req, bool rewind, + const char **error_r) +{ + /* Rewind payload stream */ + if (rewind && req->payload_input != NULL && req->payload_size > 0) { + if (req->payload_input->v_offset != req->payload_offset && + !req->payload_input->seekable) { + *error_r = "Cannot resend payload; " + "stream is not seekable"; + return -1; + } + i_stream_seek(req->payload_input, req->payload_offset); + } + + /* Drop payload output stream from previous attempt */ + o_stream_unref(&req->payload_output); + + /* Reset payload state */ + req->payload_finished = FALSE; + + return 0; +} + +void http_client_request_redirect(struct http_client_request *req, + unsigned int status, const char *location) +{ + struct http_url *url; + const char *error, *target, *origin_url; + + i_assert(req->client != NULL); + i_assert(!req->payload_wait); + + req->last_status = status; + + /* parse URL */ + if (http_url_parse(location, NULL, 0, + pool_datastack_create(), &url, &error) < 0) { + http_client_request_error( + &req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT, + t_strdup_printf("Invalid redirect location: %s", + error)); + return; + } + + i_assert(req->redirects <= req->client->set.max_redirects); + if (++req->redirects > req->client->set.max_redirects) { + if (req->client->set.max_redirects > 0) { + http_client_request_error( + &req, + HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT, + t_strdup_printf( + "Redirected more than %d times", + req->client->set.max_redirects)); + } else { + http_client_request_error( + &req, + HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT, + "Redirect refused"); + } + return; + } + + if (http_client_request_reset(req, (status != 303), &error) < 0) { + http_client_request_error( + &req, HTTP_CLIENT_REQUEST_ERROR_ABORTED, + t_strdup_printf("Redirect failed: %s", error)); + return; + } + + target = http_url_create_target(url); + + http_url_copy(req->pool, &req->origin_url, url); + req->target = p_strdup(req->pool, target); + + req->host = NULL; + + origin_url = http_url_create(&req->origin_url); + + e_debug(http_client_request_result_event(req)-> + set_name("http_request_redirected")->event(), + "Redirecting to %s%s (redirects=%u)", + origin_url, target, req->redirects); + + req->label = p_strdup_printf(req->pool, "[%s %s%s]", + req->method, origin_url, req->target); + + /* RFC 7231, Section 6.4.4: + + -> A 303 `See Other' redirect status response is handled a bit + differently. Basically, the response content is located elsewhere, + but the original (POST) request is handled already. + */ + if (status == 303 && strcasecmp(req->method, "HEAD") != 0 && + strcasecmp(req->method, "GET") != 0) { + // FIXME: should we provide the means to skip this step? The + // original request was already handled at this point. + req->method = p_strdup(req->pool, "GET"); + + /* drop payload */ + i_stream_unref(&req->payload_input); + req->payload_size = 0; + req->payload_offset = 0; + } + + /* Resubmit */ + req->state = HTTP_REQUEST_STATE_NEW; + http_client_request_do_submit(req); +} + +void http_client_request_resubmit(struct http_client_request *req) +{ + const char *error; + + i_assert(!req->payload_wait); + + e_debug(req->event, "Resubmitting request"); + + if (http_client_request_reset(req, TRUE, &error) < 0) { + http_client_request_error( + &req, HTTP_CLIENT_REQUEST_ERROR_ABORTED, + t_strdup_printf("Resubmission failed: %s", error)); + return; + } + + req->peer = NULL; + req->state = HTTP_REQUEST_STATE_QUEUED; + req->redirects = 0; + req->last_status = 0; + http_client_host_submit_request(req->host, req); +} + +void http_client_request_retry(struct http_client_request *req, + unsigned int status, const char *error) +{ + if (req->client == NULL || req->client->set.no_auto_retry || + !http_client_request_try_retry(req)) + http_client_request_error(&req, status, error); +} + +bool http_client_request_try_retry(struct http_client_request *req) +{ + /* Don't ever retry if we're sending data in small blocks via + http_client_request_send_payload() and we're not waiting for a + 100 continue (there's no way to rewind the payload for a retry) + */ + if (req->payload_wait && + (!req->payload_sync || req->payload_sync_continue)) + return FALSE; + /* Limit the number of attempts for each request */ + if (req->attempts+1 >= req->max_attempts) + return FALSE; + req->attempts++; + + e_debug(http_client_request_result_event(req)-> + set_name("http_request_retried")->event(), + "Retrying (attempts=%d)", req->attempts); + + if (req->callback != NULL) + http_client_request_resubmit(req); + return TRUE; +} + +#undef http_client_request_set_destroy_callback +void http_client_request_set_destroy_callback(struct http_client_request *req, + void (*callback)(void *), + void *context) +{ + req->destroy_callback = callback; + req->destroy_context = context; +} + +void http_client_request_start_tunnel(struct http_client_request *req, + struct http_client_tunnel *tunnel) +{ + struct http_client_connection *conn = req->conn; + + i_assert(req->state == HTTP_REQUEST_STATE_GOT_RESPONSE); + + http_client_connection_start_tunnel(&conn, tunnel); +} diff --git a/src/lib-http/http-client.c b/src/lib-http/http-client.c new file mode 100644 index 0000000..5e71b63 --- /dev/null +++ b/src/lib-http/http-client.c @@ -0,0 +1,740 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "net.h" +#include "str.h" +#include "hash.h" +#include "llist.h" +#include "array.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "connection.h" +#include "dns-lookup.h" +#include "iostream-rawlog.h" +#include "iostream-ssl.h" +#include "http-url.h" + +#include "http-client-private.h" + +/* Structure: + + http_client_context: + + Shared context between multiple independent HTTP clients. This allows host + name lookup data, peer status and idle connections to be shared between + clients. + + http_client: + + Acts much like a browser; it is not dedicated to a single host. Client can + accept requests to different hosts, which can be served at different IPs. + Redirects are handled in the background by making a new connection. + Connections to new hosts are created once needed for servicing a request. + + http_client_request: + + The request semantics are similar to imapc commands. Create a request, + optionally modify some aspects of it, and finally submit it. Once finished, + a callback is called with the returned response. + + http_client_host_shared: + + We maintain a 'cache' of hosts for which we have looked up IPs. This cache + is maintained in client context, so multiple clients can share it. One host + can have multiple IPs. + + http_client_host: + + A host object maintains client-specific information for a host. The queues + that the client has for this host are listed here. For one host, there is a + separate queue for each used server port. + + http_client_queue: + + Requests are queued in a queue object. These queues are maintained for each + host:port target and listed in the host object. The queue object is + responsible for starting connection attempts to TCP port at the various IPs + known for the host. + + http_client_peer_pool: + + A peer pool lists all unused and pending connections to a peer, grouped by + a compatible configuration, e.g. in terms of SSL and rawlog. Once needed, + peers can claim/request an existing/new connection from the pool. + + http_client_peer_shared: + + The shared peer object records state information about a peer, which is a + service access point (ip:port or unix socket path). The peer object also + maintains lists of idle and pending connections to this service, which are + grouped in pools with compatible client configuration. Each client has a + separate (non-shared) peer object for client-specific state information. + + http_client_peer: + + A peer object maintains client-specific information for a peer. Claimed + connections are dedicated to one peer (and therefore one client). + + http-client-connection: + + This is an actual connection to a server. Once a connection is ready to + handle requests, it claims a request from a queue object. One connection can + service multiple hosts and one host can have multiple associated connections, + possibly to different ips and ports. + + */ + +static struct event_category event_category_http_client = { + .name = "http-client" +}; + +static struct http_client_context *http_client_global_context = NULL; + +static void +http_client_context_add_client(struct http_client_context *cctx, + struct http_client *client); +static void +http_client_context_remove_client(struct http_client_context *cctx, + struct http_client *client); + +/* + * Client + */ + +struct http_client * +http_client_init_shared(struct http_client_context *cctx, + const struct http_client_settings *set) +{ + static unsigned int id = 0; + struct http_client *client; + const char *log_prefix; + pool_t pool; + size_t pool_size; + + pool_size = (set != NULL && set->ssl != NULL) ? 8192 : 1024; /* certs will be >4K */ + pool = pool_alloconly_create("http client", pool_size); + client = p_new(pool, struct http_client, 1); + client->pool = pool; + client->ioloop = current_ioloop; + + /* create private context if none is provided */ + id++; + if (cctx != NULL) { + client->cctx = cctx; + http_client_context_ref(cctx); + log_prefix = t_strdup_printf("http-client[%u]: ", id); + } else { + i_assert(set != NULL); + client->cctx = cctx = http_client_context_create(set); + log_prefix = "http-client: "; + } + + struct event *parent_event; + if (set != NULL && set->event_parent != NULL) + parent_event = set->event_parent; + else if (cctx->event == NULL) + parent_event = NULL; + else { + /* FIXME: we could use cctx->event, but it already has a log + prefix that we don't want.. should we update event API to + support replacing parent's log prefix? */ + parent_event = event_get_parent(cctx->event); + } + client->event = event_create(parent_event); + event_add_category(client->event, &event_category_http_client); + event_set_forced_debug(client->event, + (set != NULL && set->debug) || (cctx != NULL && cctx->set.debug)); + event_set_append_log_prefix(client->event, log_prefix); + + /* merge provided settings with context defaults */ + client->set = cctx->set; + if (set != NULL) { + client->set.dns_client = set->dns_client; + client->set.dns_client_socket_path = + p_strdup_empty(pool, set->dns_client_socket_path); + client->set.dns_ttl_msecs = set->dns_ttl_msecs; + + if (set->user_agent != NULL && *set->user_agent != '\0') + client->set.user_agent = p_strdup_empty(pool, set->user_agent); + if (set->rawlog_dir != NULL && *set->rawlog_dir != '\0') + client->set.rawlog_dir = p_strdup_empty(pool, set->rawlog_dir); + + if (set->ssl != NULL) + client->set.ssl = ssl_iostream_settings_dup(pool, set->ssl); + + if (set->proxy_socket_path != NULL && *set->proxy_socket_path != '\0') { + client->set.proxy_socket_path = p_strdup(pool, set->proxy_socket_path); + client->set.proxy_url = NULL; + } else if (set->proxy_url != NULL) { + client->set.proxy_url = http_url_clone(pool, set->proxy_url); + client->set.proxy_socket_path = NULL; + } + if (set->proxy_username != NULL && *set->proxy_username != '\0') { + client->set.proxy_username = p_strdup_empty(pool, set->proxy_username); + client->set.proxy_password = p_strdup(pool, set->proxy_password); + } else if (set->proxy_url != NULL && set->proxy_url->user != NULL && + *set->proxy_url->user != '\0') { + client->set.proxy_username = + p_strdup_empty(pool, set->proxy_url->user); + client->set.proxy_password = + p_strdup(pool, set->proxy_url->password); + } + + if (set->max_idle_time_msecs > 0) + client->set.max_idle_time_msecs = set->max_idle_time_msecs; + if (set->max_parallel_connections > 0) + client->set.max_parallel_connections = set->max_parallel_connections; + if (set->max_pipelined_requests > 0) + client->set.max_pipelined_requests = set->max_pipelined_requests; + if (set->max_attempts > 0) + client->set.max_attempts = set->max_attempts; + if (set->max_connect_attempts > 0) + client->set.max_connect_attempts = set->max_connect_attempts; + if (set->connect_backoff_time_msecs > 0) { + client->set.connect_backoff_time_msecs = + set->connect_backoff_time_msecs; + } + if (set->connect_backoff_max_time_msecs > 0) { + client->set.connect_backoff_max_time_msecs = + set->connect_backoff_max_time_msecs; + } + client->set.no_auto_redirect = + client->set.no_auto_redirect || set->no_auto_redirect; + client->set.no_auto_retry = + client->set.no_auto_retry || set->no_auto_retry; + client->set.no_ssl_tunnel = + client->set.no_ssl_tunnel || set->no_ssl_tunnel; + if (set->max_redirects > 0) + client->set.max_redirects = set->max_redirects; + if (set->request_absolute_timeout_msecs > 0) { + client->set.request_absolute_timeout_msecs = + set->request_absolute_timeout_msecs; + } + if (set->request_timeout_msecs > 0) + client->set.request_timeout_msecs = set->request_timeout_msecs; + if (set->connect_timeout_msecs > 0) + client->set.connect_timeout_msecs = set->connect_timeout_msecs; + if (set->soft_connect_timeout_msecs > 0) + client->set.soft_connect_timeout_msecs = set->soft_connect_timeout_msecs; + if (set->socket_send_buffer_size > 0) + client->set.socket_send_buffer_size = set->socket_send_buffer_size; + if (set->socket_recv_buffer_size > 0) + client->set.socket_recv_buffer_size = set->socket_recv_buffer_size; + if (set->max_auto_retry_delay_secs > 0) + client->set.max_auto_retry_delay_secs = set->max_auto_retry_delay_secs; + client->set.debug = client->set.debug || set->debug; + } + + i_array_init(&client->delayed_failing_requests, 1); + + http_client_context_add_client(cctx, client); + + return client; +} + +struct http_client * +http_client_init(const struct http_client_settings *set) +{ + return http_client_init_shared(http_client_get_global_context(), set); +} + +struct http_client * +http_client_init_private(const struct http_client_settings *set) +{ + return http_client_init_shared(NULL, set); +} + +void http_client_deinit(struct http_client **_client) +{ + struct http_client *client = *_client; + struct http_client_request *req; + struct http_client_host *host; + struct http_client_peer *peer; + + *_client = NULL; + + /* destroy requests without calling callbacks */ + req = client->requests_list; + while (req != NULL) { + struct http_client_request *next_req = req->next; + http_client_request_destroy(&req); + req = next_req; + } + i_assert(client->requests_count == 0); + + /* free peers */ + while (client->peers_list != NULL) { + peer = client->peers_list; + http_client_peer_close(&peer); + } + + /* free hosts */ + while (client->hosts_list != NULL) { + host = client->hosts_list; + http_client_host_free(&host); + } + + array_free(&client->delayed_failing_requests); + timeout_remove(&client->to_failing_requests); + + if (client->ssl_ctx != NULL) + ssl_iostream_context_unref(&client->ssl_ctx); + http_client_context_remove_client(client->cctx, client); + http_client_context_unref(&client->cctx); + event_unref(&client->event); + pool_unref(&client->pool); +} + +static void http_client_do_switch_ioloop(struct http_client *client) +{ + struct http_client_peer *peer; + struct http_client_host *host; + + /* move peers */ + for (peer = client->peers_list; peer != NULL; + peer = peer->client_next) + http_client_peer_switch_ioloop(peer); + + /* move hosts/queues */ + for (host = client->hosts_list; host != NULL; + host = host->client_next) + http_client_host_switch_ioloop(host); + + /* move timeouts */ + if (client->to_failing_requests != NULL) { + client->to_failing_requests = + io_loop_move_timeout(&client->to_failing_requests); + } +} + +struct ioloop *http_client_switch_ioloop(struct http_client *client) +{ + struct ioloop *prev_ioloop = client->ioloop; + + client->ioloop = current_ioloop; + + http_client_do_switch_ioloop(client); + http_client_context_switch_ioloop(client->cctx); + + return prev_ioloop; +} + +void http_client_wait(struct http_client *client) +{ + struct ioloop *prev_ioloop, *client_ioloop, *prev_client_ioloop; + + if (client->requests_count == 0) + return; + + prev_ioloop = current_ioloop; + client_ioloop = io_loop_create(); + prev_client_ioloop = http_client_switch_ioloop(client); + if (client->set.dns_client != NULL) + dns_client_switch_ioloop(client->set.dns_client); + /* either we're waiting for network I/O or we're getting out of a + callback using timeout_add_short(0) */ + i_assert(io_loop_have_ios(client_ioloop) || + io_loop_have_immediate_timeouts(client_ioloop)); + + client->waiting = TRUE; + do { + e_debug(client->event, + "Waiting for %d requests to finish", client->requests_count); + io_loop_run(client_ioloop); + } while (client->requests_count > 0); + client->waiting = FALSE; + + e_debug(client->event, "All requests finished"); + + if (prev_client_ioloop != NULL) + io_loop_set_current(prev_client_ioloop); + else + io_loop_set_current(prev_ioloop); + (void)http_client_switch_ioloop(client); + if (client->set.dns_client != NULL) + dns_client_switch_ioloop(client->set.dns_client); + io_loop_set_current(client_ioloop); + io_loop_destroy(&client_ioloop); +} + +unsigned int http_client_get_pending_request_count(struct http_client *client) +{ + return client->requests_count; +} + +int http_client_init_ssl_ctx(struct http_client *client, const char **error_r) +{ + const char *error; + + if (client->ssl_ctx != NULL) + return 0; + + if (client->set.ssl == NULL) { + *error_r = "Requested https connection, but no SSL settings given"; + return -1; + } + if (ssl_iostream_client_context_cache_get(client->set.ssl, &client->ssl_ctx, &error) < 0) { + *error_r = t_strdup_printf("Couldn't initialize SSL context: %s", + error); + return -1; + } + return 0; +} + +/* + * Delayed request errors + */ + +static void +http_client_handle_request_errors(struct http_client *client) +{ + struct http_client_request *req; + + timeout_remove(&client->to_failing_requests); + + array_foreach_elem(&client->delayed_failing_requests, req) { + i_assert(req->refcount == 1); + http_client_request_error_delayed(&req); + } + array_clear(&client->delayed_failing_requests); +} + +void http_client_delay_request_error(struct http_client *client, + struct http_client_request *req) +{ + if (client->to_failing_requests == NULL) { + client->to_failing_requests = + timeout_add_short_to(client->ioloop, 0, + http_client_handle_request_errors, client); + } + array_push_back(&client->delayed_failing_requests, &req); +} + +void http_client_remove_request_error(struct http_client *client, + struct http_client_request *req) +{ + struct http_client_request *const *reqs; + unsigned int i, count; + + reqs = array_get(&client->delayed_failing_requests, &count); + for (i = 0; i < count; i++) { + if (reqs[i] == req) { + array_delete(&client->delayed_failing_requests, i, 1); + return; + } + } +} + +/* + * Client shared context + */ + +struct http_client_context * +http_client_context_create(const struct http_client_settings *set) +{ + struct http_client_context *cctx; + pool_t pool; + size_t pool_size; + + pool_size = (set->ssl != NULL) ? 8192 : 1024; /* certs will be >4K */ + pool = pool_alloconly_create("http client context", pool_size); + cctx = p_new(pool, struct http_client_context, 1); + cctx->pool = pool; + cctx->refcount = 1; + cctx->ioloop = current_ioloop; + + cctx->event = event_create(set->event_parent); + event_add_category(cctx->event, &event_category_http_client); + event_set_forced_debug(cctx->event, set->debug); + event_set_append_log_prefix(cctx->event, "http-client: "); + + cctx->set.dns_client = set->dns_client; + cctx->set.dns_client_socket_path = + p_strdup_empty(pool, set->dns_client_socket_path); + cctx->set.dns_ttl_msecs = (set->dns_ttl_msecs == 0 ? + HTTP_CLIENT_DEFAULT_DNS_TTL_MSECS : set->dns_ttl_msecs); + cctx->set.user_agent = p_strdup_empty(pool, set->user_agent); + cctx->set.rawlog_dir = p_strdup_empty(pool, set->rawlog_dir); + + if (set->ssl != NULL) + cctx->set.ssl = ssl_iostream_settings_dup(pool, set->ssl); + + if (set->proxy_socket_path != NULL && *set->proxy_socket_path != '\0') { + cctx->set.proxy_socket_path = p_strdup(pool, set->proxy_socket_path); + } else if (set->proxy_url != NULL) { + cctx->set.proxy_url = http_url_clone(pool, set->proxy_url); + } + if (set->proxy_username != NULL && *set->proxy_username != '\0') { + cctx->set.proxy_username = p_strdup_empty(pool, set->proxy_username); + cctx->set.proxy_password = p_strdup(pool, set->proxy_password); + } else if (set->proxy_url != NULL) { + cctx->set.proxy_username = + p_strdup_empty(pool, set->proxy_url->user); + cctx->set.proxy_password = + p_strdup(pool, set->proxy_url->password); + } + + cctx->set.max_idle_time_msecs = set->max_idle_time_msecs; + cctx->set.max_pipelined_requests = + (set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1); + cctx->set.max_parallel_connections = + (set->max_parallel_connections > 0 ? set->max_parallel_connections : 1); + cctx->set.max_attempts = set->max_attempts; + cctx->set.max_connect_attempts = set->max_connect_attempts; + cctx->set.connect_backoff_time_msecs = + set->connect_backoff_time_msecs == 0 ? + HTTP_CLIENT_DEFAULT_BACKOFF_TIME_MSECS : + set->connect_backoff_time_msecs; + cctx->set.connect_backoff_max_time_msecs = + set->connect_backoff_max_time_msecs == 0 ? + HTTP_CLIENT_DEFAULT_BACKOFF_MAX_TIME_MSECS : + set->connect_backoff_max_time_msecs; + cctx->set.no_auto_redirect = set->no_auto_redirect; + cctx->set.no_auto_retry = set->no_auto_retry; + cctx->set.no_ssl_tunnel = set->no_ssl_tunnel; + cctx->set.max_redirects = set->max_redirects; + cctx->set.response_hdr_limits = set->response_hdr_limits; + cctx->set.request_absolute_timeout_msecs = + set->request_absolute_timeout_msecs; + cctx->set.request_timeout_msecs = + set->request_timeout_msecs == 0 ? + HTTP_CLIENT_DEFAULT_REQUEST_TIMEOUT_MSECS : + set->request_timeout_msecs; + cctx->set.connect_timeout_msecs = set->connect_timeout_msecs; + cctx->set.soft_connect_timeout_msecs = set->soft_connect_timeout_msecs; + cctx->set.max_auto_retry_delay_secs = set->max_auto_retry_delay_secs; + cctx->set.socket_send_buffer_size = set->socket_send_buffer_size; + cctx->set.socket_recv_buffer_size = set->socket_recv_buffer_size; + cctx->set.debug = set->debug; + + cctx->conn_list = http_client_connection_list_init(); + + hash_table_create(&cctx->hosts, default_pool, 0, str_hash, strcmp); + + hash_table_create(&cctx->peers, default_pool, 0, + http_client_peer_addr_hash, http_client_peer_addr_cmp); + + return cctx; +} + +void http_client_context_ref(struct http_client_context *cctx) +{ + cctx->refcount++; +} + +void http_client_context_unref(struct http_client_context **_cctx) +{ + struct http_client_context *cctx = *_cctx; + struct http_client_peer_shared *peer; + struct http_client_host_shared *hshared; + + *_cctx = NULL; + + i_assert(cctx->refcount > 0); + if (--cctx->refcount > 0) + return; + + /* free hosts */ + while (cctx->hosts_list != NULL) { + hshared = cctx->hosts_list; + http_client_host_shared_free(&hshared); + } + hash_table_destroy(&cctx->hosts); + + /* close all idle connections */ + while (cctx->peers_list != NULL) { + peer = cctx->peers_list; + http_client_peer_shared_close(&peer); + i_assert(peer == NULL); + } + hash_table_destroy(&cctx->peers); + + connection_list_deinit(&cctx->conn_list); + + event_unref(&cctx->event); + pool_unref(&cctx->pool); +} + +static unsigned int +http_client_get_dns_lookup_timeout_msecs(const struct http_client_settings *set) +{ + if (set->connect_timeout_msecs > 0) + return set->connect_timeout_msecs; + if (set->request_timeout_msecs > 0) + return set->request_timeout_msecs; + return HTTP_CLIENT_DEFAULT_DNS_LOOKUP_TIMEOUT_MSECS; +} + +static void +http_client_context_update_settings(struct http_client_context *cctx) +{ + struct http_client *client; + bool debug; + + /* revert back to context settings */ + cctx->dns_client = cctx->set.dns_client; + cctx->dns_client_socket_path = cctx->set.dns_client_socket_path; + cctx->dns_ttl_msecs = cctx->set.dns_ttl_msecs; + cctx->dns_lookup_timeout_msecs = + http_client_get_dns_lookup_timeout_msecs(&cctx->set); + debug = cctx->set.debug; + + i_assert(cctx->dns_ttl_msecs > 0); + i_assert(cctx->dns_lookup_timeout_msecs > 0); + + /* override with available client settings */ + for (client = cctx->clients_list; client != NULL; + client = client->next) { + unsigned dns_lookup_timeout_msecs = + http_client_get_dns_lookup_timeout_msecs(&client->set); + + if (cctx->dns_client == NULL) + cctx->dns_client = client->set.dns_client; + if (cctx->dns_client_socket_path == NULL) { + cctx->dns_client_socket_path = + client->set.dns_client_socket_path; + } + if (client->set.dns_ttl_msecs != 0 && + cctx->dns_ttl_msecs > client->set.dns_ttl_msecs) + cctx->dns_ttl_msecs = client->set.dns_ttl_msecs; + if (dns_lookup_timeout_msecs != 0 && + cctx->dns_lookup_timeout_msecs > dns_lookup_timeout_msecs) { + cctx->dns_lookup_timeout_msecs = + dns_lookup_timeout_msecs; + } + debug = debug || client->set.debug; + } + + event_set_forced_debug(cctx->event, debug); +} + +static void +http_client_context_add_client(struct http_client_context *cctx, + struct http_client *client) +{ + DLLIST_PREPEND(&cctx->clients_list, client); + http_client_context_update_settings(cctx); +} + +static void +http_client_context_remove_client(struct http_client_context *cctx, + struct http_client *client) +{ + DLLIST_REMOVE(&cctx->clients_list, client); + http_client_context_update_settings(cctx); + + if (cctx->ioloop != current_ioloop && + cctx->ioloop == client->ioloop && + cctx->clients_list != NULL) { + struct ioloop *prev_ioloop = current_ioloop; + + io_loop_set_current(cctx->clients_list->ioloop); + http_client_context_switch_ioloop(cctx); + io_loop_set_current(prev_ioloop); + } +} + +static void http_client_context_close(struct http_client_context *cctx) +{ + struct connection *_conn, *_conn_next; + struct http_client_host_shared *hshared; + struct http_client_peer_shared *pshared; + + /* Switching to NULL ioloop; + close all hosts, peers, and connections */ + i_assert(cctx->clients_list == NULL); + + _conn = cctx->conn_list->connections; + while (_conn != NULL) { + struct http_client_connection *conn = + (struct http_client_connection *)_conn; + _conn_next = _conn->next; + http_client_connection_close(&conn); + _conn = _conn_next; + } + while (cctx->hosts_list != NULL) { + hshared = cctx->hosts_list; + http_client_host_shared_free(&hshared); + } + while (cctx->peers_list != NULL) { + pshared = cctx->peers_list; + http_client_peer_shared_close(&pshared); + } +} + +static void +http_client_context_do_switch_ioloop(struct http_client_context *cctx) +{ + struct connection *_conn = cctx->conn_list->connections; + struct http_client_host_shared *hshared; + struct http_client_peer_shared *pshared; + + /* move connections */ + /* FIXME: we wouldn't necessarily need to switch all of them + immediately, only those that have requests now. but also connections + that get new requests before ioloop is switched again.. */ + for (; _conn != NULL; _conn = _conn->next) { + struct http_client_connection *conn = + (struct http_client_connection *)_conn; + + http_client_connection_switch_ioloop(conn); + } + + /* move backoff timeouts */ + for (pshared = cctx->peers_list; pshared != NULL; + pshared = pshared->next) + http_client_peer_shared_switch_ioloop(pshared); + + /* move dns lookups and delayed requests */ + for (hshared = cctx->hosts_list; hshared != NULL; + hshared = hshared->next) + http_client_host_shared_switch_ioloop(hshared); +} + +void http_client_context_switch_ioloop(struct http_client_context *cctx) +{ + cctx->ioloop = current_ioloop; + + http_client_context_do_switch_ioloop(cctx); +} + +static void +http_client_global_context_ioloop_switched( + struct ioloop *prev_ioloop ATTR_UNUSED) +{ + struct http_client_context *cctx = http_client_global_context; + + i_assert(cctx != NULL); + if (current_ioloop == NULL) { + http_client_context_close(cctx); + return; + } + if (cctx->clients_list == NULL) { + /* follow the current ioloop if there is no client */ + http_client_context_switch_ioloop(cctx); + } +} + +static void http_client_global_context_free(void) +{ + /* drop ioloop switch callback to make absolutely sure there is no + recursion. */ + io_loop_remove_switch_callback(http_client_global_context_ioloop_switched); + + http_client_context_unref(&http_client_global_context); +} + +struct http_client_context *http_client_get_global_context(void) +{ + if (http_client_global_context != NULL) + return http_client_global_context; + + struct http_client_settings set; + i_zero(&set); + http_client_global_context = http_client_context_create(&set); + /* keep this a bit higher than lib-ssl-iostream */ + lib_atexit_priority(http_client_global_context_free, LIB_ATEXIT_PRIORITY_LOW-1); + io_loop_add_switch_callback(http_client_global_context_ioloop_switched); + return http_client_global_context; +} diff --git a/src/lib-http/http-client.h b/src/lib-http/http-client.h new file mode 100644 index 0000000..4f04222 --- /dev/null +++ b/src/lib-http/http-client.h @@ -0,0 +1,496 @@ +#ifndef HTTP_CLIENT_H +#define HTTP_CLIENT_H + +#include "net.h" + +#include "http-common.h" +#include "http-response.h" + +struct timeval; +struct http_response; + +struct http_client_request; +struct http_client; +struct http_client_context; + +struct ssl_iostream_settings; + +/* + * Client settings + */ + +struct http_client_settings { + /* a) If dns_client is set, all lookups are done via it. + b) If dns_client_socket_path is set, each DNS lookup does its own + dns-lookup UNIX socket connection. + c) Otherwise, blocking gethostbyname() lookups are used. */ + struct dns_client *dns_client; + const char *dns_client_socket_path; + /* How long to cache DNS records internally + (default = HTTP_CLIENT_DEFAULT_DNS_TTL_MSECS) */ + unsigned int dns_ttl_msecs; + + const struct ssl_iostream_settings *ssl; + + /* User-Agent: header (default: none) */ + const char *user_agent; + + /* proxy on unix socket */ + const char *proxy_socket_path; + /* URL for normal proxy (ignored if proxy_socket_path is set) */ + const struct http_url *proxy_url; + /* credentials for proxy */ + const char *proxy_username; + const char *proxy_password; + + /* directory for writing raw log data for debugging purposes */ + const char *rawlog_dir; + + /* maximum time a connection will idle. if parallel connections are idle, + the duplicates will end earlier based on how many idle connections exist + to that same service */ + unsigned int max_idle_time_msecs; + + /* maximum number of parallel connections per peer (default = 1) */ + unsigned int max_parallel_connections; + + /* maximum number of pipelined requests per connection (default = 1) */ + unsigned int max_pipelined_requests; + + /* don't automatically act upon redirect responses */ + bool no_auto_redirect; + + /* never automatically retry requests */ + bool no_auto_retry; + + /* if we use a proxy, delegate SSL negotiation to proxy, rather than + creating a CONNECT tunnel through the proxy for the SSL link */ + bool no_ssl_tunnel; + + /* maximum number of redirects for a request + (default = 0; redirects refused) + */ + unsigned int max_redirects; + + /* maximum number of attempts for a request */ + unsigned int max_attempts; + + /* maximum number of connection attempts to a host before all associated + requests fail. + + if > 1, the maximum will be enforced across all IPs for that host, + meaning that IPs may be tried more than once eventually if the number + of IPs is smaller than the specified maximum attempts. If the number of IPs + is higher than the maximum attempts, not all IPs are tried. If <= 1, all + IPs are tried at most once. + */ + unsigned int max_connect_attempts; + + /* Initial backoff time; doubled at each connection failure + (default = HTTP_CLIENT_DEFAULT_BACKOFF_TIME_MSECS) */ + unsigned int connect_backoff_time_msecs; + /* Maximum backoff time + (default = HTTP_CLIENT_DEFAULT_BACKOFF_MAX_TIME_MSECS) */ + unsigned int connect_backoff_max_time_msecs; + + /* response header limits */ + struct http_header_limits response_hdr_limits; + + /* max total time to wait for HTTP request to finish + this can be overridden/reset for individual requests using + http_client_request_set_timeout() and friends. + (default is no timeout) + */ + unsigned int request_absolute_timeout_msecs; + /* max time to wait for HTTP request to finish before retrying + (default = HTTP_CLIENT_DEFAULT_REQUEST_TIMEOUT_MSECS) */ + unsigned int request_timeout_msecs; + /* max time to wait for connect() (and SSL handshake) to finish before + retrying (default = request_timeout_msecs) */ + unsigned int connect_timeout_msecs; + /* time to wait for connect() (and SSL handshake) to finish for the first + connection before trying the next IP in parallel + (default = 0; wait until current connection attempt finishes) */ + unsigned int soft_connect_timeout_msecs; + + /* maximum acceptable delay in seconds for automatically + retrying/redirecting requests. if a server sends a response with a + Retry-After header that causes a delay longer than this, the request + is not automatically retried and the response is returned */ + unsigned int max_auto_retry_delay_secs; + + /* the kernel send/receive buffer sizes used for the connection sockets. + Configuring this is mainly useful for the test suite. The kernel + defaults are used when these settings are 0. */ + size_t socket_send_buffer_size; + size_t socket_recv_buffer_size; + + /* Event to use as parent for the http client event. For specific + requests this can be overridden with http_client_request_set_event(). + */ + struct event *event_parent; + + /* enable logging debug messages */ + bool debug; +}; + +/* + * Request + */ + +enum http_client_request_error { + /* The request was aborted */ + HTTP_CLIENT_REQUEST_ERROR_ABORTED = HTTP_RESPONSE_STATUS_INTERNAL, + /* Failed to parse HTTP target url */ + HTTP_CLIENT_REQUEST_ERROR_INVALID_URL, + /* Failed to perform DNS lookup for the host */ + HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED, + /* Failed to setup any connection for the host and client settings allowed + no more attempts */ + HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, + /* Service returned an invalid redirect response for this request */ + HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT, + /* The connection was lost unexpectedly while handling the request and + client settings allowed no more attempts */ + HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, + /* The input stream passed to the request using + http_client_request_set_payload() returned an error while sending the + request. */ + HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD, + /* The service returned a bad response */ + HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE, + /* The request timed out (either this was the last attempt or the + absolute timeout was hit) */ + HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT, +}; + +enum http_request_state { + /* New request; not yet submitted */ + HTTP_REQUEST_STATE_NEW = 0, + /* Request is queued; waiting for a connection */ + HTTP_REQUEST_STATE_QUEUED, + /* Request header is sent; still sending request payload to server */ + HTTP_REQUEST_STATE_PAYLOAD_OUT, + /* Request is fully sent; waiting for response */ + HTTP_REQUEST_STATE_WAITING, + /* Response header is received for the request */ + HTTP_REQUEST_STATE_GOT_RESPONSE, + /* Reading response payload; response handler still needs to read more + payload. */ + HTTP_REQUEST_STATE_PAYLOAD_IN, + /* Request is finished; still lingering due to references */ + HTTP_REQUEST_STATE_FINISHED, + /* Request is aborted; still lingering due to references */ + HTTP_REQUEST_STATE_ABORTED +}; +extern const char *http_request_state_names[]; + +struct http_client_tunnel { + int fd_in, fd_out; + struct istream *input; + struct ostream *output; +}; + +struct http_client_request_stats { + /* Total elapsed time since message was submitted */ + unsigned int total_msecs; + /* Elapsed time since message was first sent */ + unsigned int first_sent_msecs; + /* Elapsed time since message was last sent */ + unsigned int last_sent_msecs; + + /* Time spent in other ioloops */ + unsigned int other_ioloop_msecs; + /* Time spent in the http-client's own ioloop */ + unsigned int http_ioloop_msecs; + /* Total time spent on waiting for file locks */ + unsigned int lock_msecs; + + /* Number of times this request was retried */ + unsigned int attempts; + /* Number of times the client attempted to actually send the request + to a server */ + unsigned int send_attempts; +}; + +typedef void +http_client_request_callback_t(const struct http_response *response, + void *context); + +/* create new HTTP request */ +struct http_client_request * +http_client_request(struct http_client *client, + const char *method, const char *host, const char *target, + http_client_request_callback_t *callback, void *context); +#define http_client_request(client, method, host, target, callback, context) \ + http_client_request(client, method, host, target - \ + CALLBACK_TYPECHECK(callback, void (*)( \ + const struct http_response *response, typeof(context))), \ + (http_client_request_callback_t *)callback, context) + +/* create new HTTP request using provided URL. This implicitly sets + port, ssl, and username:password if provided. */ +struct http_client_request * +http_client_request_url(struct http_client *client, + const char *method, const struct http_url *target_url, + http_client_request_callback_t *callback, void *context); +#define http_client_request_url(client, method, target_url, callback, context) \ + http_client_request_url(client, method, target_url - \ + CALLBACK_TYPECHECK(callback, void (*)( \ + const struct http_response *response, typeof(context))), \ + (http_client_request_callback_t *)callback, context) +struct http_client_request * +http_client_request_url_str(struct http_client *client, + const char *method, const char *url_str, + http_client_request_callback_t *callback, void *context); +#define http_client_request_url_str(client, method, url_str, callback, context) \ + http_client_request_url_str(client, method, url_str - \ + CALLBACK_TYPECHECK(callback, void (*)( \ + const struct http_response *response, typeof(context))), \ + (http_client_request_callback_t *)callback, context) + +/* create new HTTP CONNECT request. If this HTTP is configured to use a proxy, + a CONNECT request will be submitted at that proxy, otherwise the connection + is created directly. Call http_client_request_start_tunnel() to + to take over the connection. + */ +struct http_client_request * +http_client_request_connect(struct http_client *client, + const char *host, in_port_t port, + http_client_request_callback_t *callback, + void *context); +#define http_client_request_connect(client, host, port, callback, context) \ + http_client_request_connect(client, host, port - \ + CALLBACK_TYPECHECK(callback, void (*)( \ + const struct http_response *response, typeof(context))), \ + (http_client_request_callback_t *)callback, context) + +/* same as http_client_request_connect, but uses an IP rather than a host + name. */ +struct http_client_request * +http_client_request_connect_ip(struct http_client *client, + const struct ip_addr *ip, in_port_t port, + http_client_request_callback_t *callback, + void *context); +#define http_client_request_connect_ip(client, ip, port, callback, context) \ + http_client_request_connect_ip(client, ip, port - \ + CALLBACK_TYPECHECK(callback, void (*)( \ + const struct http_response *response, typeof(context))), \ + (http_client_request_callback_t *)callback, context) + +void http_client_request_set_event(struct http_client_request *req, + struct event *event); +/* set the port for the service the request is directed at */ +void http_client_request_set_port(struct http_client_request *req, + in_port_t port); +/* indicate whether service the request is directed at uses ssl */ +void http_client_request_set_ssl(struct http_client_request *req, + bool ssl); +/* set the urgent flag: this means that this request will get priority over + non-urgent request. Also, if no idle connection is available, a new + connection is created. Urgent requests are never pipelined. */ +void http_client_request_set_urgent(struct http_client_request *req); +void http_client_request_set_preserve_exact_reason(struct http_client_request *req); + +/* add a custom header to the request. This can override headers that are + otherwise created implicitly. If the same header key was already added, + the value is replaced. */ +void http_client_request_add_header(struct http_client_request *req, + const char *key, const char *value); +/* add a custom header to the request. Do nothing if it was already added. */ +void http_client_request_add_missing_header(struct http_client_request *req, + const char *key, const char *value); +/* remove a header added earlier. This has no influence on implicitly created + headers. */ +void http_client_request_remove_header(struct http_client_request *req, + const char *key); +/* lookup the value for a header added earlier. Returns NULL if not found. */ +const char *http_client_request_lookup_header(struct http_client_request *req, + const char *key); + +/* set the value of the "Date" header for the request using a time_t value. + Use this instead of setting it directly using + http_client_request_add_header() */ +void http_client_request_set_date(struct http_client_request *req, + time_t date); + +/* assign an input stream for the outgoing payload of this request. The input + stream is read asynchronously while the request is sent to the server. + + when sync=TRUE a "100 Continue" response is requested from the service. The + client will then postpone sending the payload until a provisional response + with code 100 is received. This way, an error response can be sent by the + service before any potentially big payload is transmitted. Use this only for + payload that can be large. */ +void http_client_request_set_payload(struct http_client_request *req, + struct istream *input, bool sync); +/* assign payload data to the request. The data is copied to the request pool. + If your data is already durably allocated during the existence of the + request, you should consider using http_client_request_set_payload() with + a data input stream instead. This will avoid copying the data unnecessarily. + */ +void http_client_request_set_payload_data(struct http_client_request *req, + const unsigned char *data, size_t size); +/* send an empty payload for this request. This means that a Content-Length + header is generated with zero size. Calling this function is not necessary + for the standard POST and PUT methods, for which this is done implicitly if + there is no payload set. */ +void http_client_request_set_payload_empty(struct http_client_request *req); + +/* set an absolute timeout for this request specifically, overriding the + default client-wide absolute request timeout */ +void http_client_request_set_timeout_msecs(struct http_client_request *req, + unsigned int msecs); +void http_client_request_set_timeout(struct http_client_request *req, + const struct timeval *time); + +/* Override http_client_settings.request_timeout_msecs */ +void http_client_request_set_attempt_timeout_msecs(struct http_client_request *req, + unsigned int msecs); +/* Override http_client_settings.max_attempts */ +void http_client_request_set_max_attempts(struct http_client_request *req, + unsigned int max_attempts); + +/* Include the specified HTTP response headers in the http_request_finished + event parameters with "http_hdr_" prefix. */ +void http_client_request_set_event_headers(struct http_client_request *req, + const char *const *headers); + +/* set the username:password credentials for this request for simple + authentication. This function is meant for simple schemes that use a + password. More complex schemes will need to be handled manually. + + This currently only supports the "basic" authentication scheme. */ +void http_client_request_set_auth_simple(struct http_client_request *req, + const char *username, const char *password); + +/* Assign a proxy to use for this particular request. This overrides any + proxy defined in the client settings. */ +void http_client_request_set_proxy_url(struct http_client_request *req, + const struct http_url *proxy_url); +/* Like http_client_request_set_proxy_url(), but the proxy is behind a unix + socket. */ +void http_client_request_set_proxy_socket(struct http_client_request *req, + const char *proxy_socket); + +/* delay handling of this request to a later time. This way, a request can be + submitted that is held for some time until a certain time period has passed. + */ +void http_client_request_delay_until(struct http_client_request *req, + time_t time); +void http_client_request_delay(struct http_client_request *req, + time_t seconds); +void http_client_request_delay_msecs(struct http_client_request *req, + unsigned int msecs); + +/* Try to set request delay based on the Retry-After header. Returns 1 if + successful, 0 if it doesn't exist or is already expired, -1 if the delay + would be too long. */ +int http_client_request_delay_from_response(struct http_client_request *req, + const struct http_response *response); + +/* return the HTTP method for the request */ +const char * +http_client_request_get_method(const struct http_client_request *req) + ATTR_PURE; +/* return the HTTP target for the request */ +const char * +http_client_request_get_target(const struct http_client_request *req) + ATTR_PURE; +/* return the request state */ +enum http_request_state +http_client_request_get_state(const struct http_client_request *req) + ATTR_PURE; +/* return number of retry attempts */ +unsigned int +http_client_request_get_attempts(const struct http_client_request *req) + ATTR_PURE; +/* return origin_url */ +const struct http_url * +http_client_request_get_origin_url(const struct http_client_request *req) + ATTR_PURE; + +/* get statistics for the request */ +void http_client_request_get_stats(struct http_client_request *req, + struct http_client_request_stats *stats); +/* append text with request statistics to provided string buffer */ +void http_client_request_append_stats_text(struct http_client_request *req, + string_t *str); + +/* submit the request. It is queued for transmission to the service */ +void http_client_request_submit(struct http_client_request *req); + +/* attempt to retry the request. This function is called within the request + callback. It returns false if the request cannot be retried */ +bool http_client_request_try_retry(struct http_client_request *req); + +/* abort the request immediately. It may still linger for a while when it is + already sent to the service, but the callback will not be called anymore. */ +void http_client_request_abort(struct http_client_request **req); + +/* call the specified callback when HTTP request is destroyed. */ +void http_client_request_set_destroy_callback(struct http_client_request *req, + void (*callback)(void *), + void *context); +#define http_client_request_set_destroy_callback(req, callback, context) \ + http_client_request_set_destroy_callback(req, (void(*)(void*))callback, \ + TRUE ? context : \ + CALLBACK_TYPECHECK(callback, void (*)(typeof(context)))) + +/* submits request and blocks until the provided payload is sent. Multiple + calls are allowed; payload transmission is ended with + http_client_request_finish_payload(). If the sending fails, returns -1 + and sets req=NULL to indicate that the request was freed, otherwise + returns 0 and req is unchanged. */ +int http_client_request_send_payload(struct http_client_request **req, + const unsigned char *data, size_t size); +/* finish sending the payload. Always frees req and sets it to NULL. + Returns 0 on success, -1 on error. */ +int http_client_request_finish_payload(struct http_client_request **req); + +/* take over the connection this request was sent over for use as a HTTP + CONNECT tunnel. This only applies to requests that were created using + http_client_request_connect() or http_client_request_connect_ip(). */ +void http_client_request_start_tunnel(struct http_client_request *req, + struct http_client_tunnel *tunnel); + +/* + * Client + */ + +/* Create a client using the global shared client context. */ +struct http_client * +http_client_init(const struct http_client_settings *set); +/* Create a client without a shared context. */ +struct http_client * +http_client_init_private(const struct http_client_settings *set); +struct http_client * +http_client_init_shared(struct http_client_context *cctx, + const struct http_client_settings *set) ATTR_NULL(1); +void http_client_deinit(struct http_client **_client); + +/* switch this client to the current ioloop */ +struct ioloop *http_client_switch_ioloop(struct http_client *client); + +/* blocks until all currently submitted requests are handled */ +void http_client_wait(struct http_client *client); + +/* Returns the total number of pending HTTP requests. */ +unsigned int +http_client_get_pending_request_count(struct http_client *client); + +/* + * Client shared context + */ + +struct http_client_context * +http_client_context_create(const struct http_client_settings *set); +void http_client_context_ref(struct http_client_context *cctx); +void http_client_context_unref(struct http_client_context **_cctx); + +/* Return the default global shared client context, creating it if necessary. + The context is freed automatically at exit. Don't unreference the + returned context. */ +struct http_client_context *http_client_get_global_context(void); + +#endif diff --git a/src/lib-http/http-common.h b/src/lib-http/http-common.h new file mode 100644 index 0000000..9aa217f --- /dev/null +++ b/src/lib-http/http-common.h @@ -0,0 +1,7 @@ +#ifndef HTTP_COMMON_H +#define HTTP_COMMON_H + +#define HTTP_DEFAULT_PORT 80 +#define HTTPS_DEFAULT_PORT 443 + +#endif diff --git a/src/lib-http/http-date.c b/src/lib-http/http-date.c new file mode 100644 index 0000000..6f0c5a7 --- /dev/null +++ b/src/lib-http/http-date.c @@ -0,0 +1,487 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "utc-mktime.h" +#include "http-date.h" + +#include <ctype.h> + +/* RFC 7231, Section 7.1.1.1: Date/Time Formats + + The defined syntax is as follows: + + HTTP-date = IMF-fixdate / obs-date + + Preferred format: + + IMF-fixdate = day-name "," SP date1 SP time-of-day SP GMT + ; fixed length/zone/capitalization subset of the format + ; see Section 3.3 of [RFC5322] + day-name = %x4D.6F.6E ; "Mon", case-sensitive + / %x54.75.65 ; "Tue", case-sensitive + / %x57.65.64 ; "Wed", case-sensitive + / %x54.68.75 ; "Thu", case-sensitive + / %x46.72.69 ; "Fri", case-sensitive + / %x53.61.74 ; "Sat", case-sensitive + / %x53.75.6E ; "Sun", case-sensitive + date1 = day SP month SP year + ; e.g., 02 Jun 1982 + day = 2DIGIT + month = %x4A.61.6E ; "Jan", case-sensitive + / %x46.65.62 ; "Feb", case-sensitive + / %x4D.61.72 ; "Mar", case-sensitive + / %x41.70.72 ; "Apr", case-sensitive + / %x4D.61.79 ; "May", case-sensitive + / %x4A.75.6E ; "Jun", case-sensitive + / %x4A.75.6C ; "Jul", case-sensitive + / %x41.75.67 ; "Aug", case-sensitive + / %x53.65.70 ; "Sep", case-sensitive + / %x4F.63.74 ; "Oct", case-sensitive + / %x4E.6F.76 ; "Nov", case-sensitive + / %x44.65.63 ; "Dec", case-sensitive + year = 4DIGIT + GMT = %x47.4D.54 ; "GMT", case-sensitive + time-of-day = hour ":" minute ":" second + ; 00:00:00 - 23:59:60 (leap second) + hour = 2DIGIT + minute = 2DIGIT + second = 2DIGIT + + Obsolete formats: + + obs-date = rfc850-date / asctime-date + + rfc850-date = day-name-l "," SP date2 SP time-of-day SP GMT + date2 = day "-" month "-" 2DIGIT + ; e.g., 02-Jun-82 + day-name-l = %x4D.6F.6E.64.61.79 ; "Monday", case-sensitive + / %x54.75.65.73.64.61.79 ; "Tuesday", case-sensitive + / %x57.65.64.6E.65.73.64.61.79 ; "Wednesday", case-sensitive + / %x54.68.75.72.73.64.61.79 ; "Thursday", case-sensitive + / %x46.72.69.64.61.79 ; "Friday", case-sensitive + / %x53.61.74.75.72.64.61.79 ; "Saturday", case-sensitive + / %x53.75.6E.64.61.79 ; "Sunday", case-sensitive + + asctime-date = day-name SP date3 SP time-of-day SP year + date3 = month SP ( 2DIGIT / ( SP 1DIGIT )) + ; e.g., Jun 2 + */ + +static const char *month_names[] = { + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" +}; + +static const char *weekday_names[] = { + "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" +}; + +static const char *weekday_names_long[] = { + "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" +}; + +struct http_date_parser { + const unsigned char *cur, *end; + + struct tm tm; + int timezone_offset; +}; + +static inline int +http_date_parse_sp(struct http_date_parser *parser) +{ + if (parser->cur >= parser->end) + return -1; + if (parser->cur[0] != ' ') + return 0; + parser->cur++; + return 1; +} + +static inline int +http_date_parse_number(struct http_date_parser *parser, + int digits, int *number_r) +{ + int i; + + if (parser->cur >= parser->end || !i_isdigit(parser->cur[0])) + return 0; + + *number_r = parser->cur[0] - '0'; + parser->cur++; + + for (i=0; i < digits-1; i++) { + if (parser->cur >= parser->end || !i_isdigit(parser->cur[0])) + return -1; + *number_r = ((*number_r) * 10) + parser->cur[0] - '0'; + parser->cur++; + } + return 1; +} + +static inline int +http_date_parse_word(struct http_date_parser *parser, + int maxchars, string_t **word_r) +{ + string_t *word; + int i; + + if (parser->cur >= parser->end || !i_isalpha(parser->cur[0])) + return 0; + + word = t_str_new(maxchars); + str_append_c(word, parser->cur[0]); + parser->cur++; + + for (i=0; i < maxchars-1; i++) { + if (parser->cur >= parser->end || !i_isalpha(parser->cur[0])) + break; + str_append_c(word, parser->cur[0]); + parser->cur++; + } + + if (parser->cur < parser->end && i_isalpha(parser->cur[0])) + return -1; + *word_r = word; + return 1; +} + +static inline int +http_date_parse_year(struct http_date_parser *parser) +{ + /* year = 4DIGIT */ + if (http_date_parse_number(parser, 4, &parser->tm.tm_year) <= 0) + return -1; + if (parser->tm.tm_year < 1900) + return -1; + parser->tm.tm_year -= 1900; + return 1; +} + +static inline int +http_date_parse_month(struct http_date_parser *parser) +{ + string_t *month; + int i; + + if (http_date_parse_word(parser, 3, &month) <= 0 || str_len(month) != 3) + return -1; + + for (i = 0; i < 12; i++) { + if (strcmp(month_names[i], str_c(month)) == 0) { + break; + } + } + if (i >= 12) + return -1; + + parser->tm.tm_mon = i; + return 1; +} + +static inline int +http_date_parse_day(struct http_date_parser *parser) +{ + /* day = 2DIGIT */ + if (http_date_parse_number(parser, 2, &parser->tm.tm_mday) <= 0) + return -1; + return 1; +} + +static int +http_date_parse_time_of_day(struct http_date_parser *parser) +{ + /* time-of-day = hour ":" minute ":" second + ; 00:00:00 - 23:59:59 + hour = 2DIGIT + minute = 2DIGIT + second = 2DIGIT + */ + + /* hour = 2DIGIT */ + if (http_date_parse_number(parser, 2, &parser->tm.tm_hour) <= 0) + return -1; + + /* ":" */ + if (parser->cur >= parser->end || parser->cur[0] != ':') + return -1; + parser->cur++; + + /* minute = 2DIGIT */ + if (http_date_parse_number(parser, 2, &parser->tm.tm_min) <= 0) + return -1; + + /* ":" */ + if (parser->cur >= parser->end || parser->cur[0] != ':') + return -1; + parser->cur++; + + /* second = 2DIGIT */ + if (http_date_parse_number(parser, 2, &parser->tm.tm_sec) <= 0) + return -1; + return 1; +} + +static inline int +http_date_parse_time_gmt(struct http_date_parser *parser) +{ + string_t *gmt; + + /* Remaining: {...} SP time-of-day SP GMT + */ + + /* SP time-of-day */ + if (http_date_parse_sp(parser) <= 0) + return -1; + if (http_date_parse_time_of_day(parser) <= 0) + return -1; + + /* SP GMT */ + if (http_date_parse_sp(parser) <= 0) + return -1; + if (http_date_parse_word(parser, 3, &gmt) <= 0 || + strcmp("GMT", str_c(gmt)) != 0) + return -1; + return 1; +} + +static int +http_date_parse_format_imf_fixdate(struct http_date_parser *parser) +{ + /* + IMF-fixdate = day-name "," SP date1 SP time-of-day SP GMT + ; fixed length/zone/capitalization subset of the format + ; see Section 3.3 of [RFC5322] + date1 = day SP month SP year + ; e.g., 02 Jun 1982 + + Remaining: {...} SP day SP month SP year SP time-of-day SP GMT + + */ + + /* SP day */ + if (http_date_parse_sp(parser) <= 0) + return -1; + if (http_date_parse_day(parser) <= 0) + return -1; + + /* SP month */ + if (http_date_parse_sp(parser) <= 0) + return -1; + if (http_date_parse_month(parser) <= 0) + return -1; + + /* SP year */ + if (http_date_parse_sp(parser) <= 0) + return -1; + if (http_date_parse_year(parser) <= 0) + return -1; + + /* SP time-of-day SP GMT */ + return http_date_parse_time_gmt(parser); +} + +static int +http_date_parse_format_rfc850(struct http_date_parser *parser) +{ + /* + rfc850-date = day-name-l "," SP date2 SP time-of-day SP GMT + date2 = day "-" month "-" 2DIGIT + ; day-month-year (e.g., 02-Jun-82) + + Remaining: "," SP day "-" month "-" 2DIGIT SP time-of-day SP GMT + */ + + /* "," SP */ + if (parser->cur >= parser->end || parser->cur[0] != ',') + return -1; + parser->cur++; + if (http_date_parse_sp(parser) <= 0) + return -1; + + /* day */ + if (http_date_parse_day(parser) <= 0) + return -1; + + /* "-" */ + if (parser->cur >= parser->end || parser->cur[0] != '-') + return -1; + parser->cur++; + + /* month */ + if (http_date_parse_month(parser) <= 0) + return -1; + + /* "-" */ + if (parser->cur >= parser->end || parser->cur[0] != '-') + return -1; + parser->cur++; + + /* 2DIGIT */ + if (http_date_parse_number(parser, 2, &parser->tm.tm_year) <= 0) + return -1; + if (parser->tm.tm_year < 70) + parser->tm.tm_year += 100; + + /* SP time-of-day SP GMT */ + return http_date_parse_time_gmt(parser); +} + +static int +http_date_parse_format_asctime(struct http_date_parser *parser) +{ + int ret; + + /* + asctime-date = day-name SP date3 SP time-of-day SP year + date3 = month SP ( 2DIGIT / ( SP 1DIGIT )) + ; month day (e.g., Jun 2) + + Remaining: {...} month SP ( 2DIGIT / ( SP 1DIGIT )) SP time-of-day SP year + */ + + /* month */ + if (http_date_parse_month(parser) <= 0) + return -1; + + /* SP */ + if (http_date_parse_sp(parser) <= 0) + return -1; + + /* SP 1DIGIT / 2DIGIT */ + if ((ret=http_date_parse_sp(parser)) < 0) + return -1; + if (ret == 0) { + if (http_date_parse_number(parser, 2, &parser->tm.tm_mday) <= 0) + return -1; + } else { + if (http_date_parse_number(parser, 1, &parser->tm.tm_mday) <= 0) + return -1; + } + + /* SP time-of-day */ + if (http_date_parse_sp(parser) <= 0) + return -1; + if (http_date_parse_time_of_day(parser) <= 0) + return -1; + + /* SP year */ + if (http_date_parse_sp(parser) <= 0) + return -1; + + return http_date_parse_year(parser); +} + +static int +http_date_parse_format_any(struct http_date_parser *parser) +{ + string_t *dayname; + int i; + + /* + HTTP-date = IMF-fixdate / obs-date + IMF-fixdate = day-name "," SP date1 SP time-of-day SP GMT + ; fixed length/zone/capitalization subset of the format + ; see Section 3.3 of [RFC5322] + obs-date = rfc850-date / asctime-date + rfc850-date = day-name-l "," SP date2 SP time-of-day SP GMT + asctime-date = day-name SP date3 SP time-of-day SP year + */ + + if (http_date_parse_word(parser, 9, &dayname) <= 0) + return -1; + + if (str_len(dayname) > 3) { + /* rfc850-date */ + for (i = 0; i < 7; i++) { + if (strcmp(weekday_names_long[i], str_c(dayname)) == 0) { + break; + } + } + if (i >= 7) + return -1; + return http_date_parse_format_rfc850(parser); + } + + /* IMF-fixdate / asctime-date */ + for (i = 0; i < 7; i++) { + if (strcmp(weekday_names[i], str_c(dayname)) == 0) { + break; + } + } + + if (i >= 7 || parser->cur >= parser->end) + return -1; + + if (parser->cur[0] == ' ') { + /* asctime-date */ + parser->cur++; + return http_date_parse_format_asctime(parser); + } + + if (parser->cur[0] != ',') + return -1; + + /* IMF-fixdate */ + parser->cur++; + return http_date_parse_format_imf_fixdate(parser); +} + +bool http_date_parse(const unsigned char *data, size_t size, + time_t *timestamp_r) +{ + struct http_date_parser parser; + time_t timestamp; + + i_zero(&parser); + parser.cur = data; + parser.end = data + size; + + if (http_date_parse_format_any(&parser) <= 0) + return FALSE; + + if (parser.cur != parser.end) + return FALSE; + + parser.tm.tm_isdst = -1; + timestamp = utc_mktime(&parser.tm); + if (timestamp == (time_t)-1) + return FALSE; + + *timestamp_r = timestamp; + return TRUE; +} + +bool http_date_parse_tm(const unsigned char *data, size_t size, + struct tm *tm_r) +{ + time_t timestamp; + struct tm *tm; + + if (!http_date_parse(data, size, ×tamp)) + return FALSE; + + tm = gmtime(×tamp); + *tm_r = *tm; + return TRUE; +} + +const char *http_date_create_tm(struct tm *tm) +{ + return t_strdup_printf("%s, %02d %s %04d %02d:%02d:%02d GMT", + weekday_names[tm->tm_wday], + tm->tm_mday, + month_names[tm->tm_mon], + tm->tm_year+1900, + tm->tm_hour, tm->tm_min, tm->tm_sec); +} + +const char *http_date_create(time_t timestamp) +{ + struct tm *tm; + tm = gmtime(×tamp); + + return http_date_create_tm(tm); +} + diff --git a/src/lib-http/http-date.h b/src/lib-http/http-date.h new file mode 100644 index 0000000..e46299d --- /dev/null +++ b/src/lib-http/http-date.h @@ -0,0 +1,17 @@ +#ifndef HTTP_DATE_H +#define HTTP_DATE_H + +/* Parses HTTP-date string into time_t timestamp. */ +bool http_date_parse(const unsigned char *data, size_t size, + time_t *timestamp_r); +/* Equal to http_date_parse, but writes uncompensated timestamp to tm_r. */ +bool http_date_parse_tm(const unsigned char *data, size_t size, + struct tm *tm_r); + +/* Create HTTP-date string from given time struct. */ +const char *http_date_create_tm(struct tm *tm); + +/* Create HTTP-date string from given time. */ +const char *http_date_create(time_t timestamp); + +#endif diff --git a/src/lib-http/http-header-parser.c b/src/lib-http/http-header-parser.c new file mode 100644 index 0000000..29de688 --- /dev/null +++ b/src/lib-http/http-header-parser.c @@ -0,0 +1,367 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "istream.h" +#include "str.h" +#include "str-sanitize.h" +#include "http-parser.h" +#include "http-header.h" + +#include "http-header-parser.h" + +enum http_header_parse_state { + HTTP_HEADER_PARSE_STATE_INIT = 0, + HTTP_HEADER_PARSE_STATE_NAME, + HTTP_HEADER_PARSE_STATE_COLON, + HTTP_HEADER_PARSE_STATE_OWS, + HTTP_HEADER_PARSE_STATE_CONTENT, + HTTP_HEADER_PARSE_STATE_CR, + HTTP_HEADER_PARSE_STATE_LF, + HTTP_HEADER_PARSE_STATE_NEW_LINE, + HTTP_HEADER_PARSE_STATE_EOH +}; + +struct http_header_parser { + struct istream *input; + + struct http_header_limits limits; + enum http_header_parse_flags flags; + + uoff_t size, field_size; + unsigned int field_count; + + const unsigned char *begin, *cur, *end; + const char *error; + + string_t *name; + buffer_t *value_buf; + + enum http_header_parse_state state; +}; + +struct http_header_parser * +http_header_parser_init(struct istream *input, + const struct http_header_limits *limits, + enum http_header_parse_flags flags) +{ + struct http_header_parser *parser; + + parser = i_new(struct http_header_parser, 1); + parser->input = input; + + if (limits != NULL) + parser->limits = *limits; + + if (parser->limits.max_size == 0) + parser->limits.max_size = UOFF_T_MAX; + if (parser->limits.max_field_size == 0) + parser->limits.max_field_size = UOFF_T_MAX; + if (parser->limits.max_fields == 0) + parser->limits.max_fields = (unsigned int)-1; + + parser->flags = flags; + + parser->name = str_new(default_pool, 128); + parser->value_buf = buffer_create_dynamic(default_pool, 4096); + + return parser; +} + +void http_header_parser_deinit(struct http_header_parser **_parser) +{ + struct http_header_parser *parser = *_parser; + + *_parser = NULL; + + //i_stream_skip(ctx->input, ctx->skip); + buffer_free(&parser->value_buf); + str_free(&parser->name); + i_free(parser); +} + +void http_header_parser_reset(struct http_header_parser *parser) +{ + parser->state = HTTP_HEADER_PARSE_STATE_INIT; + parser->size = 0; + parser->field_size = 0; + parser->field_count = 0; +} + +static int http_header_parse_name(struct http_header_parser *parser) +{ + const unsigned char *first = parser->cur; + + /* field-name = token + token = 1*tchar + */ + while (parser->cur < parser->end && http_char_is_token(*parser->cur)) + parser->cur++; + + str_append_data(parser->name, first, parser->cur-first); + + if (parser->cur == parser->end) + return 0; + if (str_len(parser->name) == 0) { + parser->error = "Empty header field name"; + return -1; + } + return 1; +} + +static int http_header_parse_ows(struct http_header_parser *parser) +{ + /* OWS = *( SP / HTAB ) + ; "optional" whitespace + */ + while (parser->cur < parser->end && + (*parser->cur == ' ' || *parser->cur == '\t')) + parser->cur++; + return (parser->cur == parser->end ? 0 : 1); +} + +static int http_header_parse_content(struct http_header_parser *parser) +{ + const unsigned char *first; + + /* field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] + field-vchar = VCHAR / obs-text + */ + do { + first = parser->cur; + while (parser->cur < parser->end && http_char_is_text(*parser->cur)) { + parser->cur++; + } + buffer_append(parser->value_buf, first, parser->cur-first); + + if ((parser->flags & HTTP_HEADER_PARSE_FLAG_STRICT) != 0) + break; + + /* We'll be lenient here to accommodate for some bad servers. We just + drop offending characters */ + while (parser->cur < parser->end && !http_char_is_text(*parser->cur) && + (*parser->cur != '\r' && *parser->cur != '\n')) + parser->cur++; + } while (parser->cur < parser->end && + (*parser->cur != '\r' && *parser->cur != '\n')); + + if (parser->cur == parser->end) + return 0; + return 1; +} + +static inline const char *_chr_sanitize(unsigned char c) +{ + if (c >= 0x20 && c < 0x7F) + return t_strdup_printf("'%c'", c); + return t_strdup_printf("0x%02x", c); +} + +static int http_header_parse(struct http_header_parser *parser) +{ + int ret; + + /* RFC 7230, Section 3.2: Header Fields + + 'header' = *( header-field CRLF ) CRLF + ; Actually part of HTTP-message syntax + + header-field = field-name ":" OWS field-value OWS + field-name = token + field-value = *( field-content / obs-fold ) + field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] + field-vchar = VCHAR / obs-text + obs-fold = CRLF 1*( SP / HTAB ) + ; obsolete line folding + ; see Section 3.2.4 + */ + + for (;;) { + switch (parser->state) { + case HTTP_HEADER_PARSE_STATE_INIT: + buffer_set_used_size(parser->value_buf, 0); + str_truncate(parser->name, 0); + if (*parser->cur == '\r') { + /* last CRLF */ + parser->cur++; + parser->state = HTTP_HEADER_PARSE_STATE_EOH; + if (parser->cur == parser->end) + return 0; + break; + } else if (*parser->cur == '\n') { + /* last LF */ + parser->state = HTTP_HEADER_PARSE_STATE_EOH; + break; + } + /* next line */ + parser->state = HTTP_HEADER_PARSE_STATE_NAME; + /* fall through */ + case HTTP_HEADER_PARSE_STATE_NAME: + if ((ret=http_header_parse_name(parser)) <= 0) + return ret; + parser->state = HTTP_HEADER_PARSE_STATE_COLON; + /* fall through */ + case HTTP_HEADER_PARSE_STATE_COLON: + if (*parser->cur != ':') { + parser->error = t_strdup_printf + ("Expected ':' after header field name '%s', but found %s", + str_sanitize(str_c(parser->name),64), + _chr_sanitize(*parser->cur)); + return -1; + } + parser->cur++; + if (str_len(parser->name) == 0) { + parser->error = "Empty header field name"; + return -1; + } + if (++parser->field_count > parser->limits.max_fields) { + parser->error = "Excessive number of header fields"; + return -1; + } + parser->state = HTTP_HEADER_PARSE_STATE_OWS; + /* fall through */ + case HTTP_HEADER_PARSE_STATE_OWS: + if ((ret=http_header_parse_ows(parser)) <= 0) + return ret; + parser->state = HTTP_HEADER_PARSE_STATE_CONTENT; + /* fall through */ + case HTTP_HEADER_PARSE_STATE_CONTENT: + if ((ret=http_header_parse_content(parser)) <= 0) + return ret; + parser->state = HTTP_HEADER_PARSE_STATE_CR; + /* fall through */ + case HTTP_HEADER_PARSE_STATE_CR: + if (*parser->cur == '\r') { + parser->cur++; + } else if (*parser->cur != '\n') { + parser->error = t_strdup_printf + ("Invalid character %s in content of header field '%s'", + _chr_sanitize(*parser->cur), + str_sanitize(str_c(parser->name),64)); + return -1; + } + parser->state = HTTP_HEADER_PARSE_STATE_LF; + if (parser->cur == parser->end) + return 0; + /* fall through */ + case HTTP_HEADER_PARSE_STATE_LF: + if (*parser->cur != '\n') { + parser->error = t_strdup_printf + ("Expected LF after CR at end of header field '%s', but found %s", + str_sanitize(str_c(parser->name),64), + _chr_sanitize(*parser->cur)); + return -1; + } + parser->cur++; + parser->state = HTTP_HEADER_PARSE_STATE_NEW_LINE; + if (parser->cur == parser->end) + return 0; + /* fall through */ + case HTTP_HEADER_PARSE_STATE_NEW_LINE: + if (*parser->cur == ' ' || *parser->cur == '\t') { + /* obs-fold */ + buffer_append_c(parser->value_buf, ' '); + parser->state = HTTP_HEADER_PARSE_STATE_OWS; + break; + } + /* next header line */ + parser->state = HTTP_HEADER_PARSE_STATE_INIT; + return 1; + case HTTP_HEADER_PARSE_STATE_EOH: + if (*parser->cur != '\n') { + parser->error = t_strdup_printf + ("Encountered stray CR at beginning of header line, followed by %s", + _chr_sanitize(*parser->cur)); + return -1; + } + /* header fully parsed */ + parser->cur++; + return 1; + + default: + i_unreached(); + } + } + + i_unreached(); + return -1; +} + +int http_header_parse_next_field(struct http_header_parser *parser, + const char **name_r, const unsigned char **data_r, size_t *size_r, + const char **error_r) +{ + const uoff_t max_size = parser->limits.max_size; + const uoff_t max_field_size = parser->limits.max_field_size; + const unsigned char *data; + size_t size; + int ret; + + *error_r = NULL; + + while ((ret=i_stream_read_more(parser->input, &parser->begin, &size)) > 0) { + + /* check header size limits */ + if (parser->size >= max_size) { + *error_r = "Excessive header size"; + return -1; + } + if (parser->field_size > max_field_size) { + *error_r = "Excessive header field size"; + return -1; + } + + /* don't parse beyond header size limits */ + if (size > (max_size - parser->size)) + size = max_size - parser->size; + if (size > (max_field_size - parser->field_size)) { + size = max_field_size - parser->field_size; + size = (size == 0 ? 1 : size); /* need to parse one more byte */ + } + + parser->cur = parser->begin; + parser->end = parser->cur + size; + + if ((ret=http_header_parse(parser)) < 0) { + *error_r = parser->error; + return -1; + } + + i_stream_skip(parser->input, parser->cur - parser->begin); + parser->size += parser->cur - parser->begin; + parser->field_size += parser->cur - parser->begin; + + if (ret == 1) { + parser->field_size = 0; + + if (parser->state != HTTP_HEADER_PARSE_STATE_EOH) { + data = buffer_get_data(parser->value_buf, &size); + + /* trim trailing OWS */ + while (size > 0 && + (data[size-1] == ' ' || data[size-1] == '\t')) + size--; + + *name_r = str_c(parser->name); + *data_r = data; + *size_r = size; + parser->state = HTTP_HEADER_PARSE_STATE_INIT; + } else { + *name_r = NULL; + *data_r = NULL; + } + return 1; + } + } + + i_assert(ret != -2); + if (ret < 0) { + i_assert(parser->input->eof); + if (parser->input->stream_errno == 0) + *error_r = "Premature end of input"; + else + *error_r = t_strdup_printf("Stream error: %s", + i_stream_get_error(parser->input)); + } + return ret; +} diff --git a/src/lib-http/http-header-parser.h b/src/lib-http/http-header-parser.h new file mode 100644 index 0000000..a746520 --- /dev/null +++ b/src/lib-http/http-header-parser.h @@ -0,0 +1,24 @@ +#ifndef HTTP_HEADER_PARSER_H +#define HTTP_HEADER_PARSER_H + +struct http_header_limits; +struct http_header_parser; + +enum http_header_parse_flags { + /* Strictly adhere to the HTTP protocol specification */ + HTTP_HEADER_PARSE_FLAG_STRICT = BIT(0) +}; + +struct http_header_parser * +http_header_parser_init(struct istream *input, + const struct http_header_limits *limits, + enum http_header_parse_flags flags); +void http_header_parser_deinit(struct http_header_parser **_parser); + +void http_header_parser_reset(struct http_header_parser *parser); + +int http_header_parse_next_field(struct http_header_parser *parser, + const char **name_r, const unsigned char **data_r, size_t *size_r, + const char **error_r); + +#endif diff --git a/src/lib-http/http-header.c b/src/lib-http/http-header.c new file mode 100644 index 0000000..e8ef21a --- /dev/null +++ b/src/lib-http/http-header.c @@ -0,0 +1,98 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" + +#include "http-header.h" + +struct http_header { + ARRAY_TYPE(http_header_field) fields; + /* FIXME: ARRAY(struct http_header_field *) *btree; */ +}; + +struct http_header * +http_header_create(pool_t pool, unsigned int init_count) +{ + struct http_header *header; + + header = p_new(pool, struct http_header, 1); + p_array_init(&header->fields, pool, init_count); + + return header; +} + +const struct http_header_field * +http_header_field_add(struct http_header *header, + const char *name, const unsigned char *data, size_t size) +{ + struct http_header_field *hfield; + pool_t pool = array_get_pool(&header->fields); + void *value; + + hfield = array_append_space(&header->fields); + hfield->name = p_strdup(pool, name); + hfield->size = size; + + value = p_malloc(pool, size+1); + memcpy(value, data, size); + hfield->value = (const char *)value; + + return hfield; +} + +void http_header_field_delete(struct http_header *header, const char *name) +{ + ARRAY_TYPE(http_header_field) *hfields = &header->fields; + const struct http_header_field *hfield; + + array_foreach(hfields, hfield) { + if (http_header_field_is(hfield, name)) { + array_delete(hfields, array_foreach_idx(hfields, hfield), 1); + } + } +} + +const ARRAY_TYPE(http_header_field) * +http_header_get_fields(const struct http_header *header) +{ + return &header->fields; +} + +const struct http_header_field * +http_header_field_find(const struct http_header *header, const char *name) +{ + const struct http_header_field *hfield; + + array_foreach(&header->fields, hfield) { + if (http_header_field_is(hfield, name)) + return hfield; + } + + return NULL; +} + +const char * +http_header_field_get(const struct http_header *header, const char *name) +{ + const struct http_header_field *hfield = + http_header_field_find(header, name); + return (hfield == NULL ? NULL : hfield->value); +} + +int http_header_field_find_unique(const struct http_header *header, + const char *name, const struct http_header_field **hfield_r) +{ + const struct http_header_field *hfield, *hfield_found = NULL; + + array_foreach(&header->fields, hfield) { + if (http_header_field_is(hfield, name)) { + if (hfield_found != NULL) + return -1; + hfield_found = hfield; + } + } + + *hfield_r = hfield_found; + return (hfield_found == NULL ? 0 : 1); +} + diff --git a/src/lib-http/http-header.h b/src/lib-http/http-header.h new file mode 100644 index 0000000..59941d9 --- /dev/null +++ b/src/lib-http/http-header.h @@ -0,0 +1,45 @@ +#ifndef HTTP_HEADER_H +#define HTTP_HEADER_H + +struct http_header; + +struct http_header_limits { + uoff_t max_size; + uoff_t max_field_size; + unsigned int max_fields; +}; + +struct http_header_field { + const char *name; + const char *value; + size_t size; +}; +ARRAY_DEFINE_TYPE(http_header_field, struct http_header_field); + +static inline bool http_header_field_is(const struct http_header_field *hfield, + const char *name) +{ + return (strcasecmp(hfield->name, name) == 0); +} + +struct http_header * +http_header_create(pool_t pool, unsigned int init_count); + +const struct http_header_field * +http_header_field_add(struct http_header *header, + const char *name, const unsigned char *data, size_t size); +void http_header_field_delete(struct http_header *header, const char *name); + +const ARRAY_TYPE(http_header_field) * +http_header_get_fields(const struct http_header *header) ATTR_PURE; + +const struct http_header_field * +http_header_field_find(const struct http_header *header, const char *name) + ATTR_PURE; +const char * +http_header_field_get(const struct http_header *header, const char *name) + ATTR_PURE; +int http_header_field_find_unique(const struct http_header *header, + const char *name, const struct http_header_field **hfield_r); + +#endif diff --git a/src/lib-http/http-message-parser.c b/src/lib-http/http-message-parser.c new file mode 100644 index 0000000..a42bddd --- /dev/null +++ b/src/lib-http/http-message-parser.c @@ -0,0 +1,658 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "iostream.h" +#include "istream-sized.h" +#include "http-parser.h" +#include "http-header.h" +#include "http-header-parser.h" +#include "http-date.h" +#include "http-transfer.h" +#include "http-message-parser.h" + +#include <ctype.h> + +void http_message_parser_init(struct http_message_parser *parser, + struct istream *input, + const struct http_header_limits *hdr_limits, + uoff_t max_payload_size, + enum http_message_parse_flags flags) +{ + i_zero(parser); + parser->input = input; + i_stream_ref(parser->input); + if (hdr_limits != NULL) + parser->header_limits = *hdr_limits; + parser->max_payload_size = max_payload_size; + parser->flags = flags; +} + +void http_message_parser_deinit(struct http_message_parser *parser) +{ + if (parser->header_parser != NULL) + http_header_parser_deinit(&parser->header_parser); + pool_unref(&parser->msg.pool); + i_stream_unref(&parser->payload); + i_stream_unref(&parser->input); +} + +void http_message_parser_restart(struct http_message_parser *parser, + pool_t pool) +{ + i_assert(parser->payload == NULL); + + if (parser->header_parser == NULL) { + enum http_header_parse_flags hdr_flags = 0; + + if ((parser->flags & HTTP_MESSAGE_PARSE_FLAG_STRICT) != 0) + hdr_flags |= HTTP_HEADER_PARSE_FLAG_STRICT; + parser->header_parser = http_header_parser_init( + parser->input, &parser->header_limits, hdr_flags); + } else { + http_header_parser_reset(parser->header_parser); + } + + pool_unref(&parser->msg.pool); + i_zero(&parser->msg); + if (pool != NULL) { + parser->msg.pool = pool; + pool_ref(pool); + } + parser->msg.date = (time_t)-1; +} + +pool_t http_message_parser_get_pool(struct http_message_parser *parser) +{ + if (parser->msg.pool == NULL) + parser->msg.pool = pool_alloconly_create("http_message", 4096); + return parser->msg.pool; +} + +int http_message_parse_version(struct http_message_parser *parser) +{ + const unsigned char *p = parser->cur; + const size_t size = parser->end - parser->cur; + + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_NONE; + parser->error = NULL; + + /* RFC 7230, Section 2.6: Protocol Versioning + + HTTP-version = HTTP-name "/" DIGIT "." DIGIT + HTTP-name = %x48.54.54.50 ; "HTTP", case-sensitive + */ + if (size < 8) + return 0; + if (memcmp(p, "HTTP/", 5) != 0 || + !i_isdigit(p[5]) || p[6] != '.' || !i_isdigit(p[7])) { + parser->error = "Bad HTTP version"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + return -1; + } + parser->msg.version_major = p[5] - '0'; + parser->msg.version_minor = p[7] - '0'; + parser->cur += 8; + return 1; +} + +static void +http_message_parse_finish_payload_error(struct http_message_parser *parser) +{ + if (parser->payload->stream_errno == EMSGSIZE) { + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_PAYLOAD_TOO_LARGE; + parser->error = "Payload is too large"; + } else if (parser->payload->stream_errno == EIO) { + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + parser->error = t_strdup_printf( + "Invalid payload: %s", + i_stream_get_error(parser->payload)); + } else { + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_STREAM; + parser->error = t_strdup_printf( + "Stream error while skipping payload: %s", + i_stream_get_error(parser->payload)); + } +} + +int http_message_parse_finish_payload(struct http_message_parser *parser) +{ + const unsigned char *data; + size_t size; + int ret; + + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_NONE; + parser->error = NULL; + + if (parser->payload == NULL) + return 1; + + while ((ret = i_stream_read_more(parser->payload, &data, &size)) > 0) + i_stream_skip(parser->payload, size); + if (ret == 0 || parser->payload->stream_errno != 0) { + if (ret < 0) + http_message_parse_finish_payload_error(parser); + return ret; + } + + i_stream_destroy(&parser->payload); + return 1; +} + +static int +http_message_parse_hdr_connection(struct http_message_parser *parser, + const unsigned char *data, size_t size) +{ + pool_t pool = http_message_parser_get_pool(parser); + struct http_parser hparser; + const char **opt_idx; + const char *option; + unsigned int num_tokens = 0; + + /* RFC 7230, Section 6.1: Connection + + Connection = 1#connection-option + connection-option = token + */ + + /* Multiple Connection headers are allowed and combined + into one */ + http_parser_init(&hparser, data, size); + for (;;) { + if (http_parse_token_list_next(&hparser, &option) <= 0) + break; + num_tokens++; + if (strcasecmp(option, "close") == 0) + parser->msg.connection_close = TRUE; + if (!array_is_created(&parser->msg.connection_options)) + p_array_init(&parser->msg.connection_options, pool, 4); + opt_idx = array_append_space(&parser->msg.connection_options); + *opt_idx = p_strdup(pool, option); + } + + if (hparser.cur < hparser.end || num_tokens == 0) { + parser->error = "Invalid Connection header"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + return -1; + } + + return 0; +} + +static int +http_message_parse_hdr_content_length(struct http_message_parser *parser, + const struct http_header_field *hdr) +{ + if (parser->msg.have_content_length) { + /* There is no acceptable way to allow duplicates for this + header. */ + parser->error = "Duplicate Content-Length header"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + return -1; + } + + /* RFC 7230, Section 3.3.2: Content-Length + + Content-Length = 1*DIGIT + */ + if (str_to_uoff(hdr->value, &parser->msg.content_length) < 0) { + parser->error = "Invalid Content-Length header"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + return -1; + } + parser->msg.have_content_length = TRUE; + return 0; +} + +static int +http_message_parse_hdr_date(struct http_message_parser *parser, + const unsigned char *data, size_t size) +{ + if (parser->msg.date != (time_t)-1) { + if ((parser->flags & HTTP_MESSAGE_PARSE_FLAG_STRICT) != 0) { + parser->error = "Duplicate Date header"; + parser->error_code = + HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + return -1; + } + /* Allow the duplicate; last instance is used */ + } + + /* RFC 7231, Section 7.1.1.2: Date + + Date = HTTP-date + */ + if (!http_date_parse(data, size, &parser->msg.date) && + (parser->flags & HTTP_MESSAGE_PARSE_FLAG_STRICT) != 0) { + parser->error = "Invalid Date header"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + return -1; + } + return 0; +} + +static int +http_message_parse_hdr_location(struct http_message_parser *parser, + const struct http_header_field *hdr) +{ + /* RFC 7231, Section 7.1.2: Location + + Location = URI-reference + + -> not parsed here + */ + /* FIXME: move this to response parser */ + parser->msg.location = hdr->value; + return 0; + +} + +static int +http_message_parse_hdr_transfer_encoding(struct http_message_parser *parser, + const unsigned char *data, size_t size) +{ + pool_t pool = http_message_parser_get_pool(parser); + struct http_parser hparser; + const char *trenc = NULL; + + /* Multiple Transfer-Encoding headers are allowed and combined into one + */ + if (!array_is_created(&parser->msg.transfer_encoding)) + p_array_init(&parser->msg.transfer_encoding, pool, 4); + + /* RFC 7230, Section 3.3.1: Transfer-Encoding + + Transfer-Encoding = 1#transfer-coding + + RFC 7230, Section 4: Transfer Codings + + transfer-coding = "chunked" ; RFC 7230, Section 4.1 + / "compress" ; RFC 7230, Section 4.2.1 + / "deflate" ; RFC 7230, Section 4.2.2 + / "gzip" ; RFC 7230, Section 4.2.3 + / transfer-extension + transfer-extension = token *( OWS ";" OWS transfer-parameter ) + transfer-parameter = token BWS "=" BWS ( token / quoted-string ) + */ + http_parser_init(&hparser, data, size); + for (;;) { + /* transfer-coding */ + if (http_parse_token(&hparser, &trenc) > 0) { + struct http_transfer_coding *coding; + bool parse_error; + + coding = array_append_space( + &parser->msg.transfer_encoding); + coding->name = p_strdup(pool, trenc); + + /* *( OWS ";" OWS transfer-parameter ) */ + parse_error = FALSE; + for (;;) { + struct http_transfer_param *param; + const char *attribute, *value; + + /* OWS ";" OWS */ + http_parse_ows(&hparser); + if (hparser.cur >= hparser.end || + *hparser.cur != ';') + break; + hparser.cur++; + http_parse_ows(&hparser); + + /* attribute */ + if (http_parse_token(&hparser, + &attribute) <= 0) { + parse_error = TRUE; + break; + } + + /* BWS "=" BWS */ + http_parse_ows(&hparser); + if (hparser.cur >= hparser.end || + *hparser.cur != '=') { + parse_error = TRUE; + break; + } + hparser.cur++; + http_parse_ows(&hparser); + + /* token / quoted-string */ + if (http_parse_token_or_qstring(&hparser, + &value) <= 0) { + parse_error = TRUE; + break; + } + + if (!array_is_created(&coding->parameters)) { + p_array_init(&coding->parameters, + pool, 2); + } + param = array_append_space(&coding->parameters); + param->attribute = p_strdup(pool, attribute); + param->value = p_strdup(pool, value); + } + if (parse_error) + break; + + } else { + /* RFC 7230, Section 7: ABNF List Extension: #rule + + For compatibility with legacy list rules, a recipient + MUST parse and ignore a reasonable number of empty + list elements: enough to handle common mistakes by + senders that merge values, but not so much that they + could be used as a denial-of-service mechanism. + */ + // FIXME: limit allowed number of empty list elements + // FIXME: handle invalid transfer encoding + } + http_parse_ows(&hparser); + if (hparser.cur >= hparser.end || *hparser.cur != ',') + break; + hparser.cur++; + http_parse_ows(&hparser); + } + + if (hparser.cur < hparser.end || + array_count(&parser->msg.transfer_encoding) == 0) { + parser->error = "Invalid Transfer-Encoding header"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + return -1; + } + return 0; +} + +static int +http_message_parse_header(struct http_message_parser *parser, + const char *name, const unsigned char *data, + size_t size) +{ + const struct http_header_field *hdr; + pool_t pool; + + pool = http_message_parser_get_pool(parser); + if (parser->msg.header == NULL) + parser->msg.header = http_header_create(pool, 32); + hdr = http_header_field_add(parser->msg.header, name, data, size); + + /* RFC 7230, Section 3.2.2: Field Order + + A sender MUST NOT generate multiple header fields with the same field + name in a message unless either the entire field value for that + header field is defined as a comma-separated list [i.e., #(values)] + or the header field is a well-known exception. + */ + + switch (name[0]) { + case 'C': case 'c': + /* Connection: */ + if (strcasecmp(name, "Connection") == 0) { + return http_message_parse_hdr_connection( + parser, data, size); + } + /* Content-Length: */ + if (strcasecmp(name, "Content-Length") == 0) + return http_message_parse_hdr_content_length( + parser, hdr); + break; + case 'D': case 'd': + /* Date: */ + if (strcasecmp(name, "Date") == 0) + return http_message_parse_hdr_date(parser, data, size); + break; + case 'L': case 'l': + /* Location: */ + if (strcasecmp(name, "Location") == 0) + return http_message_parse_hdr_location(parser, hdr); + break; + case 'T': case 't': + /* Transfer-Encoding: */ + if (strcasecmp(name, "Transfer-Encoding") == 0) { + return http_message_parse_hdr_transfer_encoding( + parser, data, size); + } + break; + default: + break; + } + return 0; +} + +static int http_message_parse_eoh(struct http_message_parser *parser) +{ + struct http_message *msg = &parser->msg; + pool_t pool; + + /* EOH */ + + /* Create empty header if there is none */ + pool = http_message_parser_get_pool(parser); + if (msg->header == NULL) + msg->header = http_header_create(pool, 1); + + /* handle HTTP/1.0 persistence */ + if (msg->version_major == 1 && msg->version_minor == 0 && + !msg->connection_close) { + const char *option; + + msg->connection_close = TRUE; + if (array_is_created(&msg->connection_options)) { + array_foreach_elem(&msg->connection_options, option) { + if (strcasecmp(option, "Keep-Alive") == 0) { + msg->connection_close = FALSE; + break; + } + } + } + } + return 1; +} + +int http_message_parse_headers(struct http_message_parser *parser) +{ + const unsigned char *field_data; + const char *field_name, *error; + size_t field_size; + int ret; + + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_NONE; + parser->error = NULL; + + /* *( header-field CRLF ) CRLF */ + while ((ret = http_header_parse_next_field( + parser->header_parser, &field_name, &field_data, &field_size, + &error)) > 0) { + if (field_name == NULL) + return http_message_parse_eoh(parser); + + if (http_message_parse_header(parser, + field_name, field_data, field_size) < 0) + return -1; + } + + if (ret < 0) { + if (parser->input->eof || parser->input->stream_errno != 0) { + parser->error_code = + HTTP_MESSAGE_PARSE_ERROR_BROKEN_STREAM; + parser->error = "Broken stream"; + } else { + parser->error_code = + HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + parser->error = t_strdup_printf( + "Failed to parse header: %s", error); + } + + } + return ret; +} + +static const char * +http_istream_error_callback(const struct istream_sized_error_data *data, + struct istream *input) +{ + i_assert(data->eof); + i_assert(data->v_offset + data->new_bytes < data->wanted_size); + + return t_strdup_printf( + "Disconnected while reading message payload at offset %"PRIuUOFF_T + " (wanted %"PRIuUOFF_T"): %s", data->v_offset + data->new_bytes, + data->wanted_size, io_stream_get_disconnect_reason(input, NULL)); +} + +static int +http_message_parse_body_coding(struct http_message_parser *parser, + const struct http_transfer_coding *coding, + bool *seen_chunked) +{ + if (strcasecmp(coding->name, "chunked") == 0) { + *seen_chunked = TRUE; + + if ((parser->error_code == HTTP_MESSAGE_PARSE_ERROR_NONE) + && array_is_created(&coding->parameters) + && array_count(&coding->parameters) > 0) { + const struct http_transfer_param *param = + array_front(&coding->parameters); + + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BAD_MESSAGE; + parser->error = t_strdup_printf( + "Unexpected parameter `%s' specified" + "for the `%s' transfer coding", + param->attribute, coding->name); + /* recoverable */ + } + } else if (*seen_chunked) { + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + parser->error = "Chunked Transfer-Encoding must be last"; + return -1; + } else if (parser->error_code == HTTP_MESSAGE_PARSE_ERROR_NONE) { + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_NOT_IMPLEMENTED; + parser->error = t_strdup_printf( + "Unknown transfer coding `%s'", coding->name); + /* recoverable */ + } + return 0; +} + +static int +http_message_parse_body_encoded(struct http_message_parser *parser, + bool request) +{ + const struct http_transfer_coding *coding; + bool seen_chunked = FALSE; + + array_foreach(&parser->msg.transfer_encoding, coding) { + if (http_message_parse_body_coding(parser, coding, + &seen_chunked) < 0) + return -1; + } + + if (seen_chunked) { + parser->payload = http_transfer_chunked_istream_create( + parser->input, parser->max_payload_size); + } else if (!request) { + /* RFC 7230, Section 3.3.3: Message Body Length + + If a Transfer-Encoding header field is present in a response + and the chunked transfer coding is not the final encoding, + the message body length is determined by reading the + connection until it is closed by the server. + */ + /* FIXME: enforce max payload size (relevant to http-client + only) */ + parser->payload = + i_stream_create_limit(parser->input, SIZE_MAX); + } else { + /* RFC 7230, Section 3.3.3: Message Body Length + + If a Transfer-Encoding header field is present in a request + and the chunked transfer coding is not the final encoding, + the message body length cannot be determined reliably; the + server MUST respond with the 400 (Bad Request) status code + and then close the connection. + */ + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + parser->error = + "Final Transfer-Encoding in request is not chunked"; + return -1; + } + + /* RFC 7230, Section 3.3.3: Message Body Length + + If a message is received with both a Transfer-Encoding and a + Content-Length header field, the Transfer-Encoding overrides the + Content-Length. Such a message might indicate an attempt to perform + request smuggling (Section 9.5 of [RFC7230]) or response splitting + (Section 9.4 of [RFC7230]) and ought to be handled as an error. A + sender MUST remove the received Content-Length field prior to + forwarding such a message downstream. + */ + // FIXME: make this an error? + if (parser->msg.have_content_length) + http_header_field_delete(parser->msg.header, "Content-Length"); + + return 0; +} + +static int http_message_parse_body_sized(struct http_message_parser *parser) +{ + struct istream *input; + + if (parser->max_payload_size > 0 + && parser->msg.content_length > parser->max_payload_size) { + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_PAYLOAD_TOO_LARGE; + parser->error = "Payload is too large"; + return -1; + } + + /* Got explicit message size from Content-Length: header */ + input = i_stream_create_limit(parser->input, + parser->msg.content_length); + /* Make sure we return failure if HTTP connection closes before we've + finished reading the full input. */ + parser->payload = i_stream_create_sized_with_callback(input, + parser->msg.content_length, + http_istream_error_callback, input); + i_stream_unref(&input); + return 0; +} + +static int http_message_parse_body_closed(struct http_message_parser *parser) +{ + /* RFC 7230, Section 3.3.3: Message Body Length + + 6. If this is a request message and none of the above are true, then + the message body length is zero (no message body is present). + + 7. Otherwise, this is a response message without a declared message + body length, so the message body length is determined by the + number of octets received prior to the server closing the + connection + */ + // FIXME: enforce max payload size (relevant to http-client only) + // FIXME: handle request case correctly. + parser->payload = i_stream_create_limit(parser->input, SIZE_MAX); + return 0; +} + +int http_message_parse_body(struct http_message_parser *parser, bool request) +{ + i_assert(parser->payload == NULL); + + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_NONE; + parser->error = NULL; + + if (array_is_created(&parser->msg.transfer_encoding)) { + if (http_message_parse_body_encoded(parser, request) < 0) + return -1; + } else if (parser->msg.content_length > 0) { + if (http_message_parse_body_sized(parser) < 0) + return -1; + } else if (!parser->msg.have_content_length && !request) { + if (http_message_parse_body_closed(parser) < 0) + return -1; + } + if (parser->error_code != HTTP_MESSAGE_PARSE_ERROR_NONE) + return -1; + return 0; +} diff --git a/src/lib-http/http-message-parser.h b/src/lib-http/http-message-parser.h new file mode 100644 index 0000000..17c17cb --- /dev/null +++ b/src/lib-http/http-message-parser.h @@ -0,0 +1,77 @@ +#ifndef HTTP_MESSAGE_PARSER_H +#define HTTP_MESSAGE_PARSER_H + +#include "http-response.h" +#include "http-transfer.h" + +#include "http-header.h" + +enum http_message_parse_error { + HTTP_MESSAGE_PARSE_ERROR_NONE = 0, /* no error */ + HTTP_MESSAGE_PARSE_ERROR_BROKEN_STREAM, /* stream error */ + HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE, /* unrecoverable generic error */ + HTTP_MESSAGE_PARSE_ERROR_BAD_MESSAGE, /* recoverable generic error */ + HTTP_MESSAGE_PARSE_ERROR_NOT_IMPLEMENTED, /* used unimplemented feature + (recoverable) */ + HTTP_MESSAGE_PARSE_ERROR_PAYLOAD_TOO_LARGE /* message payload is too large + (fatal) */ +}; + +enum http_message_parse_flags { + /* Strictly adhere to the HTTP protocol specification */ + HTTP_MESSAGE_PARSE_FLAG_STRICT = BIT(0) +}; + +struct http_message { + pool_t pool; + + unsigned int version_major; + unsigned int version_minor; + + struct http_header *header; + + time_t date; + uoff_t content_length; + const char *location; + ARRAY_TYPE(http_transfer_coding) transfer_encoding; + ARRAY_TYPE(const_string) connection_options; + + bool connection_close:1; + bool have_content_length:1; +}; + +struct http_message_parser { + struct istream *input; + + struct http_header_limits header_limits; + uoff_t max_payload_size; + enum http_message_parse_flags flags; + + const unsigned char *begin, *cur, *end; + + const char *error; + enum http_message_parse_error error_code; + + struct http_header_parser *header_parser; + struct istream *payload; + + pool_t msg_pool; + struct http_message msg; +}; + +void http_message_parser_init(struct http_message_parser *parser, + struct istream *input, const struct http_header_limits *hdr_limits, + uoff_t max_payload_size, enum http_message_parse_flags flags) + ATTR_NULL(3); +void http_message_parser_deinit(struct http_message_parser *parser); +void http_message_parser_restart(struct http_message_parser *parser, + pool_t pool); + +pool_t http_message_parser_get_pool(struct http_message_parser *parser); + +int http_message_parse_finish_payload(struct http_message_parser *parser); +int http_message_parse_version(struct http_message_parser *parser); +int http_message_parse_headers(struct http_message_parser *parser); +int http_message_parse_body(struct http_message_parser *parser, bool request); + +#endif diff --git a/src/lib-http/http-parser.c b/src/lib-http/http-parser.c new file mode 100644 index 0000000..7cd6c2a --- /dev/null +++ b/src/lib-http/http-parser.c @@ -0,0 +1,208 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "net.h" +#include "str.h" +#include "strescape.h" +#include "http-url.h" + +#include "http-parser.h" + +/* + Character definitions: + + tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" + / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" + / DIGIT / ALPHA + ; any VCHAR, except special + special = "(" / ")" / "<" / ">" / "@" / "," + / ";" / ":" / "\" / DQUOTE / "/" / "[" + / "]" / "?" / "=" / "{" / "}" + qdtext = OWS / %x21 / %x23-5B / %x5D-7E / obs-text + qdtext-nf = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text + ctext = OWS / %x21-27 / %x2A-5B / %x5D-7E / obs-text + obs-text = %x80-FF + OWS = *( SP / HTAB ) + VCHAR = %x21-7E + 't68char' = ALPHA / DIGIT / "-" / "." / "_" / "~" / "+" / "/" + + 'text' = ( HTAB / SP / VCHAR / obs-text ) + + Character bit mappings: + + (1<<0) => ALPHA / DIGIT / "-" / "." / "_" / "~" / "+" + (1<<1) => "!" / "#" / "$" / "%" / "&" / "'" / "*" / "^" / "`" / "|" + (1<<2) => special + (1<<3) => %x21 / %x2A-5B / %x5D-7E + (1<<4) => %x23-29 + (1<<5) => %x22-27 + (1<<6) => HTAB / SP / obs-text + (1<<7) => "/" + */ + +const unsigned char _http_token_char_mask = (1<<0)|(1<<1); +const unsigned char _http_value_char_mask = (1<<0)|(1<<1)|(1<<2); +const unsigned char _http_text_char_mask = (1<<0)|(1<<1)|(1<<2)|(1<<6); +const unsigned char _http_qdtext_char_mask = (1<<3)|(1<<4)|(1<<6); +const unsigned char _http_ctext_char_mask = (1<<3)|(1<<5)|(1<<6); +const unsigned char _http_token68_char_mask = (1<<0)|(1<<7); + +const unsigned char _http_char_lookup[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, // 00 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 10 + 64, 10, 36, 50, 50, 50, 50, 50, 20, 20, 10, 9, 12, 9, 9, 140, // 20 + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 12, 12, 12, 12, 12, 12, // 30 + 12, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 40 + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 12, 4, 12, 10, 9, // 50 + 10, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 60 + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 12, 10, 12, 9, 0, // 70 + + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, // 80 + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, // 90 + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, // A0 + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, // B0 + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, // C0 + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, // D0 + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, // E0 + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, // F0 +}; + +/* + * HTTP value parsing + */ + +void http_parser_init(struct http_parser *parser, + const unsigned char *data, size_t size) +{ + i_zero(parser); + parser->begin = data; + parser->cur = data; + parser->end = data + size; +} + +void http_parse_ows(struct http_parser *parser) +{ + /* OWS = *( SP / HTAB ) */ + if (parser->cur >= parser->end) + return; + while (parser->cur < parser->end && + (parser->cur[0] == ' ' || parser->cur[0] == '\t')) { + parser->cur++; + } +} + +int http_parser_skip_token(struct http_parser *parser) +{ + /* token = 1*tchar */ + + if (parser->cur >= parser->end || !http_char_is_token(*parser->cur)) + return 0; + parser->cur++; + + while (parser->cur < parser->end && http_char_is_token(*parser->cur)) + parser->cur++; + return 1; +} + +int http_parse_token(struct http_parser *parser, const char **token_r) +{ + const unsigned char *first = parser->cur; + int ret; + + if ((ret=http_parser_skip_token(parser)) <= 0) + return ret; + *token_r = t_strndup(first, parser->cur - first); + return 1; +} + +int http_parse_token_list_next(struct http_parser *parser, + const char **token_r) +{ + /* http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21; + Appendix B: + + For compatibility with legacy list rules, recipients SHOULD accept + empty list elements. In other words, consumers would follow the list + productions: + + #element => [ ( "," / element ) *( OWS "," [ OWS element ] ) ] + 1#element => *( "," OWS ) element *( OWS "," [ OWS element ] ) + */ + + for (;;) { + if (http_parse_token(parser, token_r) > 0) + break; + http_parse_ows(parser); + if (parser->cur >= parser->end || parser->cur[0] != ',') + return 0; + parser->cur++; + http_parse_ows(parser); + } + + return 1; +} + +int http_parse_quoted_string(struct http_parser *parser, const char **str_r) +{ + string_t *str; + + /* quoted-string = DQUOTE *( qdtext / quoted-pair ) DQUOTE + qdtext = HTAB / SP / "!" / %x23-5B ; '#'-'[' + / %x5D-7E ; ']'-'~' + / obs-text + quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text ) + obs-text = %x80-FF + */ + + /* DQUOTE */ + if (parser->cur >= parser->end || parser->cur[0] != '"') + return 0; + parser->cur++; + + /* *( qdtext / quoted-pair ) */ + str = t_str_new(256); + for (;;) { + const unsigned char *first; + + /* *qdtext */ + first = parser->cur; + while (parser->cur < parser->end && http_char_is_qdtext(*parser->cur)) + parser->cur++; + + if (parser->cur >= parser->end) + return -1; + + str_append_data(str, first, parser->cur - first); + + /* DQUOTE */ + if (*parser->cur == '"') { + parser->cur++; + break; + + /* "\" */ + } else if (*parser->cur == '\\') { + parser->cur++; + + if (parser->cur >= parser->end || !http_char_is_text(*parser->cur)) + return -1; + str_append_c(str, *parser->cur); + parser->cur++; + + /* ERROR */ + } else { + return -1; + } + } + *str_r = str_c(str); + return 1; +} + +int http_parse_token_or_qstring(struct http_parser *parser, + const char **word_r) +{ + if (parser->cur >= parser->end) + return 0; + if (parser->cur[0] == '"') + return http_parse_quoted_string(parser, word_r); + return http_parse_token(parser, word_r); +} diff --git a/src/lib-http/http-parser.h b/src/lib-http/http-parser.h new file mode 100644 index 0000000..915e484 --- /dev/null +++ b/src/lib-http/http-parser.h @@ -0,0 +1,63 @@ +#ifndef HTTP_PARSER_H +#define HTTP_PARSER_H + +/* + * Character definitions + */ + +extern const unsigned char _http_token_char_mask; +extern const unsigned char _http_value_char_mask; +extern const unsigned char _http_text_char_mask; +extern const unsigned char _http_qdtext_char_mask; +extern const unsigned char _http_ctext_char_mask; +extern const unsigned char _http_token68_char_mask; + +extern const unsigned char _http_char_lookup[256]; + +static inline bool http_char_is_token(unsigned char ch) { + return (_http_char_lookup[ch] & _http_token_char_mask) != 0; +} + +static inline bool http_char_is_value(unsigned char ch) { + return (_http_char_lookup[ch] & _http_value_char_mask) != 0; +} + +static inline bool http_char_is_text(unsigned char ch) { + return (_http_char_lookup[ch] & _http_text_char_mask) != 0; +} + +static inline bool http_char_is_qdtext(unsigned char ch) { + return (_http_char_lookup[ch] & _http_qdtext_char_mask) != 0; +} + +static inline bool http_char_is_ctext(unsigned char ch) { + return (_http_char_lookup[ch] & _http_ctext_char_mask) != 0; +} + +static inline bool http_char_is_token68(unsigned char ch) { + return (_http_char_lookup[ch] & _http_token68_char_mask) != 0; +} + +/* + * HTTP value parsing + */ + +struct http_parser { + const unsigned char *begin, *cur, *end; +}; + +void http_parser_init(struct http_parser *parser, + const unsigned char *data, size_t size); + +void http_parse_ows(struct http_parser *parser); + +int http_parser_skip_token(struct http_parser *parser); +int http_parse_token(struct http_parser *parser, const char **token_r); +int http_parse_token_list_next(struct http_parser *parser, + const char **token_r); + +int http_parse_quoted_string(struct http_parser *parser, const char **str_r); +int http_parse_token_or_qstring(struct http_parser *parser, + const char **word_r); + +#endif diff --git a/src/lib-http/http-request-parser.c b/src/lib-http/http-request-parser.c new file mode 100644 index 0000000..8cb44ba --- /dev/null +++ b/src/lib-http/http-request-parser.c @@ -0,0 +1,635 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "http-url.h" +#include "http-parser.h" +#include "http-message-parser.h" +#include "http-request-parser.h" + +#define HTTP_REQUEST_PARSER_MAX_METHOD_LENGTH 32 + +enum http_request_parser_state { + HTTP_REQUEST_PARSE_STATE_INIT = 0, + HTTP_REQUEST_PARSE_STATE_SKIP_LINE, + HTTP_REQUEST_PARSE_STATE_METHOD, + HTTP_REQUEST_PARSE_STATE_SP1, + HTTP_REQUEST_PARSE_STATE_TARGET, + HTTP_REQUEST_PARSE_STATE_SP2, + HTTP_REQUEST_PARSE_STATE_VERSION, + HTTP_REQUEST_PARSE_STATE_CR, + HTTP_REQUEST_PARSE_STATE_LF, + HTTP_REQUEST_PARSE_STATE_HEADER +}; + +struct http_request_parser { + struct http_message_parser parser; + pool_t pool; + + enum http_request_parser_state state; + + struct http_url *default_base_url; + + uoff_t max_target_length; + + enum http_request_parse_error error_code; + + const char *request_method; + const char *request_target; + + bool skipping_line:1; +}; + +struct http_request_parser * +http_request_parser_init(struct istream *input, + const struct http_url *default_base_url, + const struct http_request_limits *limits, + enum http_request_parse_flags flags) +{ + struct http_request_parser *parser; + pool_t pool; + struct http_header_limits hdr_limits; + uoff_t max_payload_size; + enum http_message_parse_flags msg_flags = 0; + + pool = pool_alloconly_create("http request parser", 1024); + parser = p_new(pool, struct http_request_parser, 1); + parser->pool = pool; + + if (default_base_url != NULL) { + parser->default_base_url = + http_url_clone_authority(pool, default_base_url); + } + + if (limits != NULL) { + hdr_limits = limits->header; + max_payload_size = limits->max_payload_size; + } else { + i_zero(&hdr_limits); + max_payload_size = 0; + } + + /* substitute default limits */ + if (parser->max_target_length == 0) + parser->max_target_length = HTTP_REQUEST_DEFAULT_MAX_TARGET_LENGTH; + if (hdr_limits.max_size == 0) + hdr_limits.max_size = HTTP_REQUEST_DEFAULT_MAX_HEADER_SIZE; + if (hdr_limits.max_field_size == 0) + hdr_limits.max_field_size = HTTP_REQUEST_DEFAULT_MAX_HEADER_FIELD_SIZE; + if (hdr_limits.max_fields == 0) + hdr_limits.max_fields = HTTP_REQUEST_DEFAULT_MAX_HEADER_FIELDS; + if (max_payload_size == 0) + max_payload_size = HTTP_REQUEST_DEFAULT_MAX_PAYLOAD_SIZE; + + if ((flags & HTTP_REQUEST_PARSE_FLAG_STRICT) != 0) + msg_flags |= HTTP_MESSAGE_PARSE_FLAG_STRICT; + http_message_parser_init(&parser->parser, input, + &hdr_limits, max_payload_size, msg_flags); + return parser; +} + +void http_request_parser_deinit(struct http_request_parser **_parser) +{ + struct http_request_parser *parser = *_parser; + + *_parser = NULL; + + http_message_parser_deinit(&parser->parser); + pool_unref(&parser->pool); +} + +static void +http_request_parser_restart(struct http_request_parser *parser, + pool_t pool) +{ + http_message_parser_restart(&parser->parser, pool); + parser->request_method = NULL; + parser->request_target = NULL; +} + +static int http_request_parse_method(struct http_request_parser *parser) +{ + const unsigned char *p = parser->parser.cur; + pool_t pool; + + /* method = token + */ + while (p < parser->parser.end && http_char_is_token(*p)) + p++; + + if ((p - parser->parser.cur) > HTTP_REQUEST_PARSER_MAX_METHOD_LENGTH) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_METHOD_TOO_LONG; + parser->parser.error = "HTTP request method is too long"; + return -1; + } + if (p == parser->parser.end) + return 0; + pool = http_message_parser_get_pool(&parser->parser); + parser->request_method = + p_strdup_until(pool, parser->parser.cur, p); + parser->parser.cur = p; + return 1; +} + +static int http_request_parse_target(struct http_request_parser *parser) +{ + struct http_message_parser *_parser = &parser->parser; + const unsigned char *p = parser->parser.cur; + pool_t pool; + + /* We'll just parse anything up to the first SP or a control char. + We could also implement workarounds for buggy HTTP clients and + parse anything up to the HTTP-version and return 301 with the + target properly encoded (FIXME). */ + while (p < _parser->end && *p > ' ') + p++; + + /* target is too long when explicit limit is exceeded or when input buffer + runs out of space */ + /* FIXME: put limit on full request line rather than target and method + separately */ + /* FIXME: is it wise to keep target in stream buffer? It can become very + large for some applications, increasing the stream buffer size */ + if ((uoff_t)(p - _parser->cur) > parser->max_target_length || + (p == _parser->end && ((uoff_t)(p - _parser->cur) >= + i_stream_get_max_buffer_size(_parser->input)))) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_TARGET_TOO_LONG; + parser->parser.error = "HTTP request target is too long"; + return -1; + } + if (p == _parser->end) + return 0; + pool = http_message_parser_get_pool(_parser); + parser->request_target = p_strdup_until(pool, _parser->cur, p); + parser->parser.cur = p; + return 1; +} + +static inline const char *_chr_sanitize(unsigned char c) +{ + if (c >= 0x20 && c < 0x7F) + return t_strdup_printf("`%c'", c); + if (c == 0x0a) + return "<LF>"; + if (c == 0x0d) + return "<CR>"; + return t_strdup_printf("<0x%02x>", c); +} + +static int http_request_parse(struct http_request_parser *parser, + pool_t pool) +{ + struct http_message_parser *_parser = &parser->parser; + int ret; + + /* RFC 7230, Section 3.1.1: Request Line + + request-line = method SP request-target SP HTTP-version CRLF + method = token + */ + for (;;) { + switch (parser->state) { + case HTTP_REQUEST_PARSE_STATE_INIT: + http_request_parser_restart(parser, pool); + parser->state = HTTP_REQUEST_PARSE_STATE_SKIP_LINE; + if (_parser->cur == _parser->end) + return 0; + /* fall through */ + case HTTP_REQUEST_PARSE_STATE_SKIP_LINE: + if (*_parser->cur == '\r' || *_parser->cur == '\n') { + if (parser->skipping_line) { + /* second extra CRLF; not allowed */ + parser->error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST; + _parser->error = "Empty request line"; + return -1; + } + /* HTTP/1.0 client sent one extra CRLF after body. + ignore it. */ + parser->skipping_line = TRUE; + parser->state = HTTP_REQUEST_PARSE_STATE_CR; + break; + } + parser->state = HTTP_REQUEST_PARSE_STATE_METHOD; + parser->skipping_line = FALSE; + /* fall through */ + case HTTP_REQUEST_PARSE_STATE_METHOD: + if ((ret=http_request_parse_method(parser)) <= 0) + return ret; + parser->state = HTTP_REQUEST_PARSE_STATE_SP1; + if (_parser->cur == _parser->end) + return 0; + /* fall through */ + case HTTP_REQUEST_PARSE_STATE_SP1: + if (*_parser->cur != ' ') { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST; + _parser->error = t_strdup_printf + ("Unexpected character %s in request method", + _chr_sanitize(*_parser->cur)); + return -1; + } + _parser->cur++; + parser->state = HTTP_REQUEST_PARSE_STATE_TARGET; + if (_parser->cur >= _parser->end) + return 0; + /* fall through */ + case HTTP_REQUEST_PARSE_STATE_TARGET: + if ((ret=http_request_parse_target(parser)) <= 0) + return ret; + parser->state = HTTP_REQUEST_PARSE_STATE_SP2; + if (_parser->cur == _parser->end) + return 0; + /* fall through */ + case HTTP_REQUEST_PARSE_STATE_SP2: + if (*_parser->cur != ' ') { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST; + _parser->error = t_strdup_printf + ("Unexpected character %s in request target", + _chr_sanitize(*_parser->cur)); + return -1; + } + _parser->cur++; + parser->state = HTTP_REQUEST_PARSE_STATE_VERSION; + if (_parser->cur >= _parser->end) + return 0; + /* fall through */ + case HTTP_REQUEST_PARSE_STATE_VERSION: + if ((ret=http_message_parse_version(&parser->parser)) <= 0) { + if (ret < 0) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST; + _parser->error = "Invalid HTTP version in request"; + } + return ret; + } + parser->state = HTTP_REQUEST_PARSE_STATE_CR; + if (_parser->cur == _parser->end) + return 0; + /* fall through */ + case HTTP_REQUEST_PARSE_STATE_CR: + if (*_parser->cur == '\r') + _parser->cur++; + parser->state = HTTP_REQUEST_PARSE_STATE_LF; + if (_parser->cur == _parser->end) + return 0; + /* fall through */ + case HTTP_REQUEST_PARSE_STATE_LF: + if (*_parser->cur != '\n') { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST; + _parser->error = t_strdup_printf + ("Unexpected character %s at end of request line", + _chr_sanitize(*_parser->cur)); + return -1; + } + _parser->cur++; + if (!parser->skipping_line) { + parser->state = HTTP_REQUEST_PARSE_STATE_HEADER; + return 1; + } + parser->state = HTTP_REQUEST_PARSE_STATE_INIT; + break; + case HTTP_REQUEST_PARSE_STATE_HEADER: + default: + i_unreached(); + } + } + + i_unreached(); + return -1; +} + +static int http_request_parse_request_line(struct http_request_parser *parser, + pool_t pool) +{ + struct http_message_parser *_parser = &parser->parser; + const unsigned char *begin; + size_t size, old_bytes = 0; + int ret; + + while ((ret = i_stream_read_bytes(_parser->input, &begin, &size, + old_bytes + 1)) > 0) { + _parser->begin = _parser->cur = begin; + _parser->end = _parser->begin + size; + + if ((ret = http_request_parse(parser, pool)) < 0) + return -1; + + i_stream_skip(_parser->input, _parser->cur - begin); + if (ret > 0) + return 1; + old_bytes = i_stream_get_data_size(_parser->input); + } + + if (ret == -2) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST; + _parser->error = "HTTP request line is too long"; + return -1; + } + if (ret < 0) { + if (_parser->input->eof && + parser->state == HTTP_REQUEST_PARSE_STATE_INIT) + return 0; + parser->error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_STREAM; + _parser->error = "Broken stream"; + return -1; + } + return 0; +} + +static inline enum http_request_parse_error +http_request_parser_message_error(struct http_request_parser *parser) +{ + switch (parser->parser.error_code) { + case HTTP_MESSAGE_PARSE_ERROR_BROKEN_STREAM: + return HTTP_REQUEST_PARSE_ERROR_BROKEN_STREAM; + case HTTP_MESSAGE_PARSE_ERROR_BAD_MESSAGE: + return HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST; + case HTTP_MESSAGE_PARSE_ERROR_NOT_IMPLEMENTED: + return HTTP_REQUEST_PARSE_ERROR_NOT_IMPLEMENTED; + case HTTP_MESSAGE_PARSE_ERROR_PAYLOAD_TOO_LARGE: + return HTTP_REQUEST_PARSE_ERROR_PAYLOAD_TOO_LARGE; + case HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE: + return HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST; + default: + break; + } + i_unreached(); + return HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST; +} + +bool http_request_parser_pending_payload( + struct http_request_parser *parser) +{ + if (parser->parser.payload == NULL) + return FALSE; + return i_stream_have_bytes_left(parser->parser.payload); +} + +static int +http_request_parse_expect_header(struct http_request_parser *parser, + struct http_request *request, const struct http_header_field *hdr) +{ + struct http_message_parser *_parser = &parser->parser; + struct http_parser hparser; + bool parse_error = FALSE; + unsigned int num_expectations = 0; + + /* RFC 7231, Section 5.1.1: + + Expect = "100-continue" + */ + // FIXME: simplify; RFC 7231 discarded Expect extension mechanism + http_parser_init(&hparser, (const unsigned char *)hdr->value, hdr->size); + while (!parse_error) { + const char *expect_name, *expect_value; + + /* expect-name */ + if (http_parse_token(&hparser, &expect_name) > 0) { + num_expectations++; + if (strcasecmp(expect_name, "100-continue") == 0) { + request->expect_100_continue = TRUE; + } else { + if (parser->error_code == HTTP_REQUEST_PARSE_ERROR_NONE) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_EXPECTATION_FAILED; + _parser->error = t_strdup_printf + ("Unknown Expectation `%s'", expect_name); + } + } + + /* BWS "=" BWS */ + http_parse_ows(&hparser); + if (hparser.cur >= hparser.end) + break; + + if (*hparser.cur == '=') { + hparser.cur++; + http_parse_ows(&hparser); + + /* value */ + if (http_parse_token_or_qstring(&hparser, &expect_value) <= 0) { + parse_error = TRUE; + break; + } + + if (parser->error_code == HTTP_REQUEST_PARSE_ERROR_NONE) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_EXPECTATION_FAILED; + _parser->error = t_strdup_printf + ("Expectation `%s' has unexpected value", expect_name); + } + } + + /* *( OWS ";" [ OWS expect-param ] ) */ + while (!parse_error) { + const char *attribute, *value; + + /* OWS ";" */ + http_parse_ows(&hparser); + if (hparser.cur >= hparser.end || *hparser.cur != ';') + break; + hparser.cur++; + http_parse_ows(&hparser); + + /* expect-param */ + if (http_parse_token(&hparser, &attribute) <= 0) { + parse_error = TRUE; + break; + } + + /* BWS "=" BWS */ + http_parse_ows(&hparser); + if (hparser.cur >= hparser.end || *hparser.cur != '=') { + parse_error = TRUE; + break; + } + hparser.cur++; + http_parse_ows(&hparser); + + /* value */ + if (http_parse_token_or_qstring(&hparser, &value) <= 0) { + parse_error = TRUE; + break; + } + + if (parser->error_code == HTTP_REQUEST_PARSE_ERROR_NONE) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_EXPECTATION_FAILED; + _parser->error = t_strdup_printf + ("Expectation `%s' has unknown parameter `'%s'", + expect_name, attribute); + } + } + if (parse_error) + break; + } + http_parse_ows(&hparser); + if (hparser.cur >= hparser.end || *hparser.cur != ',') + break; + hparser.cur++; + http_parse_ows(&hparser); + } + + if (parse_error || hparser.cur < hparser.end) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST; + _parser->error = "Invalid Expect header"; + return -1; + } + + if (parser->error_code != HTTP_REQUEST_PARSE_ERROR_NONE) + return -1; + + if (num_expectations == 0) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST; + _parser->error = "Empty Expect header"; + return -1; + } + return 0; +} + +static int +http_request_parse_headers(struct http_request_parser *parser, + struct http_request *request) +{ + const ARRAY_TYPE(http_header_field) *hdrs; + const struct http_header_field *hdr; + + hdrs = http_header_get_fields(parser->parser.msg.header); + array_foreach(hdrs, hdr) { + int ret = 0; + + /* Expect: */ + if (http_header_field_is(hdr, "Expect")) + ret = http_request_parse_expect_header(parser, request, hdr); + + if (ret < 0) + return -1; + } + return 0; +} + +int http_request_parse_finish_payload( + struct http_request_parser *parser, + enum http_request_parse_error *error_code_r, + const char **error_r) +{ + int ret; + + *error_code_r = parser->error_code = HTTP_REQUEST_PARSE_ERROR_NONE; + *error_r = parser->parser.error = NULL; + + /* make sure we finished streaming payload from previous request + before we continue. */ + if ((ret = http_message_parse_finish_payload(&parser->parser)) <= 0) { + if (ret < 0) { + *error_code_r = http_request_parser_message_error(parser); + *error_r = parser->parser.error; + } + } + return ret; +} + +int http_request_parse_next(struct http_request_parser *parser, + pool_t pool, struct http_request *request, + enum http_request_parse_error *error_code_r, const char **error_r) +{ + const struct http_header_field *hdr; + const char *host_hdr, *error; + int ret; + + /* initialize and get rid of any payload of previous request */ + if ((ret=http_request_parse_finish_payload + (parser, error_code_r, error_r)) <= 0) + return ret; + + /* RFC 7230, Section 3: + + HTTP-message = start-line + *( header-field CRLF ) + CRLF + [ message-body ] + */ + if (parser->state != HTTP_REQUEST_PARSE_STATE_HEADER) { + ret = http_request_parse_request_line(parser, pool); + + /* assign early for error reporting */ + request->method = parser->request_method; + request->target_raw = parser->request_target; + request->version_major = parser->parser.msg.version_major; + request->version_minor = parser->parser.msg.version_minor; + + if (ret <= 0) { + if (ret < 0) { + *error_code_r = parser->error_code; + *error_r = parser->parser.error; + } + return ret; + } + } + + if ((ret = http_message_parse_headers(&parser->parser)) <= 0) { + if (ret < 0) { + *error_code_r = http_request_parser_message_error(parser); + *error_r = parser->parser.error; + } + return ret; + } + + if (http_message_parse_body(&parser->parser, TRUE) < 0) { + *error_code_r = http_request_parser_message_error(parser); + *error_r = parser->parser.error; + return -1; + } + parser->state = HTTP_REQUEST_PARSE_STATE_INIT; + + /* RFC 7230, Section 5.4: Host + + A server MUST respond with a 400 (Bad Request) status code to any + HTTP/1.1 request message that lacks a Host header field and to any + request message that contains more than one Host header field or a + Host header field with an invalid field-value. + */ + host_hdr = NULL; + if (parser->parser.msg.version_major == 1 && + parser->parser.msg.version_minor > 0) { + if ((ret=http_header_field_find_unique( + parser->parser.msg.header, "Host", &hdr)) <= 0) { + *error_code_r = HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST; + if (ret == 0) + *error_r = "Missing Host header"; + else + *error_r = "Duplicate Host header"; + return -1; + } + + host_hdr = hdr->value; + } + + i_zero(request); + + pool = http_message_parser_get_pool(&parser->parser); + if (http_url_request_target_parse(parser->request_target, host_hdr, + parser->default_base_url, pool, &request->target, &error) < 0) { + *error_code_r = HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST; + *error_r = t_strdup_printf("Bad request target `%s': %s", + parser->request_target, error); + return -1; + } + + /* parse request-specific headers */ + if (http_request_parse_headers(parser, request) < 0) { + *error_code_r = parser->error_code; + *error_r = parser->parser.error; + return -1; + } + + request->method = parser->request_method; + request->target_raw = parser->request_target; + request->version_major = parser->parser.msg.version_major; + request->version_minor = parser->parser.msg.version_minor; + request->date = parser->parser.msg.date; + request->payload = parser->parser.payload; + request->header = parser->parser.msg.header; + request->connection_options = parser->parser.msg.connection_options; + request->connection_close = parser->parser.msg.connection_close; + + /* reset this state early */ + parser->request_method = NULL; + parser->request_target = NULL; + return 1; +} diff --git a/src/lib-http/http-request-parser.h b/src/lib-http/http-request-parser.h new file mode 100644 index 0000000..3956062 --- /dev/null +++ b/src/lib-http/http-request-parser.h @@ -0,0 +1,43 @@ +#ifndef HTTP_REQUEST_PARSER_H +#define HTTP_REQUEST_PARSER_H + +#include "http-request.h" + +enum http_request_parse_error { + HTTP_REQUEST_PARSE_ERROR_NONE = 0, /* no error */ + HTTP_REQUEST_PARSE_ERROR_BROKEN_STREAM, /* stream error */ + HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST, /* unrecoverable generic error */ + HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST, /* recoverable generic error */ + HTTP_REQUEST_PARSE_ERROR_NOT_IMPLEMENTED, /* used unimplemented feature + (recoverable) */ + HTTP_REQUEST_PARSE_ERROR_EXPECTATION_FAILED, /* unknown item in Expect: + header (recoverable) */ + HTTP_REQUEST_PARSE_ERROR_METHOD_TOO_LONG, /* method too long (fatal) */ + HTTP_REQUEST_PARSE_ERROR_TARGET_TOO_LONG, /* target too long (fatal) */ + HTTP_REQUEST_PARSE_ERROR_PAYLOAD_TOO_LARGE /* payload too large (fatal) */ +}; + +enum http_request_parse_flags { + /* Strictly adhere to the HTTP protocol specification */ + HTTP_REQUEST_PARSE_FLAG_STRICT = BIT(0) +}; + +struct http_request_parser * +http_request_parser_init(struct istream *input, + const struct http_url *default_base_url, + const struct http_request_limits *limits, + enum http_request_parse_flags flags) ATTR_NULL(2); +void http_request_parser_deinit(struct http_request_parser **_parser); + +int http_request_parse_finish_payload( + struct http_request_parser *parser, + enum http_request_parse_error *error_code_r, + const char **error_r); + +int http_request_parse_next(struct http_request_parser *parser, + pool_t pool, struct http_request *request, + enum http_request_parse_error *error_code_r, const char **error_r); + +bool http_request_parser_pending_payload(struct http_request_parser *parser); + +#endif diff --git a/src/lib-http/http-request.c b/src/lib-http/http-request.c new file mode 100644 index 0000000..aed9975 --- /dev/null +++ b/src/lib-http/http-request.c @@ -0,0 +1,32 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" + +#include "http-request.h" + +bool http_request_has_connection_option(const struct http_request *req, + const char *option) +{ + const char *opt; + + if (!array_is_created(&req->connection_options)) + return FALSE; + array_foreach_elem(&req->connection_options, opt) { + if (strcasecmp(opt, option) == 0) + return TRUE; + } + return FALSE; +} + +int http_request_get_payload_size(const struct http_request *req, + uoff_t *size_r) +{ + if (req->payload == NULL) { + *size_r = 0; + return 1; + } + + return i_stream_get_size(req->payload, TRUE, size_r); +} diff --git a/src/lib-http/http-request.h b/src/lib-http/http-request.h new file mode 100644 index 0000000..feee28f --- /dev/null +++ b/src/lib-http/http-request.h @@ -0,0 +1,84 @@ +#ifndef HTTP_REQUEST_H +#define HTTP_REQUEST_H + +#include "http-header.h" + +struct http_url; + +#define HTTP_REQUEST_DEFAULT_MAX_TARGET_LENGTH (8 * 1024) +#define HTTP_REQUEST_DEFAULT_MAX_HEADER_SIZE (200 * 1024) +#define HTTP_REQUEST_DEFAULT_MAX_HEADER_FIELD_SIZE (8 * 1024) +#define HTTP_REQUEST_DEFAULT_MAX_HEADER_FIELDS 50 +#define HTTP_REQUEST_DEFAULT_MAX_PAYLOAD_SIZE (1 * 1024 * 1024) + +struct http_request_limits { + uoff_t max_target_length; + uoff_t max_payload_size; + + struct http_header_limits header; +}; + +enum http_request_target_format { + HTTP_REQUEST_TARGET_FORMAT_ORIGIN = 0, + HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE, + HTTP_REQUEST_TARGET_FORMAT_AUTHORITY, + HTTP_REQUEST_TARGET_FORMAT_ASTERISK +}; + +struct http_request_target { + enum http_request_target_format format; + struct http_url *url; +}; + +struct http_request { + const char *method; + + const char *target_raw; + struct http_request_target target; + + unsigned char version_major; + unsigned char version_minor; + + time_t date; + const struct http_header *header; + struct istream *payload; + + ARRAY_TYPE(const_string) connection_options; + + bool connection_close:1; + bool expect_100_continue:1; +}; + +static inline bool +http_request_method_is(const struct http_request *req, const char *method) +{ + if (req->method == NULL) + return FALSE; + + return (strcmp(req->method, method) == 0); +} + +static inline const struct http_header_field * +http_request_header_find(const struct http_request *req, const char *name) +{ + return http_header_field_find(req->header, name); +} + +static inline const char * +http_request_header_get(const struct http_request *req, const char *name) +{ + return http_header_field_get(req->header, name); +} + +static inline const ARRAY_TYPE(http_header_field) * +http_request_header_get_fields(const struct http_request *req) +{ + return http_header_get_fields(req->header); +} + +bool http_request_has_connection_option(const struct http_request *req, + const char *option); +int http_request_get_payload_size(const struct http_request *req, + uoff_t *size_r); + +#endif diff --git a/src/lib-http/http-response-parser.c b/src/lib-http/http-response-parser.c new file mode 100644 index 0000000..e381403 --- /dev/null +++ b/src/lib-http/http-response-parser.c @@ -0,0 +1,422 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "istream.h" +#include "http-parser.h" +#include "http-date.h" +#include "http-message-parser.h" +#include "http-response-parser.h" + +#include <ctype.h> + +enum http_response_parser_state { + HTTP_RESPONSE_PARSE_STATE_INIT = 0, + HTTP_RESPONSE_PARSE_STATE_VERSION, + HTTP_RESPONSE_PARSE_STATE_SP1, + HTTP_RESPONSE_PARSE_STATE_STATUS, + HTTP_RESPONSE_PARSE_STATE_SP2, + HTTP_RESPONSE_PARSE_STATE_REASON, + HTTP_RESPONSE_PARSE_STATE_CR, + HTTP_RESPONSE_PARSE_STATE_LF, + HTTP_RESPONSE_PARSE_STATE_HEADER +}; + +struct http_response_parser { + struct http_message_parser parser; + enum http_response_parser_state state; + + unsigned int response_status; + const char *response_reason; + + uoff_t response_offset; +}; + +struct http_response_parser * +http_response_parser_init(struct istream *input, + const struct http_header_limits *hdr_limits, + enum http_response_parse_flags flags) +{ + struct http_response_parser *parser; + enum http_message_parse_flags msg_flags = 0; + + /* FIXME: implement status line limit */ + if ((flags & HTTP_RESPONSE_PARSE_FLAG_STRICT) != 0) + msg_flags |= HTTP_MESSAGE_PARSE_FLAG_STRICT; + parser = i_new(struct http_response_parser, 1); + http_message_parser_init(&parser->parser, + input, hdr_limits, 0, msg_flags); + return parser; +} + +void http_response_parser_deinit(struct http_response_parser **_parser) +{ + struct http_response_parser *parser = *_parser; + + *_parser = NULL; + + http_message_parser_deinit(&parser->parser); + i_free(parser); +} + +static void +http_response_parser_restart(struct http_response_parser *parser) +{ + http_message_parser_restart(&parser->parser, NULL); + parser->response_status = 0; + parser->response_reason = NULL; + parser->response_offset = UOFF_T_MAX; +} + +static int http_response_parse_status(struct http_response_parser *parser) +{ + const unsigned char *p = parser->parser.cur; + const size_t size = parser->parser.end - parser->parser.cur; + + /* status-code = 3DIGIT + */ + if (size < 3) + return 0; + if (!i_isdigit(p[0]) || !i_isdigit(p[1]) || !i_isdigit(p[2])) + return -1; + parser->response_status = + (p[0] - '0')*100 + (p[1] - '0')*10 + (p[2] - '0'); + if (parser->response_status < 100 || + parser->response_status >= 600) + return -1; + parser->parser.cur += 3; + return 1; +} + +static int http_response_parse_reason(struct http_response_parser *parser) +{ + const unsigned char *p = parser->parser.cur; + pool_t pool; + + /* reason-phrase = *( HTAB / SP / VCHAR / obs-text ) + */ + // FIXME: limit length + while (p < parser->parser.end && http_char_is_text(*p)) + p++; + + if (p == parser->parser.end) + return 0; + pool = http_message_parser_get_pool(&parser->parser); + parser->response_reason = + p_strdup_until(pool, parser->parser.cur, p); + parser->parser.cur = p; + return 1; +} + +static const char *_reply_sanitize(struct http_message_parser *parser) +{ + string_t *str = t_str_new(32); + const unsigned char *p; + unsigned int i; + bool quote_open = FALSE; + + i_assert(parser->cur < parser->end); + for (p = parser->cur, i = 0; p < parser->end && i < 20; p++, i++) { + if (*p >= 0x20 && *p < 0x7F) { + if (!quote_open) { + str_append_c(str, '`'); + quote_open = TRUE; + } + str_append_c(str, *p); + } else { + if (quote_open) { + str_append_c(str, '\''); + quote_open = FALSE; + } + if (*p == 0x0a) + str_append(str, "<LF>"); + else if (*p == 0x0d) + str_append(str, "<CR>"); + else + str_printfa(str, "<0x%02x>", *p); + } + } + if (quote_open) + str_append_c(str, '\''); + return str_c(str); +} + +static int http_response_parse(struct http_response_parser *parser) +{ + struct http_message_parser *_parser = &parser->parser; + int ret; + + /* RFC 7230, Section 3.1.2: Status Line + + status-line = HTTP-version SP status-code SP reason-phrase CRLF + status-code = 3DIGIT + reason-phrase = *( HTAB / SP / VCHAR / obs-text ) + */ + switch (parser->state) { + case HTTP_RESPONSE_PARSE_STATE_INIT: + parser->state = HTTP_RESPONSE_PARSE_STATE_VERSION; + parser->response_offset = _parser->input->v_offset + + (_parser->cur - _parser->begin); + /* fall through */ + case HTTP_RESPONSE_PARSE_STATE_VERSION: + if ((ret=http_message_parse_version(_parser)) <= 0) { + if (ret < 0) + _parser->error = t_strdup_printf( + "Invalid HTTP version in response: %s", + _reply_sanitize(_parser)); + return ret; + } + parser->state = HTTP_RESPONSE_PARSE_STATE_SP1; + if (_parser->cur == _parser->end) + return 0; + /* fall through */ + case HTTP_RESPONSE_PARSE_STATE_SP1: + if (*_parser->cur != ' ') { + _parser->error = t_strdup_printf + ("Expected ' ' after response version, but found %s", + _reply_sanitize(_parser)); + return -1; + } + _parser->cur++; + parser->state = HTTP_RESPONSE_PARSE_STATE_STATUS; + if (_parser->cur >= _parser->end) + return 0; + /* fall through */ + case HTTP_RESPONSE_PARSE_STATE_STATUS: + if ((ret=http_response_parse_status(parser)) <= 0) { + if (ret < 0) + _parser->error = "Invalid HTTP status code in response"; + return ret; + } + parser->state = HTTP_RESPONSE_PARSE_STATE_SP2; + if (_parser->cur == _parser->end) + return 0; + /* fall through */ + case HTTP_RESPONSE_PARSE_STATE_SP2: + if (*_parser->cur != ' ') { + _parser->error = t_strdup_printf + ("Expected ' ' after response status code, but found %s", + _reply_sanitize(_parser)); + return -1; + } + _parser->cur++; + parser->state = HTTP_RESPONSE_PARSE_STATE_REASON; + if (_parser->cur >= _parser->end) + return 0; + /* fall through */ + case HTTP_RESPONSE_PARSE_STATE_REASON: + if ((ret=http_response_parse_reason(parser)) <= 0) { + i_assert(ret == 0); + return 0; + } + parser->state = HTTP_RESPONSE_PARSE_STATE_CR; + if (_parser->cur == _parser->end) + return 0; + /* fall through */ + case HTTP_RESPONSE_PARSE_STATE_CR: + if (*_parser->cur == '\r') + _parser->cur++; + parser->state = HTTP_RESPONSE_PARSE_STATE_LF; + if (_parser->cur == _parser->end) + return 0; + /* fall through */ + case HTTP_RESPONSE_PARSE_STATE_LF: + if (*_parser->cur != '\n') { + _parser->error = t_strdup_printf + ("Expected line end after response, but found %s", + _reply_sanitize(_parser)); + return -1; + } + _parser->cur++; + parser->state = HTTP_RESPONSE_PARSE_STATE_HEADER; + return 1; + case HTTP_RESPONSE_PARSE_STATE_HEADER: + default: + break; + } + + i_unreached(); + return -1; +} + +static int +http_response_parse_status_line(struct http_response_parser *parser) +{ + struct http_message_parser *_parser = &parser->parser; + const unsigned char *begin; + size_t size, old_bytes = 0; + int ret; + + while ((ret = i_stream_read_bytes(_parser->input, &begin, &size, + old_bytes + 1)) > 0) { + _parser->begin = _parser->cur = begin; + _parser->end = _parser->begin + size; + + if ((ret = http_response_parse(parser)) < 0) + return -1; + + i_stream_skip(_parser->input, _parser->cur - begin); + if (ret > 0) + return 1; + old_bytes = i_stream_get_data_size(_parser->input); + } + + if (ret == -2) { + _parser->error = "HTTP status line is too long"; + return -1; + } + if (ret < 0) { + if (_parser->input->eof && + parser->state == HTTP_RESPONSE_PARSE_STATE_INIT) + return 0; + _parser->error = t_strdup_printf("Stream error: %s", + i_stream_get_error(_parser->input)); + return -1; + } + return 0; +} + +static int +http_response_parse_retry_after(const char *hdrval, time_t resp_time, + time_t *retry_after_r) +{ + time_t delta; + + /* RFC 7231, Section 7.1.3: Retry-After + + The value of this field can be either an HTTP-date or a number of + seconds to delay after the response is received. + + Retry-After = HTTP-date / delta-seconds + + A delay-seconds value is a non-negative decimal integer, representing + time in seconds. + + delta-seconds = 1*DIGIT + */ + if (str_to_time(hdrval, &delta) >= 0) { + if (resp_time == (time_t)-1) { + return -1; + } + *retry_after_r = resp_time + delta; + return 0; + } + + return (http_date_parse + ((const unsigned char *)hdrval, strlen(hdrval), retry_after_r) ? 0 : -1); +} + +uoff_t http_response_parser_get_last_offset(struct http_response_parser *parser) +{ + return parser->response_offset; +} + +int http_response_parse_next(struct http_response_parser *parser, + enum http_response_payload_type payload_type, + struct http_response *response, const char **error_r) +{ + const char *hdrval; + time_t retry_after = (time_t)-1; + int ret; + + i_zero(response); + + /* make sure we finished streaming payload from previous response + before we continue. */ + if ((ret = http_message_parse_finish_payload(&parser->parser)) <= 0) { + *error_r = parser->parser.error; + return ret; + } + + if (parser->state == HTTP_RESPONSE_PARSE_STATE_INIT) + http_response_parser_restart(parser); + + /* RFC 7230, Section 3: + + HTTP-message = start-line + *( header-field CRLF ) + CRLF + [ message-body ] + */ + if (parser->state != HTTP_RESPONSE_PARSE_STATE_HEADER) { + if ((ret = http_response_parse_status_line(parser)) <= 0) { + *error_r = parser->parser.error; + return ret; + } + } + if ((ret = http_message_parse_headers(&parser->parser)) <= 0) { + *error_r = parser->parser.error; + return ret; + } + + /* RFC 7230, Section 3.3.2: Content-Length + + A server MUST NOT send a Content-Length header field in any response + with a status code of 1xx (Informational) or 204 (No Content). + */ + if ((parser->response_status / 100 == 1 || parser->response_status == 204) && + parser->parser.msg.content_length > 0) { + *error_r = t_strdup_printf( + "Unexpected Content-Length header field for %u response " + "(length=%"PRIuUOFF_T")", parser->response_status, + parser->parser.msg.content_length); + return -1; + } + + /* RFC 7230, Section 3.3.3: Message Body Length + + 1. Any response to a HEAD request and any response with a 1xx + (Informational), 204 (No Content), or 304 (Not Modified) status + code is always terminated by the first empty line after the + header fields, regardless of the header fields present in the + message, and thus cannot contain a message body. + */ + if (parser->response_status / 100 == 1 || parser->response_status == 204 + || parser->response_status == 304) { // HEAD is handled in caller + payload_type = HTTP_RESPONSE_PAYLOAD_TYPE_NOT_PRESENT; + } + + if ((payload_type == HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED) || + (payload_type == HTTP_RESPONSE_PAYLOAD_TYPE_ONLY_UNSUCCESSFUL && + parser->response_status / 100 != 2)) { + /* [ message-body ] */ + if (http_message_parse_body(&parser->parser, FALSE) < 0) { + *error_r = parser->parser.error; + return -1; + } + } + + /* RFC 7231, Section 7.1.3: Retry-After + + Servers send the "Retry-After" header field to indicate how long the + user agent ought to wait before making a follow-up request. When + sent with a 503 (Service Unavailable) response, Retry-After indicates + how long the service is expected to be unavailable to the client. + When sent with any 3xx (Redirection) response, Retry-After indicates + the minimum time that the user agent is asked to wait before issuing + the redirected request. + */ + if (parser->response_status == 503 || (parser->response_status / 100) == 3) { + hdrval = http_header_field_get(parser->parser.msg.header, "Retry-After"); + if (hdrval != NULL) { + (void)http_response_parse_retry_after + (hdrval, parser->parser.msg.date, &retry_after); + /* broken Retry-After header is ignored */ + } + } + + parser->state = HTTP_RESPONSE_PARSE_STATE_INIT; + + response->status = parser->response_status; + response->reason = parser->response_reason; + response->version_major = parser->parser.msg.version_major; + response->version_minor = parser->parser.msg.version_minor; + response->location = parser->parser.msg.location; + response->date = parser->parser.msg.date; + response->retry_after = retry_after; + response->payload = parser->parser.payload; + response->header = parser->parser.msg.header; + response->connection_options = parser->parser.msg.connection_options; + response->connection_close = parser->parser.msg.connection_close; + return 1; +} diff --git a/src/lib-http/http-response-parser.h b/src/lib-http/http-response-parser.h new file mode 100644 index 0000000..8777048 --- /dev/null +++ b/src/lib-http/http-response-parser.h @@ -0,0 +1,26 @@ +#ifndef HTTP_RESPONSE_PARSER_H +#define HTTP_RESPONSE_PARSER_H + +#include "http-response.h" + +struct http_header_limits; +struct http_response_parser; + +enum http_response_parse_flags { + /* Strictly adhere to the HTTP protocol specification */ + HTTP_RESPONSE_PARSE_FLAG_STRICT = BIT(0) +}; + +struct http_response_parser * +http_response_parser_init(struct istream *input, + const struct http_header_limits *hdr_limits, + enum http_response_parse_flags flags) ATTR_NULL(2); +void http_response_parser_deinit(struct http_response_parser **_parser); + +uoff_t http_response_parser_get_last_offset(struct http_response_parser *parser); + +int http_response_parse_next(struct http_response_parser *parser, + enum http_response_payload_type payload_type, + struct http_response *response, const char **error_r); + +#endif diff --git a/src/lib-http/http-response.c b/src/lib-http/http-response.c new file mode 100644 index 0000000..e85a730 --- /dev/null +++ b/src/lib-http/http-response.c @@ -0,0 +1,46 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "istream.h" + +#include "http-response.h" + +void +http_response_init(struct http_response *resp, + unsigned int status, const char *reason) +{ + i_zero(resp); + resp->version_major = 1; + resp->version_minor = 1; + resp->date = ioloop_time; + resp->status = status; + resp->reason = reason; +} + +bool http_response_has_connection_option(const struct http_response *resp, + const char *option) +{ + const char *opt; + + if (!array_is_created(&resp->connection_options)) + return FALSE; + array_foreach_elem(&resp->connection_options, opt) { + if (strcasecmp(opt, option) == 0) + return TRUE; + } + return FALSE; +} + +int http_response_get_payload_size(const struct http_response *resp, + uoff_t *size_r) +{ + if (resp->payload == NULL) { + *size_r = 0; + return 1; + } + + return i_stream_get_size(resp->payload, TRUE, size_r); +} + diff --git a/src/lib-http/http-response.h b/src/lib-http/http-response.h new file mode 100644 index 0000000..30f6147 --- /dev/null +++ b/src/lib-http/http-response.h @@ -0,0 +1,87 @@ +#ifndef HTTP_RESPONSE_H +#define HTTP_RESPONSE_H + +#include "array.h" + +#include "http-header.h" + +#define HTTP_RESPONSE_STATUS_INTERNAL 9000 + +enum http_response_payload_type { + HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED, + HTTP_RESPONSE_PAYLOAD_TYPE_NOT_PRESENT, + HTTP_RESPONSE_PAYLOAD_TYPE_ONLY_UNSUCCESSFUL +}; + +struct http_response { + unsigned char version_major; + unsigned char version_minor; + + unsigned int status; + + const char *reason; + const char *location; + + time_t date, retry_after; + const struct http_header *header; + struct istream *payload; + + ARRAY_TYPE(const_string) connection_options; + + bool connection_close:1; +}; + +static inline bool +http_response_is_success(const struct http_response *resp) +{ + return ((resp->status / 100) == 2); +} + +static inline bool +http_response_is_internal_error(const struct http_response *resp) +{ + return (resp->status >= HTTP_RESPONSE_STATUS_INTERNAL); +} + +void +http_response_init(struct http_response *resp, + unsigned int status, const char *reason); + +static inline const struct http_header_field * +http_response_header_find(const struct http_response *resp, const char *name) +{ + if (resp->header == NULL) + return NULL; + return http_header_field_find(resp->header, name); +} + +static inline const char * +http_response_header_get(const struct http_response *resp, const char *name) +{ + if (resp->header == NULL) + return NULL; + return http_header_field_get(resp->header, name); +} + +static inline const ARRAY_TYPE(http_header_field) * +http_response_header_get_fields(const struct http_response *resp) +{ + if (resp->header == NULL) + return NULL; + return http_header_get_fields(resp->header); +} + +static inline const char * +http_response_get_message(const struct http_response *resp) +{ + if (resp->status >= HTTP_RESPONSE_STATUS_INTERNAL) + return resp->reason; + return t_strdup_printf("%u %s", resp->status, resp->reason); +} + +bool http_response_has_connection_option(const struct http_response *resp, + const char *option); +int http_response_get_payload_size(const struct http_response *resp, + uoff_t *size_r); + +#endif diff --git a/src/lib-http/http-server-connection.c b/src/lib-http/http-server-connection.c new file mode 100644 index 0000000..801a703 --- /dev/null +++ b/src/lib-http/http-server-connection.c @@ -0,0 +1,1180 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "llist.h" +#include "array.h" +#include "str.h" +#include "hostpid.h" +#include "ioloop.h" +#include "istream.h" +#include "istream-timeout.h" +#include "ostream.h" +#include "connection.h" +#include "iostream-rawlog.h" +#include "iostream-ssl.h" +#include "master-service.h" +#include "master-service-ssl.h" +#include "http-date.h" +#include "http-url.h" +#include "http-request-parser.h" + +#include "http-server-private.h" + +static void +http_server_connection_disconnect(struct http_server_connection *conn, + const char *reason); + +static bool +http_server_connection_unref_is_closed(struct http_server_connection *conn); + +/* + * Logging + */ + +static inline void +http_server_connection_client_error(struct http_server_connection *conn, + const char *format, ...) ATTR_FORMAT(2, 3); + +static inline void +http_server_connection_client_error(struct http_server_connection *conn, + const char *format, ...) +{ + va_list args; + + va_start(args, format); + e_info(conn->event, "%s", t_strdup_vprintf(format, args)); + va_end(args); +} + +/* + * Connection + */ + +static void http_server_connection_input(struct connection *_conn); + +static void +http_server_connection_update_stats(struct http_server_connection *conn) +{ + if (conn->conn.input != NULL) + conn->stats.input = conn->conn.input->v_offset; + if (conn->conn.output != NULL) + conn->stats.output = conn->conn.output->offset; +} + +const struct http_server_stats * +http_server_connection_get_stats(struct http_server_connection *conn) +{ + http_server_connection_update_stats(conn); + return &conn->stats; +} + +void http_server_connection_input_set_pending( + struct http_server_connection *conn) +{ + i_stream_set_input_pending(conn->conn.input, TRUE); +} + +void http_server_connection_input_halt(struct http_server_connection *conn) +{ + connection_input_halt(&conn->conn); +} + +void http_server_connection_input_resume(struct http_server_connection *conn) +{ + if (conn->closed || conn->input_broken || conn->close_indicated || + conn->incoming_payload != NULL) { + /* Connection not usable */ + return; + } + + if (conn->in_req_callback) { + struct http_server_request *req = conn->request_queue_tail; + + /* Currently running request callback for this connection. Only + handle discarded request payload. */ + if (req == NULL || + req->state != HTTP_SERVER_REQUEST_STATE_SUBMITTED_RESPONSE) + return; + if (!http_server_connection_pending_payload(conn)) + return; + } + + connection_input_resume(&conn->conn); +} + +static void +http_server_connection_idle_timeout(struct http_server_connection *conn) +{ + http_server_connection_client_error( + conn, "Disconnected for inactivity"); + http_server_connection_close(&conn, "Disconnected for inactivity"); +} + +void http_server_connection_start_idle_timeout( + struct http_server_connection *conn) +{ + unsigned int timeout_msecs = + conn->server->set.max_client_idle_time_msecs; + + if (conn->to_idle == NULL && timeout_msecs > 0) { + conn->to_idle = timeout_add(timeout_msecs, + http_server_connection_idle_timeout, + conn); + } +} + +void http_server_connection_reset_idle_timeout( + struct http_server_connection *conn) +{ + if (conn->to_idle != NULL) + timeout_reset(conn->to_idle); +} + +void http_server_connection_stop_idle_timeout( + struct http_server_connection *conn) +{ + timeout_remove(&conn->to_idle); +} + +bool http_server_connection_shut_down(struct http_server_connection *conn) +{ + if (conn->request_queue_head == NULL || + conn->request_queue_head->state == HTTP_SERVER_REQUEST_STATE_NEW) { + http_server_connection_close(&conn, "Server shutting down"); + return TRUE; + } + return FALSE; +} + +static void http_server_connection_ready(struct http_server_connection *conn) +{ + const struct http_server_settings *set = &conn->server->set; + struct http_url base_url; + struct stat st; + + if (conn->server->set.rawlog_dir != NULL && + stat(conn->server->set.rawlog_dir, &st) == 0) { + iostream_rawlog_create(conn->server->set.rawlog_dir, + &conn->conn.input, &conn->conn.output); + } + + i_zero(&base_url); + if (set->default_host != NULL) + base_url.host.name = set->default_host; + else + base_url.host.name = my_hostname; + base_url.have_ssl = conn->ssl; + + conn->http_parser = http_request_parser_init( + conn->conn.input, &base_url, &conn->server->set.request_limits, + HTTP_REQUEST_PARSE_FLAG_STRICT); + o_stream_set_finish_via_child(conn->conn.output, FALSE); + o_stream_set_flush_callback(conn->conn.output, + http_server_connection_output, conn); +} + +static void http_server_connection_destroy(struct connection *_conn) +{ + struct http_server_connection *conn = + (struct http_server_connection *)_conn; + + http_server_connection_disconnect(conn, NULL); + http_server_connection_unref(&conn); +} + +static void http_server_payload_destroyed(struct http_server_request *req) +{ + struct http_server_connection *conn = req->conn; + int stream_errno; + + i_assert(conn != NULL); + i_assert(conn->request_queue_tail == req || + req->state >= HTTP_SERVER_REQUEST_STATE_FINISHED); + i_assert(conn->conn.io == NULL); + + e_debug(conn->event, "Request payload stream destroyed"); + + /* Caller is allowed to change the socket fd to blocking while reading + the payload. make sure here that it's switched back. */ + net_set_nonblock(conn->conn.fd_in, TRUE); + + stream_errno = conn->incoming_payload->stream_errno; + conn->incoming_payload = NULL; + + if (conn->payload_handler != NULL) + http_server_payload_handler_destroy(&conn->payload_handler); + + /* Handle errors in transfer stream */ + if (req->response == NULL && stream_errno != 0 && + conn->conn.input->stream_errno == 0) { + switch (stream_errno) { + case EMSGSIZE: + conn->input_broken = TRUE; + http_server_connection_client_error( + conn, "Client sent excessively large request"); + http_server_request_fail_close(req, 413, + "Payload Too Large"); + return; + case EIO: + conn->input_broken = TRUE; + http_server_connection_client_error( + conn, "Client sent invalid request payload"); + http_server_request_fail_close(req, 400, + "Bad Request"); + return; + default: + break; + } + } + + /* Resource stopped reading payload; update state */ + switch (req->state) { + case HTTP_SERVER_REQUEST_STATE_QUEUED: + case HTTP_SERVER_REQUEST_STATE_PAYLOAD_IN: + /* Finished reading request */ + req->state = HTTP_SERVER_REQUEST_STATE_PROCESSING; + http_server_connection_stop_idle_timeout(conn); + if (req->response != NULL && req->response->submitted) + http_server_request_submit_response(req); + break; + case HTTP_SERVER_REQUEST_STATE_PROCESSING: + /* No response submitted yet */ + break; + case HTTP_SERVER_REQUEST_STATE_SUBMITTED_RESPONSE: + /* Response submitted, but not all payload is necessarily read + */ + if (http_server_request_is_complete(req)) + http_server_request_ready_to_respond(req); + break; + case HTTP_SERVER_REQUEST_STATE_READY_TO_RESPOND: + case HTTP_SERVER_REQUEST_STATE_FINISHED: + case HTTP_SERVER_REQUEST_STATE_ABORTED: + /* Nothing to do */ + break; + default: + i_unreached(); + } + + /* Input stream may have pending input. */ + http_server_connection_input_resume(conn); + http_server_connection_input_set_pending(conn); +} + +static bool +http_server_connection_handle_request(struct http_server_connection *conn, + struct http_server_request *req) +{ + const struct http_server_settings *set = &conn->server->set; + unsigned int old_refcount; + struct istream *payload; + + i_assert(!conn->in_req_callback); + i_assert(conn->incoming_payload == NULL); + + if (req->req.version_major != 1) { + http_server_request_fail(req, 505, + "HTTP Version Not Supported"); + return TRUE; + } + + req->state = HTTP_SERVER_REQUEST_STATE_QUEUED; + + if (req->req.payload != NULL) { + /* Wrap the stream to capture the destroy event without + destroying the actual payload stream. */ + conn->incoming_payload = req->req.payload = + i_stream_create_timeout( + req->req.payload, + set->max_client_idle_time_msecs); + /* We've received the request itself, and we can't reset the + timeout during the payload reading. */ + http_server_connection_stop_idle_timeout(conn); + } else { + conn->incoming_payload = req->req.payload = + i_stream_create_from_data("", 0); + } + i_stream_add_destroy_callback(req->req.payload, + http_server_payload_destroyed, req); + /* The callback may add its own I/O, so we need to remove our one before + calling it. */ + http_server_connection_input_halt(conn); + + old_refcount = req->refcount; + conn->in_req_callback = TRUE; + T_BEGIN { + http_server_request_callback(req); + } T_END; + if (conn->closed) { + /* The callback managed to get this connection destroyed/closed + */ + return FALSE; + } + conn->in_req_callback = FALSE; + req->callback_refcount = req->refcount - old_refcount; + + if (req->req.payload != NULL) { + /* Send 100 Continue when appropriate */ + if (req->req.expect_100_continue && !req->payload_halted && + req->response == NULL) { + http_server_connection_output_trigger(conn); + } + + /* Delegate payload handling to request handler */ + if (req->state < HTTP_SERVER_REQUEST_STATE_PAYLOAD_IN) + req->state = HTTP_SERVER_REQUEST_STATE_PAYLOAD_IN; + payload = req->req.payload; + req->req.payload = NULL; + i_stream_unref(&payload); + } + + if (req->state < HTTP_SERVER_REQUEST_STATE_PROCESSING && + (conn->incoming_payload == NULL || + !i_stream_have_bytes_left(conn->incoming_payload))) { + /* Finished reading request */ + req->state = HTTP_SERVER_REQUEST_STATE_PROCESSING; + if (req->response != NULL && req->response->submitted) + http_server_request_submit_response(req); + } + + i_assert(conn->incoming_payload != NULL || req->callback_refcount > 0 || + (req->response != NULL && req->response->submitted)); + + if (conn->incoming_payload == NULL) { + http_server_connection_input_resume(conn); + http_server_connection_input_set_pending(conn); + return TRUE; + } + + /* Request payload is still being uploaded by the client */ + return FALSE; +} + +static int +http_server_connection_ssl_init(struct http_server_connection *conn) +{ + struct http_server *server = conn->server; + const char *error; + int ret; + + if (http_server_init_ssl_ctx(server, &error) < 0) { + e_error(conn->event, "Couldn't initialize SSL: %s", error); + return -1; + } + + e_debug(conn->event, "Starting SSL handshake"); + + http_server_connection_input_halt(conn); + if (server->ssl_ctx == NULL) { + ret = master_service_ssl_init(master_service, + &conn->conn.input, + &conn->conn.output, + &conn->ssl_iostream, &error); + } else { + ret = io_stream_create_ssl_server(server->ssl_ctx, + server->set.ssl, + &conn->conn.input, + &conn->conn.output, + &conn->ssl_iostream, &error); + } + if (ret < 0) { + e_error(conn->event, + "Couldn't initialize SSL server for %s: %s", + conn->conn.name, error); + return -1; + } + http_server_connection_input_resume(conn); + + if (ssl_iostream_handshake(conn->ssl_iostream) < 0) { + e_error(conn->event, "SSL handshake failed: %s", + ssl_iostream_get_last_error(conn->ssl_iostream)); + return -1; + } + + http_server_connection_ready(conn); + return 0; +} + +static bool +http_server_connection_pipeline_is_full(struct http_server_connection *conn) +{ + return ((conn->request_queue_count >= + conn->server->set.max_pipelined_requests) || + conn->server->shutting_down); +} + +static void +http_server_connection_pipeline_handle_full(struct http_server_connection *conn) +{ + if (conn->server->shutting_down) { + e_debug(conn->event, "Pipeline full " + "(%u requests pending; server shutting down)", + conn->request_queue_count); + } else { + e_debug(conn->event, "Pipeline full " + "(%u requests pending; %u maximum)", + conn->request_queue_count, + conn->server->set.max_pipelined_requests); + } + http_server_connection_input_halt(conn); +} + +static bool +http_server_connection_check_input(struct http_server_connection *conn) +{ + struct istream *input = conn->conn.input; + int stream_errno; + + if (input == NULL) + return FALSE; + stream_errno = input->stream_errno; + + if (input->eof || stream_errno != 0) { + /* Connection input broken; output may still be intact */ + if (stream_errno != 0 && stream_errno != EPIPE && + stream_errno != ECONNRESET) { + http_server_connection_client_error( + conn, "Connection lost: read(%s) failed: %s", + i_stream_get_name(input), + i_stream_get_error(input)); + http_server_connection_close(&conn, "Read failure"); + } else { + e_debug(conn->event, "Connection lost: " + "Remote disconnected"); + + if (conn->request_queue_head == NULL) { + /* No pending requests; close */ + http_server_connection_close( + &conn, "Remote closed connection"); + } else if (conn->request_queue_head->state < + HTTP_SERVER_REQUEST_STATE_SUBMITTED_RESPONSE) { + /* Unfinished request; close */ + http_server_connection_close(&conn, + "Remote closed connection unexpectedly"); + } else { + /* A request is still processing; only drop + input io for now. The other end may only have + shutdown one direction */ + conn->input_broken = TRUE; + http_server_connection_input_halt(conn); + } + } + return FALSE; + } + return TRUE; +} + +static bool +http_server_connection_finish_request(struct http_server_connection *conn) +{ + struct http_server_request *req; + enum http_request_parse_error error_code; + const char *error; + int ret; + + req = conn->request_queue_tail; + if (req != NULL && + req->state == HTTP_SERVER_REQUEST_STATE_SUBMITTED_RESPONSE) { + + e_debug(conn->event, "Finish receiving request"); + + ret = http_request_parse_finish_payload(conn->http_parser, + &error_code, &error); + if (ret <= 0 && !http_server_connection_check_input(conn)) + return FALSE; + if (ret < 0) { + http_server_connection_ref(conn); + + http_server_connection_client_error( + conn, "Client sent invalid request: %s", error); + + switch (error_code) { + case HTTP_REQUEST_PARSE_ERROR_PAYLOAD_TOO_LARGE: + conn->input_broken = TRUE; + http_server_request_fail_close( + req, 413, "Payload Too Large"); + break; + case HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST: + conn->input_broken = TRUE; + http_server_request_fail_close( + req, 400, "Bad request"); + break; + default: + i_unreached(); + } + + if (http_server_connection_unref_is_closed(conn)) { + /* Connection got closed */ + return FALSE; + } + + if (conn->input_broken || conn->close_indicated) + http_server_connection_input_halt(conn); + return FALSE; + } + if (ret == 0) + return FALSE; + http_server_request_ready_to_respond(req); + } + + return TRUE; +} + +static void http_server_connection_input(struct connection *_conn) +{ + struct http_server_connection *conn = + (struct http_server_connection *)_conn; + struct http_server_request *req; + enum http_request_parse_error error_code; + const char *error; + bool cont; + int ret; + + if (conn->server->shutting_down) { + if (!http_server_connection_shut_down(conn)) + http_server_connection_pipeline_handle_full(conn); + return; + } + + i_assert(!conn->input_broken && conn->incoming_payload == NULL); + i_assert(!conn->close_indicated); + + http_server_connection_reset_idle_timeout(conn); + + if (conn->ssl && conn->ssl_iostream == NULL) { + if (http_server_connection_ssl_init(conn) < 0) { + /* SSL failed */ + http_server_connection_close( + &conn, "SSL Initialization failed"); + return; + } + } + + /* Finish up pending request */ + if (!http_server_connection_finish_request(conn)) + return; + + /* Stop handling input here when running ioloop from within request + callback; we cannot read the next request, since that could mean + recursing request callbacks. */ + if (conn->in_req_callback) { + http_server_connection_input_halt(conn); + return; + } + + /* Create request object if none was created already */ + if (conn->request_queue_tail != NULL && + conn->request_queue_tail->state == HTTP_SERVER_REQUEST_STATE_NEW) { + if (conn->request_queue_count > + conn->server->set.max_pipelined_requests) { + /* Pipeline full */ + http_server_connection_pipeline_handle_full(conn); + return; + } + /* Continue last unfinished request */ + req = conn->request_queue_tail; + } else { + if (conn->request_queue_count >= + conn->server->set.max_pipelined_requests) { + /* Pipeline full */ + http_server_connection_pipeline_handle_full(conn); + return; + } + /* Start new request */ + req = http_server_request_new(conn); + } + + /* Parse requests */ + ret = 1; + while (!conn->close_indicated && ret != 0) { + http_server_connection_ref(conn); + while ((ret = http_request_parse_next( + conn->http_parser, req->pool, &req->req, + &error_code, &error)) > 0) { + conn->stats.request_count++; + http_server_request_received(req); + + http_server_request_immune_ref(req); + T_BEGIN { + cont = http_server_connection_handle_request(conn, req); + } T_END; + if (!cont) { + /* Connection closed or request body not read + yet. The request may be destroyed now. */ + http_server_request_immune_unref(&req); + http_server_connection_unref(&conn); + return; + } + if (req->req.connection_close) + conn->close_indicated = TRUE; + http_server_request_immune_unref(&req); + + if (conn->closed) { + /* Connection got closed in destroy callback */ + break; + } + + if (conn->close_indicated) { + /* Client indicated it will close after this + request; stop trying to read more. */ + break; + } + + /* Finish up pending request if possible */ + if (!http_server_connection_finish_request(conn)) { + http_server_connection_unref(&conn); + return; + } + + if (http_server_connection_pipeline_is_full(conn)) { + /* Pipeline full */ + http_server_connection_pipeline_handle_full(conn); + http_server_connection_unref(&conn); + return; + } + + /* Start new request */ + req = http_server_request_new(conn); + } + + if (http_server_connection_unref_is_closed(conn)) { + /* Connection got closed */ + return; + } + + if (ret <= 0 && !http_server_connection_check_input(conn)) + return; + + if (ret < 0) { + http_server_connection_ref(conn); + + http_server_connection_client_error( + conn, "Client sent invalid request: %s", error); + + switch (error_code) { + case HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST: + conn->input_broken = TRUE; + /* fall through */ + case HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST: + http_server_request_fail( + req, 400, "Bad Request"); + break; + case HTTP_REQUEST_PARSE_ERROR_METHOD_TOO_LONG: + conn->input_broken = TRUE; + /* fall through */ + case HTTP_REQUEST_PARSE_ERROR_NOT_IMPLEMENTED: + http_server_request_fail( + req, 501, "Not Implemented"); + break; + case HTTP_REQUEST_PARSE_ERROR_TARGET_TOO_LONG: + conn->input_broken = TRUE; + http_server_request_fail_close( + req, 414, "URI Too Long"); + break; + case HTTP_REQUEST_PARSE_ERROR_EXPECTATION_FAILED: + http_server_request_fail( + req, 417, "Expectation Failed"); + break; + case HTTP_REQUEST_PARSE_ERROR_PAYLOAD_TOO_LARGE: + conn->input_broken = TRUE; + http_server_request_fail_close( + req, 413, "Payload Too Large"); + break; + default: + i_unreached(); + } + + if (http_server_connection_unref_is_closed(conn)) { + /* Connection got closed */ + return; + } + } + + if (conn->input_broken || conn->close_indicated) { + http_server_connection_input_halt(conn); + return; + } + } +} + +void http_server_connection_handle_output_error( + struct http_server_connection *conn) +{ + struct ostream *output = conn->conn.output; + + if (conn->closed) + return; + + if (output->stream_errno != EPIPE && + output->stream_errno != ECONNRESET) { + e_error(conn->event, "Connection lost: write(%s) failed: %s", + o_stream_get_name(output), + o_stream_get_error(output)); + http_server_connection_close( + &conn, "Write failure"); + } else { + e_debug(conn->event, "Connection lost: Remote disconnected"); + http_server_connection_close( + &conn, "Remote closed connection unexpectedly"); + } +} + +enum _output_result { + /* Error */ + _OUTPUT_ERROR = -1, + /* Output blocked */ + _OUTPUT_BLOCKED = 0, + /* Successful, but no more responses are ready to be sent */ + _OUTPUT_FINISHED = 1, + /* Successful and more responses can be sent */ + _OUTPUT_AVAILABLE = 2, +}; + +static enum _output_result +http_server_connection_next_response(struct http_server_connection *conn) +{ + struct http_server_request *req; + int ret; + + if (conn->output_locked) + return _OUTPUT_FINISHED; + + req = conn->request_queue_head; + if (req == NULL || req->state == HTTP_SERVER_REQUEST_STATE_NEW) { + /* No requests pending */ + e_debug(conn->event, "No more requests pending"); + http_server_connection_start_idle_timeout(conn); + return _OUTPUT_FINISHED; + } + if (req->state < HTTP_SERVER_REQUEST_STATE_READY_TO_RESPOND) { + if (req->state == HTTP_SERVER_REQUEST_STATE_PROCESSING) { + /* Server is causing idle time */ + e_debug(conn->event, "Not ready to respond: " + "Server is processing"); + http_server_connection_stop_idle_timeout(conn); + } else { + /* Client is causing idle time */ + e_debug(conn->event, "Not ready to respond: " + "Waiting for client"); + http_server_connection_start_idle_timeout(conn); + } + + /* send 100 Continue if appropriate */ + if (req->state >= HTTP_SERVER_REQUEST_STATE_QUEUED && + conn->incoming_payload != NULL && + req->response == NULL && req->req.version_minor >= 1 && + req->req.expect_100_continue && !req->payload_halted && + !req->sent_100_continue) { + static const char *response = + "HTTP/1.1 100 Continue\r\n\r\n"; + struct ostream *output = conn->conn.output; + + if (o_stream_send(output, response, + strlen(response)) < 0) { + http_server_connection_handle_output_error(conn); + return _OUTPUT_ERROR; + } + + e_debug(conn->event, "Sent 100 Continue"); + req->sent_100_continue = TRUE; + } + return _OUTPUT_FINISHED; + } + + i_assert(req->state == HTTP_SERVER_REQUEST_STATE_READY_TO_RESPOND && + req->response != NULL); + + e_debug(conn->event, "Sending response"); + http_server_connection_start_idle_timeout(conn); + + http_server_request_immune_ref(req); + ret = http_server_response_send(req->response); + http_server_request_immune_unref(&req); + + if (ret < 0) + return _OUTPUT_ERROR; + + http_server_connection_reset_idle_timeout(conn); + if (ret == 0) + return _OUTPUT_BLOCKED; + if (conn->output_locked) + return _OUTPUT_FINISHED; + return _OUTPUT_AVAILABLE; +} + +static int +http_server_connection_send_responses(struct http_server_connection *conn) +{ + enum _output_result ores = _OUTPUT_AVAILABLE; + + http_server_connection_ref(conn); + + /* Send more responses until no more responses remain, the output + blocks again, or the connection is closed */ + while (!conn->closed && ores == _OUTPUT_AVAILABLE) + ores = http_server_connection_next_response(conn); + + if (http_server_connection_unref_is_closed(conn) || + ores == _OUTPUT_ERROR) + return -1; + + /* Accept more requests if possible */ + if (conn->incoming_payload == NULL && + (conn->request_queue_count < + conn->server->set.max_pipelined_requests) && + !conn->server->shutting_down) + http_server_connection_input_resume(conn); + + switch (ores) { + case _OUTPUT_ERROR: + case _OUTPUT_AVAILABLE: + break; + case _OUTPUT_BLOCKED: + return 0; + case _OUTPUT_FINISHED: + return 1; + } + i_unreached(); +} + +int http_server_connection_flush(struct http_server_connection *conn) +{ + struct ostream *output = conn->conn.output; + int ret; + + if ((ret = o_stream_flush(output)) <= 0) { + if (ret < 0) + http_server_connection_handle_output_error(conn); + return ret; + } + + http_server_connection_reset_idle_timeout(conn); + return 0; +} + +int http_server_connection_output(struct http_server_connection *conn) +{ + bool pipeline_was_full = + http_server_connection_pipeline_is_full(conn); + int ret = 1; + + if (http_server_connection_flush(conn) < 0) + return -1; + + if (!conn->output_locked) { + ret = http_server_connection_send_responses(conn); + if (ret < 0) + return -1; + } else if (conn->request_queue_head != NULL) { + struct http_server_request *req = conn->request_queue_head; + struct http_server_response *resp = req->response; + + i_assert(resp != NULL); + + http_server_connection_ref(conn); + + http_server_request_immune_ref(req); + ret = http_server_response_send_more(resp); + http_server_request_immune_unref(&req); + + if (http_server_connection_unref_is_closed(conn) || ret < 0) + return -1; + + if (!conn->output_locked) { + /* Room for more responses */ + ret = http_server_connection_send_responses(conn); + if (ret < 0) + return -1; + } else if (conn->io_resp_payload != NULL) { + /* Server is causing idle time */ + e_debug(conn->event, "Not ready to continue response: " + "Server is producing response"); + http_server_connection_stop_idle_timeout(conn); + } else { + /* Client is causing idle time */ + e_debug(conn->event, "Not ready to continue response: " + "Waiting for client"); + http_server_connection_start_idle_timeout(conn); + } + } + + if (conn->server->shutting_down && + http_server_connection_shut_down(conn)) + return 1; + + if (!http_server_connection_pipeline_is_full(conn)) { + http_server_connection_input_resume(conn); + if (pipeline_was_full && conn->conn.io != NULL) + http_server_connection_input_set_pending(conn); + } + + return ret; +} + +void http_server_connection_output_trigger(struct http_server_connection *conn) +{ + if (conn->conn.output == NULL) + return; + o_stream_set_flush_pending(conn->conn.output, TRUE); +} + +void http_server_connection_output_halt(struct http_server_connection *conn) +{ + conn->output_halted = TRUE; + + if (conn->conn.output == NULL) + return; + + o_stream_unset_flush_callback(conn->conn.output); +} + +void http_server_connection_output_resume(struct http_server_connection *conn) +{ + if (conn->output_halted) { + conn->output_halted = FALSE; + o_stream_set_flush_callback(conn->conn.output, + http_server_connection_output, conn); + } +} + +bool http_server_connection_pending_payload( + struct http_server_connection *conn) +{ + return http_request_parser_pending_payload(conn->http_parser); +} + +static struct connection_settings http_server_connection_set = { + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .client = FALSE, + .log_connection_id = TRUE, +}; + +static const struct connection_vfuncs http_server_connection_vfuncs = { + .destroy = http_server_connection_destroy, + .input = http_server_connection_input +}; + +struct connection_list *http_server_connection_list_init(void) +{ + return connection_list_init(&http_server_connection_set, + &http_server_connection_vfuncs); +} + +struct http_server_connection * +http_server_connection_create(struct http_server *server, + int fd_in, int fd_out, bool ssl, + const struct http_server_callbacks *callbacks, + void *context) +{ + const struct http_server_settings *set = &server->set; + struct http_server_connection *conn; + struct event *conn_event; + + i_assert(!server->shutting_down); + + conn = i_new(struct http_server_connection, 1); + conn->refcount = 1; + conn->server = server; + conn->ioloop = current_ioloop; + conn->ssl = ssl; + conn->callbacks = callbacks; + conn->context = context; + + net_set_nonblock(fd_in, TRUE); + if (fd_in != fd_out) + net_set_nonblock(fd_out, TRUE); + (void)net_set_tcp_nodelay(fd_out, TRUE); + + if (set->socket_send_buffer_size > 0 && + net_set_send_buffer_size(fd_out, + set->socket_send_buffer_size) < 0) { + e_error(conn->event, + "net_set_send_buffer_size(%zu) failed: %m", + set->socket_send_buffer_size); + } + if (set->socket_recv_buffer_size > 0 && + net_set_recv_buffer_size(fd_in, + set->socket_recv_buffer_size) < 0) { + e_error(conn->event, + "net_set_recv_buffer_size(%zu) failed: %m", + set->socket_recv_buffer_size); + } + + conn_event = event_create(server->event); + conn->conn.event_parent = conn_event; + connection_init_server(server->conn_list, &conn->conn, NULL, + fd_in, fd_out); + conn->event = conn->conn.event; + event_unref(&conn_event); + + if (!ssl) + http_server_connection_ready(conn); + http_server_connection_start_idle_timeout(conn); + + e_debug(conn->event, "Connection created"); + return conn; +} + +void http_server_connection_ref(struct http_server_connection *conn) +{ + i_assert(conn->refcount > 0); + conn->refcount++; +} + +static void +http_server_connection_disconnect(struct http_server_connection *conn, + const char *reason) +{ + struct http_server_request *req, *req_next; + + if (conn->closed) + return; + + if (reason == NULL) + reason = "Connection closed"; + e_debug(conn->event, "Disconnected: %s", reason); + conn->disconnect_reason = i_strdup(reason); + conn->closed = TRUE; + + /* Preserve statistics */ + http_server_connection_update_stats(conn); + + if (conn->incoming_payload != NULL) { + /* The stream is still accessed by lib-http caller. */ + i_stream_remove_destroy_callback(conn->incoming_payload, + http_server_payload_destroyed); + conn->incoming_payload = NULL; + } + if (conn->payload_handler != NULL) + http_server_payload_handler_destroy(&conn->payload_handler); + + /* Drop all requests before connection is closed */ + req = conn->request_queue_head; + while (req != NULL) { + req_next = req->next; + http_server_request_abort(&req, reason); + req = req_next; + } + + timeout_remove(&conn->to_input); + timeout_remove(&conn->to_idle); + io_remove(&conn->io_resp_payload); + if (conn->conn.output != NULL) + o_stream_uncork(conn->conn.output); + + if (conn->http_parser != NULL) + http_request_parser_deinit(&conn->http_parser); + connection_disconnect(&conn->conn); +} + +bool http_server_connection_unref(struct http_server_connection **_conn) +{ + struct http_server_connection *conn = *_conn; + + i_assert(conn->refcount > 0); + + *_conn = NULL; + if (--conn->refcount > 0) + return TRUE; + + http_server_connection_disconnect(conn, NULL); + + e_debug(conn->event, "Connection destroy"); + + ssl_iostream_destroy(&conn->ssl_iostream); + connection_deinit(&conn->conn); + + if (conn->callbacks != NULL && + conn->callbacks->connection_destroy != NULL) T_BEGIN { + conn->callbacks->connection_destroy(conn->context, + conn->disconnect_reason); + } T_END; + + i_free(conn->disconnect_reason); + i_free(conn); + return FALSE; +} + +static bool +http_server_connection_unref_is_closed(struct http_server_connection *conn) +{ + bool closed = conn->closed; + + if (!http_server_connection_unref(&conn)) + closed = TRUE; + return closed; +} + +void http_server_connection_close(struct http_server_connection **_conn, + const char *reason) +{ + struct http_server_connection *conn = *_conn; + + http_server_connection_disconnect(conn, reason); + http_server_connection_unref(_conn); +} + +void http_server_connection_tunnel(struct http_server_connection **_conn, + http_server_tunnel_callback_t callback, + void *context) +{ + struct http_server_connection *conn = *_conn; + struct http_server_tunnel tunnel; + + /* Preserve statistics */ + http_server_connection_update_stats(conn); + + i_zero(&tunnel); + tunnel.input = conn->conn.input; + tunnel.output = conn->conn.output; + tunnel.fd_in = conn->conn.fd_in; + tunnel.fd_out = conn->conn.fd_out; + + conn->conn.input = NULL; + conn->conn.output = NULL; + conn->conn.fd_in = conn->conn.fd_out = -1; + http_server_connection_close(_conn, "Tunnel initiated"); + + callback(context, &tunnel); +} + +struct ioloop * +http_server_connection_switch_ioloop_to(struct http_server_connection *conn, + struct ioloop *ioloop) +{ + struct ioloop *prev_ioloop = conn->ioloop; + + if (conn->ioloop_switching != NULL) + return conn->ioloop_switching; + + conn->ioloop = ioloop; + conn->ioloop_switching = prev_ioloop; + connection_switch_ioloop_to(&conn->conn, ioloop); + if (conn->to_input != NULL) { + conn->to_input = + io_loop_move_timeout_to(ioloop, &conn->to_input); + } + if (conn->to_idle != NULL) { + conn->to_idle = + io_loop_move_timeout_to(ioloop, &conn->to_idle); + } + if (conn->io_resp_payload != NULL) { + conn->io_resp_payload = + io_loop_move_io_to(ioloop, &conn->io_resp_payload); + } + if (conn->payload_handler != NULL) { + http_server_payload_handler_switch_ioloop( + conn->payload_handler, ioloop); + } + if (conn->incoming_payload != NULL) + i_stream_switch_ioloop_to(conn->incoming_payload, ioloop); + conn->ioloop_switching = NULL; + + return prev_ioloop; +} + +struct ioloop * +http_server_connection_switch_ioloop(struct http_server_connection *conn) +{ + return http_server_connection_switch_ioloop_to(conn, current_ioloop); +} diff --git a/src/lib-http/http-server-ostream.c b/src/lib-http/http-server-ostream.c new file mode 100644 index 0000000..566aa70 --- /dev/null +++ b/src/lib-http/http-server-ostream.c @@ -0,0 +1,328 @@ +/* Copyright (c) 2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "dns-lookup.h" +#include "ostream-wrapper.h" + +#include "http-server-private.h" + +/* + * Payload output stream + */ + +struct http_server_ostream { + struct wrapper_ostream wostream; + + struct http_server_connection *conn; + struct http_server_response *resp; + + bool response_destroyed:1; +}; + +static void http_server_ostream_output_error(struct wrapper_ostream *wostream) +{ + struct http_server_ostream *hsostream = + (struct http_server_ostream *)wostream; + struct http_server_connection *conn = hsostream->conn; + + if (hsostream->response_destroyed) + return; + + i_assert(hsostream->resp != NULL); + http_server_connection_handle_output_error(conn); +} + +static void http_server_ostream_output_start(struct wrapper_ostream *wostream) +{ + struct http_server_ostream *hsostream = + (struct http_server_ostream *)wostream; + struct http_server_response *resp = hsostream->resp; + + i_assert(hsostream->response_destroyed || resp != NULL); + + if (!hsostream->response_destroyed && + resp->request->state <= HTTP_SERVER_REQUEST_STATE_PROCESSING) { + /* implicitly submit the request */ + http_server_response_submit(resp); + } +} + +void http_server_ostream_output_available( + struct http_server_ostream *hsostream) +{ + struct http_server_response *resp = hsostream->resp; + + i_assert(resp != NULL); + i_assert(!hsostream->response_destroyed); + wrapper_ostream_output_available(&hsostream->wostream, + resp->payload_output); +} + +static bool http_server_ostream_output_ready(struct wrapper_ostream *wostream) +{ + struct http_server_ostream *hsostream = + (struct http_server_ostream *)wostream; + struct http_server_response *resp = hsostream->resp; + + i_assert(resp != NULL); + i_assert(!hsostream->response_destroyed); + return (resp->request->state >= HTTP_SERVER_REQUEST_STATE_PAYLOAD_OUT); +} + +static int http_server_ostream_output_finish(struct wrapper_ostream *wostream) +{ + struct http_server_ostream *hsostream = + (struct http_server_ostream *)wostream; + struct http_server_response *resp = hsostream->resp; + + i_assert(resp != NULL); + i_assert(!hsostream->response_destroyed); + + e_debug(wostream->event, "Finished response payload stream"); + + /* finished sending payload */ + return http_server_response_finish_payload_out(resp); +} + +static void http_server_ostream_output_halt(struct wrapper_ostream *wostream) +{ + struct http_server_ostream *hsostream = + (struct http_server_ostream *)wostream; + struct http_server_connection *conn = hsostream->conn; + struct http_server_response *resp = hsostream->resp; + + i_assert(hsostream->response_destroyed || resp != NULL); + + if (hsostream->response_destroyed || + resp->request->state < HTTP_SERVER_REQUEST_STATE_PAYLOAD_OUT) + return; + + http_server_connection_output_halt(conn); +} + +static void http_server_ostream_output_resume(struct wrapper_ostream *wostream) +{ + struct http_server_ostream *hsostream = + (struct http_server_ostream *)wostream; + struct http_server_connection *conn = hsostream->conn; + + if (hsostream->response_destroyed) + return; + i_assert(hsostream->resp != NULL); + + http_server_connection_output_resume(conn); +} + +static void +http_server_ostream_output_update_timeouts(struct wrapper_ostream *wostream, + bool sender_blocking) +{ + struct http_server_ostream *hsostream = + (struct http_server_ostream *)wostream; + struct http_server_connection *conn = hsostream->conn; + + if (hsostream->response_destroyed) + return; + i_assert(hsostream->resp != NULL); + + if (sender_blocking) { + http_server_connection_stop_idle_timeout(conn); + return; + } + + http_server_connection_start_idle_timeout(conn); +} + +static void http_server_ostream_close(struct wrapper_ostream *wostream) +{ + struct http_server_ostream *hsostream = + (struct http_server_ostream *)wostream; + struct http_server_response *resp = hsostream->resp; + + e_debug(wostream->event, "Response payload stream closed"); + + if (hsostream->response_destroyed) { + http_server_response_unref(&hsostream->resp); + return; + } + hsostream->response_destroyed = TRUE; + + i_assert(resp != NULL); + (void)http_server_response_finish_payload_out(resp); + resp->payload_stream = NULL; + http_server_response_unref(&hsostream->resp); +} + +static void http_server_ostream_destroy(struct wrapper_ostream *wostream) +{ + struct http_server_ostream *hsostream = + (struct http_server_ostream *)wostream; + struct http_server_response *resp = hsostream->resp; + struct http_server_request *req; + + e_debug(wostream->event, "Response payload stream destroyed"); + + if (hsostream->response_destroyed) { + http_server_response_unref(&hsostream->resp); + return; + } + hsostream->response_destroyed = TRUE; + i_assert(resp != NULL); + + req = resp->request; + resp->payload_stream = NULL; + http_server_request_abort( + &req, "Response output stream destroyed prematurely"); +} + +static struct ioloop * +http_server_ostream_wait_begin(struct wrapper_ostream *wostream, + struct ioloop *ioloop) +{ + struct http_server_ostream *hsostream = + (struct http_server_ostream *)wostream; + struct http_server_connection *conn = hsostream->conn; + struct ioloop *prev_ioloop; + + i_assert(hsostream->resp != NULL); + i_assert(!hsostream->response_destroyed); + + http_server_connection_ref(conn); + + /* When the response payload output stream is written from inside the + request callback, the incoming payload stream is not destroyed yet, + even though it is read to the end. This could lead to problems, so we + make an effort to destroy it here. + */ + if (conn->incoming_payload != NULL && + i_stream_read_eof(conn->incoming_payload)) { + struct http_server_request *req = hsostream->resp->request; + struct istream *payload; + + payload = req->req.payload; + req->req.payload = NULL; + i_stream_unref(&payload); + } + + prev_ioloop = http_server_connection_switch_ioloop_to(conn, ioloop); + return prev_ioloop; +} + +static void +http_server_ostream_wait_end(struct wrapper_ostream *wostream, + struct ioloop *prev_ioloop) +{ + struct http_server_ostream *hsostream = + (struct http_server_ostream *)wostream; + struct http_server_connection *conn = hsostream->conn; + + (void)http_server_connection_switch_ioloop_to(conn, prev_ioloop); + http_server_connection_unref(&conn); +} + +int http_server_ostream_continue(struct http_server_ostream *hsostream) +{ + struct wrapper_ostream *wostream = &hsostream->wostream; + struct http_server_response *resp = hsostream->resp; + + i_assert(hsostream->response_destroyed || resp != NULL); + + i_assert(hsostream->response_destroyed || + resp->request->state >= HTTP_SERVER_REQUEST_STATE_PAYLOAD_OUT); + + return wrapper_ostream_continue(wostream); +} + +bool http_server_ostream_get_size(struct http_server_ostream *hsostream, + uoff_t *size_r) +{ + return wrapper_ostream_get_buffered_size(&hsostream->wostream, size_r); +} + +static void +http_server_ostream_switch_ioloop_to(struct wrapper_ostream *wostream, + struct ioloop *ioloop) +{ + struct http_server_ostream *hsostream = + (struct http_server_ostream *)wostream; + struct http_server_connection *conn = hsostream->conn; + + if (hsostream->response_destroyed) + return; + i_assert(hsostream->resp != NULL); + + http_server_connection_switch_ioloop_to(conn, ioloop); +} + +struct ostream * +http_server_ostream_create(struct http_server_response *resp, + size_t max_buffer_size, bool blocking) +{ + struct http_server_ostream *hsostream; + + i_assert(resp->payload_stream == NULL); + + hsostream = i_new(struct http_server_ostream, 1); + + resp->payload_stream = hsostream; + http_server_response_ref(resp); + hsostream->conn = resp->request->conn; + hsostream->resp = resp; + + hsostream->wostream.output_start = http_server_ostream_output_start; + hsostream->wostream.output_ready = http_server_ostream_output_ready; + hsostream->wostream.output_error = http_server_ostream_output_error; + hsostream->wostream.output_finish = http_server_ostream_output_finish; + hsostream->wostream.output_halt = http_server_ostream_output_halt; + hsostream->wostream.output_resume = http_server_ostream_output_resume; + hsostream->wostream.output_update_timeouts = + http_server_ostream_output_update_timeouts; + + hsostream->wostream.wait_begin = http_server_ostream_wait_begin; + hsostream->wostream.wait_end = http_server_ostream_wait_end; + + hsostream->wostream.switch_ioloop_to = + http_server_ostream_switch_ioloop_to; + + hsostream->wostream.close = http_server_ostream_close; + hsostream->wostream.destroy = http_server_ostream_destroy; + + return wrapper_ostream_create(&hsostream->wostream, max_buffer_size, + blocking, resp->event); +} + +void http_server_ostream_response_finished( + struct http_server_ostream *hsostream) +{ + e_debug(hsostream->wostream.event, "Response payload finished"); + + wrapper_ostream_output_destroyed(&hsostream->wostream); +} + +void http_server_ostream_response_destroyed( + struct http_server_ostream *hsostream) +{ + i_assert(hsostream->resp != NULL); + hsostream->resp->payload_stream = NULL; + + e_debug(hsostream->wostream.event, + "Response payload parent stream lost"); + + hsostream->response_destroyed = TRUE; + wrapper_ostream_output_destroyed(&hsostream->wostream); + wrapper_ostream_notify_error(&hsostream->wostream); +} + +struct ostream * +http_server_ostream_get_output(struct http_server_ostream *hsostream) +{ + return &hsostream->wostream.ostream.ostream; +} + +void http_server_ostream_set_error(struct http_server_ostream *hsostream, + int stream_errno, const char *stream_error) +{ + wrapper_ostream_set_error(&hsostream->wostream, stream_errno, + stream_error); +} diff --git a/src/lib-http/http-server-private.h b/src/lib-http/http-server-private.h new file mode 100644 index 0000000..c07d873 --- /dev/null +++ b/src/lib-http/http-server-private.h @@ -0,0 +1,357 @@ +#ifndef HTTP_SERVER_PRIVATE_H +#define HTTP_SERVER_PRIVATE_H + +#include "connection.h" + +#include "iostream-pump.h" +#include "http-server.h" +#include "llist.h" + +struct http_server_ostream; +struct http_server_payload_handler; +struct http_server_request; +struct http_server_connection; + +/* + * Defaults + */ + +#define HTTP_SERVER_REQUEST_MAX_TARGET_LENGTH 4096 + +/* + * Types + */ + +enum http_server_request_state { + /* New request; request header is still being parsed. */ + HTTP_SERVER_REQUEST_STATE_NEW = 0, + /* Queued request; callback to request handler executing. */ + HTTP_SERVER_REQUEST_STATE_QUEUED, + /* Reading request payload; request handler still needs to read more + payload. */ + HTTP_SERVER_REQUEST_STATE_PAYLOAD_IN, + /* This request is being processed; request payload is fully read, but + no response is yet submitted */ + HTTP_SERVER_REQUEST_STATE_PROCESSING, + /* A response is submitted for this request. If not all request payload + was read by the handler, it is first skipped on the input. */ + HTTP_SERVER_REQUEST_STATE_SUBMITTED_RESPONSE, + /* Request is ready for response; a response is submitted and the + request payload is fully read */ + HTTP_SERVER_REQUEST_STATE_READY_TO_RESPOND, + /* The response for the request is sent (apart from payload) */ + HTTP_SERVER_REQUEST_STATE_SENT_RESPONSE, + /* Sending response payload to client */ + HTTP_SERVER_REQUEST_STATE_PAYLOAD_OUT, + /* Request is finished; still lingering due to references */ + HTTP_SERVER_REQUEST_STATE_FINISHED, + /* Request is aborted; still lingering due to references */ + HTTP_SERVER_REQUEST_STATE_ABORTED +}; + +/* + * Objects + */ + +struct http_server_payload_handler { + struct http_server_request *req; + + void (*switch_ioloop)(struct http_server_payload_handler *handler, + struct ioloop *ioloop); + void (*destroy)(struct http_server_payload_handler *handler); + + bool in_callback:1; +}; + +struct http_server_response { + struct http_server_request *request; + struct event *event; + + unsigned int status; + const char *reason; + + string_t *headers; + ARRAY_TYPE(string) perm_headers; + time_t date; + ARRAY_TYPE(http_auth_challenge) auth_challenges; + + struct istream *payload_input; + uoff_t payload_size, payload_offset; + struct ostream *payload_output; + struct http_server_ostream *payload_stream; + + http_server_tunnel_callback_t tunnel_callback; + void *tunnel_context; + + bool have_hdr_connection:1; + bool have_hdr_date:1; + bool have_hdr_body_spec:1; + + bool payload_chunked:1; + bool payload_finished:1; + bool payload_corked:1; + bool submitted:1; +}; + +struct http_server_request { + struct http_request req; + pool_t pool; + unsigned int refcount, immune_refcount; + unsigned int id; + int callback_refcount; + struct event *event; + uoff_t input_start_offset, output_start_offset; + + enum http_server_request_state state; + + struct http_server_request *prev, *next; + + struct http_server *server; + struct http_server_connection *conn; + + struct istream *payload_input; + + struct http_server_response *response; + + void (*destroy_callback)(void *); + void *destroy_context; + + bool payload_halted:1; + bool sent_100_continue:1; + bool destroy_pending:1; + bool failed:1; + bool connection_close:1; +}; + +struct http_server_connection { + struct connection conn; + struct http_server *server; + struct ioloop *ioloop, *ioloop_switching; + struct event *event; + unsigned int refcount; + + const struct http_server_callbacks *callbacks; + void *context; + + struct timeout *to_input, *to_idle; + struct ssl_iostream *ssl_iostream; + struct http_request_parser *http_parser; + + struct http_server_request *request_queue_head, *request_queue_tail; + unsigned int request_queue_count; + + struct istream *incoming_payload; + struct http_server_payload_handler *payload_handler; + + struct io *io_resp_payload; + + char *disconnect_reason; + + struct http_server_stats stats; + + bool ssl:1; + bool closed:1; + bool close_indicated:1; + bool input_broken:1; + bool output_locked:1; + bool output_halted:1; + bool in_req_callback:1; /* performing request callback (busy) */ +}; + +struct http_server_location { + const char *path; + + struct http_server_resource *resource; +}; + +struct http_server_resource { + pool_t pool; + struct http_server *server; + struct event *event; + + http_server_resource_callback_t *callback; + void *context; + + void (*destroy_callback)(void *); + void *destroy_context; + + ARRAY(struct http_server_location *) locations; +}; + +struct http_server { + pool_t pool; + + struct http_server_settings set; + + struct ioloop *ioloop; + struct event *event; + struct ssl_iostream_context *ssl_ctx; + + struct connection_list *conn_list; + + ARRAY(struct http_server_resource *) resources; + ARRAY(struct http_server_location *) locations; + + bool shutting_down:1; /* shutting down server */ +}; + +/* + * Response output stream + */ + +struct ostream * +http_server_ostream_create(struct http_server_response *resp, + size_t max_buffer_size, bool blocking); +bool http_server_ostream_get_size(struct http_server_ostream *hsostream, + uoff_t *size_r); +int http_server_ostream_continue(struct http_server_ostream *hsostream); + +void http_server_ostream_output_available( + struct http_server_ostream *hsostream); +void http_server_ostream_response_finished( + struct http_server_ostream *hsostream); +void http_server_ostream_response_destroyed( + struct http_server_ostream *hsostream); + +struct ostream * +http_server_ostream_get_output(struct http_server_ostream *hsostream); + +void http_server_ostream_set_error(struct http_server_ostream *hsostream, + int stream_errno, const char *stream_error); + +/* + * Response + */ + +void http_server_response_request_free(struct http_server_response *resp); +void http_server_response_request_destroy(struct http_server_response *resp); +void http_server_response_request_abort(struct http_server_response *resp, + const char *reason); +void http_server_response_request_finished(struct http_server_response *resp); + +int http_server_response_send(struct http_server_response *resp); +int http_server_response_send_more(struct http_server_response *resp); +int http_server_response_finish_payload_out(struct http_server_response *resp); + +/* + * Request + */ + +static inline bool +http_server_request_is_new(struct http_server_request *req) +{ + return (req->state == HTTP_SERVER_REQUEST_STATE_NEW); +} + +static inline bool +http_server_request_version_equals(struct http_server_request *req, + unsigned int major, unsigned int minor) +{ + return (req->req.version_major == major && + req->req.version_minor == minor); +} + +const char *http_server_request_label(struct http_server_request *req); + +void http_server_request_update_event(struct http_server_request *req); + +struct http_server_request * +http_server_request_new(struct http_server_connection *conn); +void http_server_request_destroy(struct http_server_request **_req); +void http_server_request_abort(struct http_server_request **_req, + const char *reason) ATTR_NULL(2); + +void http_server_request_immune_ref(struct http_server_request *req); +void http_server_request_immune_unref(struct http_server_request **_req); + +bool http_server_request_is_complete(struct http_server_request *req); + +void http_server_request_received(struct http_server_request *req); +void http_server_request_callback(struct http_server_request *req); + +void http_server_request_halt_payload(struct http_server_request *req); +void http_server_request_continue_payload(struct http_server_request *req); + +void http_server_request_submit_response(struct http_server_request *req); + +void http_server_request_ready_to_respond(struct http_server_request *req); +void http_server_request_finished(struct http_server_request *req); + +/* Payload handler */ + +void http_server_payload_handler_destroy( + struct http_server_payload_handler **_handler); +void http_server_payload_handler_switch_ioloop( + struct http_server_payload_handler *handler, struct ioloop *ioloop); + +/* + * Connection + */ + +static inline void +http_server_connection_add_request(struct http_server_connection *conn, + struct http_server_request *sreq) +{ + DLLIST2_APPEND(&conn->request_queue_head, &conn->request_queue_tail, + sreq); + conn->request_queue_count++; +} +static inline void +http_server_connection_remove_request(struct http_server_connection *conn, + struct http_server_request *sreq) +{ + DLLIST2_REMOVE(&conn->request_queue_head, &conn->request_queue_tail, + sreq); + conn->request_queue_count--; +} + +struct connection_list *http_server_connection_list_init(void); + +bool http_server_connection_shut_down(struct http_server_connection *conn); + +void http_server_connection_input_set_pending( + struct http_server_connection *conn); +void http_server_connection_input_halt(struct http_server_connection *conn); +void http_server_connection_input_resume(struct http_server_connection *conn); + +void http_server_connection_start_idle_timeout( + struct http_server_connection *conn); +void http_server_connection_reset_idle_timeout( + struct http_server_connection *conn); +void http_server_connection_stop_idle_timeout( + struct http_server_connection *conn); + +void http_server_connection_handle_output_error( + struct http_server_connection *conn); + +void http_server_connection_output_trigger(struct http_server_connection *conn); +void http_server_connection_output_halt(struct http_server_connection *conn); +void http_server_connection_output_resume(struct http_server_connection *conn); + +int http_server_connection_flush(struct http_server_connection *conn); +int http_server_connection_output(struct http_server_connection *conn); + +void http_server_connection_tunnel(struct http_server_connection **_conn, + http_server_tunnel_callback_t callback, + void *context); + +bool http_server_connection_pending_payload( + struct http_server_connection *conn); + +/* + * Resource + */ + +int http_server_resource_find(struct http_server *server, const char *path, + struct http_server_resource **res_r, + const char **sub_path_r) ATTR_NULL(2); + +bool http_server_resource_callback(struct http_server_request *req); + +/* + * Server + */ + +int http_server_init_ssl_ctx(struct http_server *server, const char **error_r); + +#endif diff --git a/src/lib-http/http-server-request.c b/src/lib-http/http-server-request.c new file mode 100644 index 0000000..df2ce1a --- /dev/null +++ b/src/lib-http/http-server-request.c @@ -0,0 +1,1006 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "ostream.h" +#include "istream-private.h" +#include "str-sanitize.h" + +#include "http-server-private.h" + +/* + * Logging + */ + +static inline void +http_server_request_client_error(struct http_server_request *req, + const char *format, ...) ATTR_FORMAT(2, 3); + +static inline void +http_server_request_client_error(struct http_server_request *req, + const char *format, ...) +{ + va_list args; + + va_start(args, format); + e_info(req->event, "%s", t_strdup_vprintf(format, args)); + va_end(args); +} + +/* + * Request + */ + +const char *http_server_request_label(struct http_server_request *req) +{ + if (req->req.target_raw == NULL) { + if (req->req.method == NULL) + return t_strdup_printf("[Req%u: <NEW>]", req->id); + return t_strdup_printf("[Req%u: %s <INCOMPLETE>]", + req->id, req->req.method); + } + return t_strdup_printf("[Req%u: %s %s]", req->id, + req->req.method, req->req.target_raw); +} + +void http_server_request_update_event(struct http_server_request *req) +{ + if (req->req.method != NULL) + event_add_str(req->event, "method", req->req.method); + if (req->req.target_raw != NULL) + event_add_str(req->event, "target", req->req.target_raw); + event_add_int(req->event, "request_id", req->id); + event_set_append_log_prefix( + req->event, t_strdup_printf("request %s: ", + str_sanitize(http_server_request_label(req), 256))); +} + +struct http_server_request * +http_server_request_new(struct http_server_connection *conn) +{ + static unsigned int id_counter = 0; + pool_t pool; + struct http_server_request *req; + + pool = pool_alloconly_create( + MEMPOOL_GROWING"http_server_request", 4096); + req = p_new(pool, struct http_server_request, 1); + req->pool = pool; + req->refcount = 1; + req->conn = conn; + req->server = conn->server; + req->id = ++id_counter; + req->event = event_create(conn->event); + req->input_start_offset = conn->conn.input->v_offset; + req->output_start_offset = conn->conn.output->offset; + http_server_request_update_event(req); + + http_server_connection_add_request(conn, req); + return req; +} + +void http_server_request_ref(struct http_server_request *req) +{ + i_assert(req->refcount > 0); + req->refcount++; +} + +bool http_server_request_unref(struct http_server_request **_req) +{ + struct http_server_request *req = *_req; + struct http_server_connection *conn = req->conn; + + i_assert(req->refcount > 0); + + *_req = NULL; + if (--req->refcount > 0) + return TRUE; + + e_debug(req->event, "Free"); + + if (req->state < HTTP_SERVER_REQUEST_STATE_FINISHED) { + req->state = HTTP_SERVER_REQUEST_STATE_ABORTED; + http_server_connection_remove_request(conn, req); + } + + if (req->destroy_callback != NULL) { + req->destroy_callback(req->destroy_context); + req->destroy_callback = NULL; + } + + if (req->response != NULL) + http_server_response_request_free(req->response); + event_unref(&req->event); + pool_unref(&req->pool); + return FALSE; +} + +void http_server_request_connection_close(struct http_server_request *req, + bool close) +{ + i_assert(req->state < HTTP_SERVER_REQUEST_STATE_SENT_RESPONSE); + req->connection_close = close; +} + +void http_server_request_destroy(struct http_server_request **_req) +{ + struct http_server_request *req = *_req; + struct http_server *server = req->server; + + e_debug(req->event, "Destroy"); + + /* Just make sure the request ends in a proper state */ + if (req->state < HTTP_SERVER_REQUEST_STATE_FINISHED) + req->state = HTTP_SERVER_REQUEST_STATE_ABORTED; + + if (server->ioloop != NULL) + io_loop_stop(server->ioloop); + + if (req->immune_refcount > 0) { + req->destroy_pending = TRUE; + http_server_request_unref(_req); + return; + } + + if (req->response != NULL) + http_server_response_request_destroy(req->response); + + if (req->destroy_callback != NULL) { + void (*callback)(void *) = req->destroy_callback; + + req->destroy_callback = NULL; + callback(req->destroy_context); + } + + http_server_request_unref(_req); +} + +#undef http_server_request_set_destroy_callback +void http_server_request_set_destroy_callback(struct http_server_request *req, + void (*callback)(void *), + void *context) +{ + req->destroy_callback = callback; + req->destroy_context = context; +} + +void http_server_request_abort(struct http_server_request **_req, + const char *reason) +{ + struct http_server_request *req = *_req; + struct http_server_connection *conn = req->conn; + + if (req->state >= HTTP_SERVER_REQUEST_STATE_FINISHED) + return; + + if (reason == NULL) + e_debug(req->event, "Abort"); + else + e_debug(req->event, "Abort: %s", reason); + + if (req->response != NULL) + http_server_response_request_abort(req->response, reason); + + req->conn = NULL; + if (req->state < HTTP_SERVER_REQUEST_STATE_FINISHED) { + if (conn != NULL) { + http_server_connection_remove_request(conn, req); + + if (!conn->closed) { + /* Send best-effort response if appropriate */ + if (!conn->output_locked && + req->state >= HTTP_SERVER_REQUEST_STATE_PROCESSING && + req->state < HTTP_SERVER_REQUEST_STATE_SENT_RESPONSE) { + static const char *response = + "HTTP/1.1 500 Internal Server Error\r\n" + "Content-Length: 0\r\n" + "\r\n"; + + o_stream_nsend(conn->conn.output, + response, strlen(response)); + (void)o_stream_flush(conn->conn.output); + } + + /* Close the connection */ + http_server_connection_close(&conn, reason); + } + } + + req->state = HTTP_SERVER_REQUEST_STATE_ABORTED; + } + + http_server_request_destroy(_req); +} + +void http_server_request_immune_ref(struct http_server_request *req) +{ + http_server_request_ref(req); + req->immune_refcount++; +} + +void http_server_request_immune_unref(struct http_server_request **_req) +{ + struct http_server_request *req = *_req; + + i_assert(req->immune_refcount > 0); + + *_req = NULL; + if (--req->immune_refcount == 0 && req->destroy_pending) + http_server_request_destroy(&req); + else + http_server_request_unref(&req); +} + +const struct http_request * +http_server_request_get(struct http_server_request *req) +{ + return &req->req; +} + +pool_t http_server_request_get_pool(struct http_server_request *req) +{ + return req->pool; +} + +struct http_server_response * +http_server_request_get_response(struct http_server_request *req) +{ + return req->response; +} + +int http_server_request_get_auth(struct http_server_request *req, + struct http_auth_credentials *credentials) +{ + const char *auth; + + auth = http_request_header_get(&req->req, "Authorization"); + if (auth == NULL) + return 0; + + if (http_auth_parse_credentials((const unsigned char *)auth, + strlen(auth), credentials) < 0) + return -1; + + return 1; +} + +bool http_server_request_is_finished(struct http_server_request *req) +{ + return (req->response != NULL || + req->state == HTTP_SERVER_REQUEST_STATE_ABORTED); +} + +bool http_server_request_is_complete(struct http_server_request *req) +{ + return (req->failed || req->conn->input_broken || + (req->next != NULL && !http_server_request_is_new(req->next)) || + !http_server_connection_pending_payload(req->conn)); +} + +void http_server_request_halt_payload(struct http_server_request *req) +{ + i_assert(req->state <= HTTP_SERVER_REQUEST_STATE_QUEUED); + req->payload_halted = TRUE; +} + +void http_server_request_continue_payload(struct http_server_request *req) +{ + i_assert(req->state <= HTTP_SERVER_REQUEST_STATE_QUEUED); + req->payload_halted = FALSE; + if (req->req.expect_100_continue && !req->sent_100_continue) + http_server_connection_output_trigger(req->conn); +} + +static void +http_server_request_connect_callback(struct http_server_request *req) +{ + struct http_server_connection *conn = req->conn; + + if (conn->callbacks->handle_connect_request == NULL) { + http_server_request_fail(req, 505, "Not Implemented"); + return; + } + + if (req->req.target.format != + HTTP_REQUEST_TARGET_FORMAT_AUTHORITY) { + http_server_request_fail(req, 400, "Bad Request"); + return; + } + + conn->callbacks->handle_connect_request(conn->context, req, + req->req.target.url); +} + +static void +http_server_request_default_handler(struct http_server_request *req) +{ + const struct http_request *hreq = &req->req; + struct http_server_response *resp; + + if (strcmp(hreq->method, "OPTIONS") == 0 && + hreq->target.format == HTTP_REQUEST_TARGET_FORMAT_ASTERISK) { + resp = http_server_response_create(req, 200, "OK"); + http_server_response_submit(resp); + return; + } + + http_server_request_fail(req, 404, "Not Found"); + return; +} + +void http_server_request_received(struct http_server_request *req) +{ + http_server_request_update_event(req); + struct event_passthrough *e = event_create_passthrough(req->event)-> + set_name("http_server_request_started"); + e_debug(e->event(), "Received new request %s " + "(%u requests pending; %u maximum)", + http_server_request_label(req), + req->conn->request_queue_count, + req->conn->server->set.max_pipelined_requests); +} + +void http_server_request_callback(struct http_server_request *req) +{ + struct http_server_connection *conn = req->conn; + + if (strcmp(req->req.method, "CONNECT") == 0) { + /* CONNECT method */ + http_server_request_connect_callback(req); + return; + } + + if (http_server_resource_callback(req)) + return; + + if (array_count(&req->server->resources) > 0) + e_debug(req->event, "No matching resource found"); + + if (conn->callbacks->handle_request == NULL) { + http_server_request_default_handler(req); + return; + } + conn->callbacks->handle_request(conn->context, req); +} + +void http_server_request_ready_to_respond(struct http_server_request *req) +{ + e_debug(req->event, "Ready to respond"); + + req->state = HTTP_SERVER_REQUEST_STATE_READY_TO_RESPOND; + http_server_connection_output_trigger(req->conn); +} + +void http_server_request_submit_response(struct http_server_request *req) +{ + struct http_server_connection *conn = req->conn; + + i_assert(conn != NULL && req->response != NULL && + req->response->submitted); + + http_server_request_ref(req); + + if (conn->payload_handler != NULL && conn->payload_handler->req == req) + http_server_payload_handler_destroy(&conn->payload_handler); + + switch (req->state) { + case HTTP_SERVER_REQUEST_STATE_NEW: + case HTTP_SERVER_REQUEST_STATE_QUEUED: + case HTTP_SERVER_REQUEST_STATE_PAYLOAD_IN: + case HTTP_SERVER_REQUEST_STATE_PROCESSING: + case HTTP_SERVER_REQUEST_STATE_SUBMITTED_RESPONSE: + if (!http_server_request_is_complete(req)) { + e_debug(req->event, "Not ready to respond"); + req->state = HTTP_SERVER_REQUEST_STATE_SUBMITTED_RESPONSE; + http_server_connection_input_resume(req->conn); + http_server_connection_input_set_pending(req->conn); + break; + } + http_server_request_ready_to_respond(req); + break; + case HTTP_SERVER_REQUEST_STATE_READY_TO_RESPOND: + http_server_connection_output_trigger(req->conn); + break; + case HTTP_SERVER_REQUEST_STATE_ABORTED: + break; + default: + i_unreached(); + } + + http_server_request_unref(&req); +} + +void http_server_request_finished(struct http_server_request *req) +{ + struct http_server_connection *conn = req->conn; + struct http_server_response *resp = req->response; + http_server_tunnel_callback_t tunnel_callback = resp->tunnel_callback; + void *tunnel_context = resp->tunnel_context; + + i_assert(req->state < HTTP_SERVER_REQUEST_STATE_FINISHED); + req->state = HTTP_SERVER_REQUEST_STATE_FINISHED; + + http_server_connection_remove_request(conn, req); + conn->stats.response_count++; + + if (req->response != NULL) + http_server_response_request_finished(req->response); + + uoff_t bytes_in = req->conn->conn.input->v_offset - + req->input_start_offset; + uoff_t bytes_out = req->conn->conn.output->offset - + req->output_start_offset; + struct event_passthrough *e = event_create_passthrough(req->event)-> + set_name("http_server_request_finished")-> + add_int("bytes_in", bytes_in)-> + add_int("bytes_out", bytes_out); + e_debug(e->event(), "Finished request"); + + if (tunnel_callback == NULL) { + if (req->connection_close) { + http_server_connection_close(&conn, + t_strdup_printf( + "Server closed connection: %u %s", + resp->status, resp->reason)); + http_server_request_destroy(&req); + return; + } else if (req->conn->input_broken) { + http_server_connection_close( + &conn, "Connection input is broken"); + http_server_request_destroy(&req); + return; + } else if (req->req.connection_close) { + http_server_connection_close( + &conn, "Client requested connection close"); + http_server_request_destroy(&req); + return; + } + } + + http_server_request_destroy(&req); + if (tunnel_callback != NULL) { + http_server_connection_tunnel(&conn, tunnel_callback, + tunnel_context); + return; + } + + http_server_connection_output_trigger(conn); +} + +static struct http_server_response * +http_server_request_create_fail_response(struct http_server_request *req, + unsigned int status, + const char *reason, const char *text) + ATTR_NULL(4) +{ + struct http_server_response *resp; + + req->failed = TRUE; + + i_assert(status / 100 != 1 && status != 204 && status != 304); + + resp = http_server_response_create(req, status, reason); + if (!http_request_method_is(&req->req, "HEAD")) { + http_server_response_add_header(resp, "Content-Type", + "text/plain; charset=utf-8"); + if (text == NULL) + text = reason; + text = t_strconcat(text, "\r\n", NULL); + http_server_response_set_payload_data( + resp, (const unsigned char *)text, strlen(text)); + } + + return resp; +} + +static void +http_server_request_fail_full(struct http_server_request *req, + unsigned int status, const char *reason, + const char *text) ATTR_NULL(4) +{ + struct http_server_response *resp; + + req->failed = TRUE; + resp = http_server_request_create_fail_response(req, status, reason, + text); + http_server_response_submit(resp); + if (req->conn->input_broken) + req->connection_close = TRUE; +} + +void http_server_request_fail(struct http_server_request *req, + unsigned int status, const char *reason) +{ + http_server_request_fail_full(req, status, reason, NULL); +} + +void http_server_request_fail_close(struct http_server_request *req, + unsigned int status, const char *reason) +{ + http_server_request_connection_close(req, TRUE); + http_server_request_fail_full(req, status, reason, NULL); +} + +void http_server_request_fail_text(struct http_server_request *req, + unsigned int status, const char *reason, + const char *format, ...) +{ + va_list args; + + va_start(args, format); + http_server_request_fail_full(req, status, reason, + t_strdup_vprintf(format, args)); + va_end(args); +} + +void http_server_request_fail_auth(struct http_server_request *req, + const char *reason, + const struct http_auth_challenge *chlng) +{ + struct http_server_response *resp; + + req->failed = TRUE; + + if (reason == NULL) + reason = "Unauthenticated"; + + resp = http_server_request_create_fail_response(req, 401, reason, + reason); + http_server_response_add_auth(resp, chlng); + http_server_response_submit(resp); +} + +void http_server_request_fail_auth_basic(struct http_server_request *req, + const char *reason, const char *realm) +{ + struct http_auth_challenge chlng; + + http_auth_basic_challenge_init(&chlng, realm); + http_server_request_fail_auth(req, reason, &chlng); +} + +void http_server_request_fail_bad_method(struct http_server_request *req, + const char *allow) +{ + struct http_server_response *resp; + const char *reason = "Method Not Allowed"; + + req->failed = TRUE; + + resp = http_server_request_create_fail_response(req, 405, reason, + reason); + http_server_response_add_header(resp, "Allow", allow); + http_server_response_submit(resp); +} + +/* + * Payload input stream + */ + +struct http_server_istream { + struct istream_private istream; + + struct http_server_request *req; + + ssize_t read_status; +}; + +static void +http_server_istream_switch_ioloop_to(struct istream_private *stream, + struct ioloop *ioloop) +{ + struct http_server_istream *hsristream = + (struct http_server_istream *)stream; + + if (hsristream->istream.istream.blocking) + return; + + i_assert(ioloop == current_ioloop); + http_server_connection_switch_ioloop(hsristream->req->conn); +} + +static void +http_server_istream_read_any(struct http_server_istream *hsristream) +{ + struct istream_private *stream = &hsristream->istream; + struct http_server *server = hsristream->req->server; + ssize_t ret; + + if ((ret = i_stream_read_copy_from_parent(&stream->istream)) != 0) { + hsristream->read_status = ret; + io_loop_stop(server->ioloop); + } +} + +static ssize_t +http_server_istream_read(struct istream_private *stream) +{ + struct http_server_istream *hsristream = + (struct http_server_istream *)stream; + struct http_server_request *req = hsristream->req; + struct http_server *server; + struct http_server_connection *conn; + bool blocking = stream->istream.blocking; + ssize_t ret; + + if (req == NULL) { + /* Request already gone (we shouldn't get here) */ + stream->istream.stream_errno = EINVAL; + return -1; + } + + i_stream_seek(stream->parent, stream->parent_start_offset + + stream->istream.v_offset); + + server = hsristream->req->server; + conn = hsristream->req->conn; + + ret = i_stream_read_copy_from_parent(&stream->istream); + if (ret == 0 && blocking) { + struct ioloop *prev_ioloop = current_ioloop; + struct io *io; + + http_server_connection_ref(conn); + http_server_request_ref(req); + + i_assert(server->ioloop == NULL); + server->ioloop = io_loop_create(); + http_server_connection_switch_ioloop(conn); + + if (blocking && req->req.expect_100_continue && + !req->sent_100_continue) + http_server_connection_output_trigger(conn); + + hsristream->read_status = 0; + io = io_add_istream(&stream->istream, + http_server_istream_read_any, hsristream); + while (req->state < HTTP_SERVER_REQUEST_STATE_FINISHED && + hsristream->read_status == 0) { + io_loop_run(server->ioloop); + } + io_remove(&io); + + io_loop_set_current(prev_ioloop); + http_server_connection_switch_ioloop(conn); + io_loop_set_current(server->ioloop); + io_loop_destroy(&server->ioloop); + + ret = hsristream->read_status; + + if (!http_server_request_unref(&req)) + hsristream->req = NULL; + http_server_connection_unref(&conn); + } + + return ret; +} + +static void +http_server_istream_destroy(struct iostream_private *stream) +{ + struct http_server_istream *hsristream = + (struct http_server_istream *)stream; + uoff_t v_offset; + + v_offset = hsristream->istream.parent_start_offset + + hsristream->istream.istream.v_offset; + if (hsristream->istream.parent->seekable || + v_offset > hsristream->istream.parent->v_offset) { + /* Get to same position in parent stream */ + i_stream_seek(hsristream->istream.parent, v_offset); + } +} + +struct istream * +http_server_request_get_payload_input(struct http_server_request *req, + bool blocking) +{ + struct http_server_istream *hsristream; + struct istream *payload = req->req.payload; + + i_assert(req->payload_input == NULL); + + hsristream = i_new(struct http_server_istream, 1); + hsristream->req = req; + hsristream->istream.max_buffer_size = + payload->real_stream->max_buffer_size; + hsristream->istream.stream_size_passthrough = TRUE; + + hsristream->istream.read = http_server_istream_read; + hsristream->istream.switch_ioloop_to = + http_server_istream_switch_ioloop_to; + hsristream->istream.iostream.destroy = http_server_istream_destroy; + + hsristream->istream.istream.readable_fd = FALSE; + hsristream->istream.istream.blocking = blocking; + hsristream->istream.istream.seekable = FALSE; + + req->payload_input = i_stream_create(&hsristream->istream, payload, + i_stream_get_fd(payload), 0); + i_stream_unref(&req->req.payload); + return req->payload_input; +} + +/* + * Payload handling + */ + +static void +http_server_payload_handler_init(struct http_server_payload_handler *handler, + struct http_server_request *req) +{ + struct http_server_connection *conn = req->conn; + + i_assert(conn->payload_handler == NULL); + i_assert(conn->in_req_callback); + + conn->payload_handler = handler; + + handler->req = req; +} + +void http_server_payload_handler_destroy( + struct http_server_payload_handler **_handler) +{ + struct http_server_payload_handler *handler = *_handler; + struct http_server_connection *conn = handler->req->conn; + + if (handler->in_callback) { + /* Don't destroy handler while in callback */ + return; + } + + *_handler = NULL; + i_assert(conn->payload_handler == NULL); + + if (handler->destroy != NULL) + handler->destroy(handler); +} + +void http_server_payload_handler_switch_ioloop( + struct http_server_payload_handler *handler, struct ioloop *ioloop) +{ + if (handler->switch_ioloop != NULL) + handler->switch_ioloop(handler, ioloop); +} + +/* Pump-based */ + +struct http_server_payload_handler_pump { + struct http_server_payload_handler handler; + + struct iostream_pump *pump; + + void (*callback)(void *); + void *context; +}; + +static void +payload_handler_pump_destroy(struct http_server_payload_handler *handler) +{ + struct http_server_payload_handler_pump *phandler = + (struct http_server_payload_handler_pump *)handler; + + iostream_pump_unref(&phandler->pump); +} + +static void +payload_handler_pump_switch_ioloop(struct http_server_payload_handler *handler, + struct ioloop *ioloop) +{ + struct http_server_payload_handler_pump *phandler = + (struct http_server_payload_handler_pump *)handler; + + iostream_pump_switch_ioloop_to(phandler->pump, ioloop); +} + +static void +payload_handler_pump_callback(enum iostream_pump_status status, + struct http_server_payload_handler_pump *phandler) +{ + struct http_server_payload_handler *handler = &phandler->handler; + struct http_server_request *req = handler->req; + struct http_server_connection *conn = req->conn; + struct istream *input = iostream_pump_get_input(phandler->pump); + struct ostream *output = iostream_pump_get_output(phandler->pump); + + switch (status) { + case IOSTREAM_PUMP_STATUS_INPUT_EOF: + if (!i_stream_read_eof(conn->incoming_payload)) { + http_server_request_fail_close(req, 413, + "Payload Too Large"); + } else { + unsigned int old_refcount = req->refcount; + + handler->in_callback = TRUE; + phandler->callback(phandler->context); + req->callback_refcount += req->refcount - old_refcount; + handler->in_callback = FALSE; + + i_assert(req->callback_refcount > 0 || + (req->response != NULL && + req->response->submitted)); + } + break; + case IOSTREAM_PUMP_STATUS_INPUT_ERROR: + http_server_request_client_error( + req, "iostream_pump: read(%s) failed: %s", + i_stream_get_name(input), i_stream_get_error(input)); + http_server_request_fail_close(req, 400, "Bad Request"); + break; + case IOSTREAM_PUMP_STATUS_OUTPUT_ERROR: + if (output->stream_errno != 0) { + e_error(req->event, + "iostream_pump: write(%s) failed: %s", + o_stream_get_name(output), + o_stream_get_error(output)); + } + http_server_request_fail_close(req, 500, + "Internal Server Error"); + break; + } + + if (conn->payload_handler != NULL) + http_server_payload_handler_destroy(&conn->payload_handler); +} + +#undef http_server_request_forward_payload +void http_server_request_forward_payload(struct http_server_request *req, + struct ostream *output, + uoff_t max_size, + void (*callback)(void *), + void *context) +{ + struct http_server_connection *conn = req->conn; + struct istream *input = conn->incoming_payload; + struct http_server_payload_handler_pump *phandler; + uoff_t payload_size; + int ret; + + i_assert(req->req.payload != NULL); + + if (max_size == UOFF_T_MAX) { + i_stream_ref(input); + } else { + if ((ret = i_stream_get_size(input, TRUE, + &payload_size)) != 0) { + if (ret < 0) { + e_error(req->event, + "i_stream_get_size(%s) failed: %s", + i_stream_get_name(input), + i_stream_get_error(input)); + http_server_request_fail_close( + req, 500, "Internal Server Error"); + return; + } + if (payload_size > max_size) { + http_server_request_fail_close( + req, 413, "Payload Too Large"); + return; + } + } + input = i_stream_create_limit(input, max_size); + } + + phandler = p_new(req->pool, struct http_server_payload_handler_pump, 1); + http_server_payload_handler_init(&phandler->handler, req); + phandler->handler.switch_ioloop = payload_handler_pump_switch_ioloop; + phandler->handler.destroy = payload_handler_pump_destroy; + phandler->callback = callback; + phandler->context = context; + + phandler->pump = iostream_pump_create(input, output); + iostream_pump_set_completion_callback(phandler->pump, + payload_handler_pump_callback, + phandler); + iostream_pump_start(phandler->pump); + i_stream_unref(&input); +} + +#undef http_server_request_buffer_payload +void http_server_request_buffer_payload(struct http_server_request *req, + buffer_t *buffer, uoff_t max_size, + void (*callback)(void *), + void *context) +{ + struct ostream *output; + + output = o_stream_create_buffer(buffer); + http_server_request_forward_payload(req, + output, max_size, callback, context); + o_stream_unref(&output); +} + +/* Raw */ + +struct http_server_payload_handler_raw { + struct http_server_payload_handler handler; + + struct io *io; + + void (*callback)(void *context); + void *context; +}; + +static void +payload_handler_raw_destroy(struct http_server_payload_handler *handler) +{ + struct http_server_payload_handler_raw *rhandler = + (struct http_server_payload_handler_raw *)handler; + + io_remove(&rhandler->io); +} + +static void +payload_handler_raw_switch_ioloop(struct http_server_payload_handler *handler, + struct ioloop *ioloop) +{ + struct http_server_payload_handler_raw *rhandler = + (struct http_server_payload_handler_raw *)handler; + + rhandler->io = io_loop_move_io_to(ioloop, &rhandler->io); +} + +static void +payload_handler_raw_input(struct http_server_payload_handler_raw *rhandler) +{ + struct http_server_payload_handler *handler = &rhandler->handler; + struct http_server_request *req = handler->req; + struct http_server_connection *conn = req->conn; + struct istream *input = conn->incoming_payload; + unsigned int old_refcount = req->refcount; + + handler->in_callback = TRUE; + rhandler->callback(rhandler->context); + req->callback_refcount += req->refcount - old_refcount; + handler->in_callback = FALSE; + + if (input != NULL && input->stream_errno != 0) { + if (req->response == NULL) { + http_server_request_client_error( + req, "read(%s) failed: %s", + i_stream_get_name(input), + i_stream_get_error(input)); + http_server_request_fail_close(req, 400, "Bad Request"); + } + } else if (input == NULL || !i_stream_have_bytes_left(input)) { + i_assert(req->callback_refcount > 0 || + (req->response != NULL && req->response->submitted)); + } else { + return; + } + + if (conn->payload_handler != NULL) + http_server_payload_handler_destroy(&conn->payload_handler); + +} + +#undef http_server_request_handle_payload +void http_server_request_handle_payload(struct http_server_request *req, + void (*callback)(void *context), + void *context) +{ + struct http_server_payload_handler_raw *rhandler; + struct http_server_connection *conn = req->conn; + + rhandler = p_new(req->pool, struct http_server_payload_handler_raw, 1); + http_server_payload_handler_init(&rhandler->handler, req); + rhandler->handler.switch_ioloop = payload_handler_raw_switch_ioloop; + rhandler->handler.destroy = payload_handler_raw_destroy; + rhandler->callback = callback; + rhandler->context = context; + + rhandler->io = io_add_istream(conn->incoming_payload, + payload_handler_raw_input, rhandler); + i_stream_set_input_pending(conn->incoming_payload, TRUE); +} + +void http_server_request_add_response_header(struct http_server_request *req, + const char *key, const char *value) +{ + struct http_server_response *resp; + + resp = http_server_response_create(req, 0, ""); + http_server_response_add_permanent_header(resp, key, value); +} diff --git a/src/lib-http/http-server-resource.c b/src/lib-http/http-server-resource.c new file mode 100644 index 0000000..dc602e4 --- /dev/null +++ b/src/lib-http/http-server-resource.c @@ -0,0 +1,276 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "bsearch-insert-pos.h" +#include "str-sanitize.h" + +#include "http-url.h" +#include "http-server-private.h" + +static struct event_category event_category_http_server_resource = { + .name = "http-server-resource" +}; + +/* + * Location + */ + +static int +http_server_location_cmp(struct http_server_location *const *loc1, + struct http_server_location *const *loc2) +{ + return strcmp((*loc1)->path, (*loc2)->path); +} + +static struct http_server_location * +http_server_location_add(struct http_server *server, pool_t pool, + const char *path) +{ + struct http_server_location qloc, *loc; + unsigned int insert_idx; + + i_zero(&qloc); + qloc.path = path; + loc = &qloc; + + if (array_bsearch_insert_pos(&server->locations, &loc, + http_server_location_cmp, &insert_idx)) + return array_idx_elem(&server->locations, insert_idx); + + loc = p_new(pool, struct http_server_location, 1); + loc->path = p_strdup(pool, path); + array_insert(&server->locations, insert_idx, &loc, 1); + return loc; +} + +static int +http_server_location_find(struct http_server *server, const char *path, + struct http_server_location **loc_r, + const char **sub_path_r) +{ + struct http_server_location qloc, *loc; + size_t loc_len; + unsigned int insert_idx; + + *sub_path_r = NULL; + *loc_r = NULL; + + i_zero(&qloc); + qloc.path = path; + loc = &qloc; + + if (array_bsearch_insert_pos(&server->locations, &loc, + http_server_location_cmp, &insert_idx)) { + /* Exact match */ + *loc_r = array_idx_elem(&server->locations, insert_idx); + *sub_path_r = ""; + return 1; + } + if (insert_idx == 0) { + /* Not found at all */ + return -1; + } + loc = array_idx_elem(&server->locations, insert_idx-1); + + loc_len = strlen(loc->path); + if (!str_begins(path, loc->path)) { + /* Location isn't a prefix of path */ + return -1; + } else if (path[loc_len] != '/') { + /* Match doesn't end at '/' */ + return -1; + } + + *sub_path_r = &path[loc_len + 1]; + *loc_r = loc; + return 0; +} + +static void +http_server_location_remove(struct http_server *server, + struct http_server_location *loc) +{ + struct http_server_location *const *locp; + + array_foreach(&server->locations, locp) { + if (*locp == loc) { + array_delete( + &server->locations, + array_foreach_idx(&server->locations, locp), 1); + return; + } + } +} + +/* + * Resource + */ + +static void http_server_resource_update_event(struct http_server_resource *res) +{ + struct http_server_location *const *locs; + unsigned int locs_count; + + locs = array_get(&res->locations, &locs_count); + if (locs_count == 0) { + event_set_append_log_prefix(res->event, "resource: "); + return; + } + + event_add_str(res->event, "path", locs[0]->path); + event_set_append_log_prefix( + res->event, t_strdup_printf("resource %s: ", + str_sanitize(locs[0]->path, 128))); +} + +#undef http_server_resource_create +struct http_server_resource * +http_server_resource_create(struct http_server *server, pool_t pool, + http_server_resource_callback_t *callback, + void *context) +{ + struct http_server_resource *res; + + pool_ref(pool); + + res = p_new(pool, struct http_server_resource, 1); + res->pool = pool; + res->server = server; + + res->callback = callback; + res->context = context; + + p_array_init(&res->locations, pool, 4); + + res->event = event_create(server->event); + event_add_category(res->event, &event_category_http_server_resource); + http_server_resource_update_event(res); + + array_append(&server->resources, &res, 1); + + return res; +} + +void http_server_resource_free(struct http_server_resource **_res) +{ + struct http_server_resource *res = *_res; + struct http_server_location *loc; + + if (res == NULL) + return; + + *_res = NULL; + + e_debug(res->event, "Free"); + + if (res->destroy_callback != NULL) { + res->destroy_callback(res->destroy_context); + res->destroy_callback = NULL; + } + + array_foreach_elem(&res->locations, loc) + http_server_location_remove(res->server, loc); + + event_unref(&res->event); + pool_unref(&res->pool); +} + +pool_t http_server_resource_get_pool(struct http_server_resource *res) +{ + return res->pool; +} + +const char *http_server_resource_get_path(struct http_server_resource *res) +{ + struct http_server_location *const *locs; + unsigned int locs_count; + + locs = array_get(&res->locations, &locs_count); + i_assert(locs_count > 0); + + return locs[0]->path; +} + +struct event *http_server_resource_get_event(struct http_server_resource *res) +{ + return res->event; +} + +void http_server_resource_add_location(struct http_server_resource *res, + const char *path) +{ + struct http_server_location *loc; + + i_assert(*path == '\0' || *path == '/'); + + loc = http_server_location_add(res->server, res->pool, path); + i_assert(loc->resource == NULL); + + loc->resource = res; + array_append(&res->locations, &loc, 1); + + if (array_count(&res->locations) == 1) + http_server_resource_update_event(res); +} + +int http_server_resource_find(struct http_server *server, const char *path, + struct http_server_resource **res_r, + const char **sub_path_r) +{ + struct http_server_location *loc; + int ret; + + if (path == NULL) + return -1; + + *res_r = NULL; + *sub_path_r = NULL; + + ret = http_server_location_find(server, path, &loc, sub_path_r); + if (ret < 0) + return -1; + + i_assert(loc->resource != NULL); + *res_r = loc->resource; + return ret; +} + +bool http_server_resource_callback(struct http_server_request *req) +{ + struct http_server *server = req->server; + struct http_server_resource *res; + const char *sub_path; + + switch (req->req.target.format) { + case HTTP_REQUEST_TARGET_FORMAT_ORIGIN: + /* According to RFC 7240, Section 5.3.1 only the origin form is + applicable to local resources on an origin server. + */ + break; + case HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE: + case HTTP_REQUEST_TARGET_FORMAT_AUTHORITY: + case HTTP_REQUEST_TARGET_FORMAT_ASTERISK: + /* Not applicable for a local resource. */ + return FALSE; + } + + if (http_server_resource_find(server, req->req.target.url->path, + &res, &sub_path) < 0) + return FALSE; + + e_debug(res->event, "Got request: %s", http_server_request_label(req)); + + i_assert(res->callback != NULL); + res->callback(res->context, req, sub_path); + return TRUE; +} + +#undef http_server_resource_set_destroy_callback +void http_server_resource_set_destroy_callback(struct http_server_resource *res, + void (*callback)(void *), + void *context) +{ + res->destroy_callback = callback; + res->destroy_context = context; +} diff --git a/src/lib-http/http-server-response.c b/src/lib-http/http-server-response.c new file mode 100644 index 0000000..8d55593 --- /dev/null +++ b/src/lib-http/http-server-response.c @@ -0,0 +1,801 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "array.h" +#include "istream.h" +#include "ostream-private.h" +#include "http-date.h" +#include "http-transfer.h" +#include "http-server-private.h" + +struct http_server_response_payload { + struct http_server_response *resp; + struct const_iovec *iov; + unsigned int iov_count, iov_idx; + size_t iov_pos; +}; + +/* + * Response + */ + +static void http_server_response_update_event(struct http_server_response *resp) +{ + event_add_int(resp->event, "status", resp->status); + event_set_append_log_prefix(resp->event, + t_strdup_printf("%u response: ", + resp->status)); +} + +struct http_server_response * +http_server_response_create(struct http_server_request *req, + unsigned int status, const char *reason) +{ + struct http_server_response *resp; + + i_assert(req->state < HTTP_SERVER_REQUEST_STATE_SENT_RESPONSE); + + if (req->response == NULL) { + resp = req->response = p_new(req->pool, + struct http_server_response, 1); + } else { + /* Was already composing a response, but decided to + start a new one (would usually be a failure response) + */ + resp = req->response; + + ARRAY_TYPE(string) perm_headers = resp->perm_headers; + i_zero(&resp->perm_headers); + + http_server_response_request_free(resp); + i_zero(resp); + + resp->perm_headers = perm_headers; + } + + resp->request = req; + resp->status = status; + resp->reason = p_strdup(req->pool, reason); + resp->headers = str_new(default_pool, 256); + resp->date = (time_t)-1; + resp->event = event_create(req->event); + http_server_response_update_event(resp); + + if (array_is_created(&resp->perm_headers)) { + unsigned int i, count; + char *const *headers = array_get(&resp->perm_headers, &count); + for (i = 0; i < count; i += 2) + http_server_response_add_header(resp, headers[i], + headers[i+1]); + } + return resp; +} + +void http_server_response_request_free(struct http_server_response *resp) +{ + e_debug(resp->event, "Free"); + + /* Cannot be destroyed while payload output stream still exists */ + i_assert(resp->payload_stream == NULL); + + i_stream_unref(&resp->payload_input); + o_stream_unref(&resp->payload_output); + event_unref(&resp->event); + str_free(&resp->headers); + + if (array_is_created(&resp->perm_headers)) { + char *headers; + + array_foreach_elem(&resp->perm_headers, headers) + i_free(headers); + array_free(&resp->perm_headers); + } +} + +void http_server_response_request_destroy(struct http_server_response *resp) +{ + e_debug(resp->event, "Destroy"); + + if (resp->payload_stream != NULL) + http_server_ostream_response_destroyed(resp->payload_stream); +} + +void http_server_response_request_abort(struct http_server_response *resp, + const char *reason) +{ + if (reason == NULL) + e_debug(resp->event, "Abort"); + else + e_debug(resp->event, "Abort: %s", reason); + + if (resp->payload_stream != NULL) { + http_server_ostream_set_error(resp->payload_stream, + EPIPE, reason); + } +} + +void http_server_response_ref(struct http_server_response *resp) +{ + http_server_request_ref(resp->request); +} + +bool http_server_response_unref(struct http_server_response **_resp) +{ + struct http_server_response *resp = *_resp; + struct http_server_request *req; + + *_resp = NULL; + if (resp == NULL) + return FALSE; + + req = resp->request; + return http_server_request_unref(&req); +} + +void http_server_response_add_header(struct http_server_response *resp, + const char *key, const char *value) +{ + i_assert(!resp->submitted); + i_assert(strchr(key, '\r') == NULL && strchr(key, '\n') == NULL); + i_assert(strchr(value, '\r') == NULL && strchr(value, '\n') == NULL); + + /* Mark presence of special headers */ + switch (key[0]) { + case 'c': case 'C': + if (strcasecmp(key, "Connection") == 0) + resp->have_hdr_connection = TRUE; + else if (strcasecmp(key, "Content-Length") == 0) + resp->have_hdr_body_spec = TRUE; + break; + case 'd': case 'D': + if (strcasecmp(key, "Date") == 0) + resp->have_hdr_date = TRUE; + break; + case 't': case 'T': + if (strcasecmp(key, "Transfer-Encoding") == 0) + resp->have_hdr_body_spec = TRUE; + break; + } + str_printfa(resp->headers, "%s: %s\r\n", key, value); +} + +void http_server_response_update_status(struct http_server_response *resp, + unsigned int status, + const char *reason) +{ + i_assert(!resp->submitted); + resp->status = status; + /* Free not called because pool is alloconly */ + resp->reason = p_strdup(resp->request->pool, reason); +} + +void http_server_response_set_date(struct http_server_response *resp, + time_t date) +{ + i_assert(!resp->submitted); + + resp->date = date; +} + +void http_server_response_set_payload(struct http_server_response *resp, + struct istream *input) +{ + int ret; + + i_assert(!resp->submitted); + i_assert(resp->payload_input == NULL); + i_assert(resp->payload_stream == NULL); + + i_stream_ref(input); + resp->payload_input = input; + if ((ret = i_stream_get_size(input, TRUE, &resp->payload_size)) <= 0) { + if (ret < 0) { + e_error(resp->event, "i_stream_get_size(%s) failed: %s", + i_stream_get_name(input), + i_stream_get_error(input)); + } + resp->payload_size = 0; + resp->payload_chunked = TRUE; + } else { + i_assert(input->v_offset <= resp->payload_size); + resp->payload_size -= input->v_offset; + } + resp->payload_offset = input->v_offset; +} + +void http_server_response_set_payload_data(struct http_server_response *resp, + const unsigned char *data, + size_t size) +{ + struct istream *input; + unsigned char *payload_data; + + i_assert(!resp->submitted); + i_assert(resp->payload_input == NULL); + i_assert(resp->payload_stream == NULL); + + if (size == 0) + return; + + payload_data = p_malloc(resp->request->pool, size); + memcpy(payload_data, data, size); + input = i_stream_create_from_data(payload_data, size); + + http_server_response_set_payload(resp, input); + i_stream_unref(&input); +} + +struct ostream * +http_server_response_get_payload_output(struct http_server_response *resp, + size_t max_buffer_size, bool blocking) +{ + struct http_server_request *req = resp->request; + struct http_server_connection *conn = req->conn; + struct ostream *output; + + i_assert(conn != NULL); + i_assert(!resp->submitted); + i_assert(resp->payload_input == NULL); + i_assert(resp->payload_stream == NULL); + + output = http_server_ostream_create(resp, max_buffer_size, blocking); + o_stream_set_name(output, + t_strdup_printf("(conn %s: request %s: %u response payload)", + conn->conn.label, + http_server_request_label(req), resp->status)); + return output; +} + +void http_server_response_add_auth(struct http_server_response *resp, + const struct http_auth_challenge *chlng) +{ + struct http_auth_challenge *new; + pool_t pool = resp->request->pool; + + if (!array_is_created(&resp->auth_challenges)) + p_array_init(&resp->auth_challenges, pool, 4); + + new = array_append_space(&resp->auth_challenges); + http_auth_challenge_copy(pool, new, chlng); +} + +void http_server_response_add_auth_basic(struct http_server_response *resp, + const char *realm) +{ + struct http_auth_challenge chlng; + + http_auth_basic_challenge_init(&chlng, realm); + http_server_response_add_auth(resp, &chlng); +} + +static void +http_server_response_do_submit(struct http_server_response *resp) +{ + i_assert(!resp->submitted); + if (resp->date == (time_t)-1) + resp->date = ioloop_time; + resp->submitted = TRUE; + http_server_request_submit_response(resp->request); +} + +void http_server_response_submit(struct http_server_response *resp) +{ + e_debug(resp->event, "Submitted"); + + http_server_response_do_submit(resp); +} + +void http_server_response_submit_close(struct http_server_response *resp) +{ + http_server_request_connection_close(resp->request, TRUE); + http_server_response_submit(resp); +} + +void http_server_response_submit_tunnel(struct http_server_response *resp, + http_server_tunnel_callback_t callback, + void *context) +{ + e_debug(resp->event, "Started tunnelling"); + + resp->tunnel_callback = callback; + resp->tunnel_context = context; + http_server_request_connection_close(resp->request, TRUE); + http_server_response_do_submit(resp); +} + +static int +http_server_response_flush_payload(struct http_server_response *resp) +{ + struct http_server_request *req = resp->request; + struct http_server_connection *conn = req->conn; + int ret; + + if (resp->payload_output != conn->conn.output && + (ret = o_stream_finish(resp->payload_output)) <= 0) { + if (ret < 0) + http_server_connection_handle_output_error(conn); + else + http_server_connection_start_idle_timeout(conn); + return ret; + } + + return 1; +} + +void http_server_response_request_finished(struct http_server_response *resp) +{ + e_debug(resp->event, "Finished"); + + if (resp->payload_stream != NULL) + http_server_ostream_response_finished(resp->payload_stream); + + event_add_int(resp->request->event, "status_code", resp->status); +} + +int http_server_response_finish_payload_out(struct http_server_response *resp) +{ + struct http_server_request *req = resp->request; + struct http_server_connection *conn = req->conn; + int ret; + + if (req->state >= HTTP_SERVER_REQUEST_STATE_FINISHED) + return 1; + + resp->payload_finished = TRUE; + + if (resp->payload_output != NULL) { + ret = http_server_response_flush_payload(resp); + if (ret < 0) + return -1; + if (ret == 0) { + e_debug(resp->event, + "Not quite finished sending payload"); + return 0; + } + o_stream_unref(&resp->payload_output); + resp->payload_output = NULL; + } + + e_debug(resp->event, "Finished sending payload"); + + http_server_connection_ref(conn); + conn->output_locked = FALSE; + if (conn->conn.output != NULL && !conn->conn.output->closed) { + if (resp->payload_corked && + o_stream_uncork_flush(conn->conn.output) < 0) + http_server_connection_handle_output_error(conn); + o_stream_set_flush_callback(conn->conn.output, + http_server_connection_output, + conn); + } + + if (conn->request_queue_head == NULL || + (conn->request_queue_head->state != + HTTP_SERVER_REQUEST_STATE_PROCESSING)) + http_server_connection_start_idle_timeout(conn); + + http_server_request_finished(resp->request); + http_server_connection_unref(&conn); + return 1; +} + +static int +http_server_response_output_payload(struct http_server_response **_resp, + const unsigned char *data, size_t size) +{ + struct http_server_response *resp = *_resp; + struct http_server_request *req = resp->request; + struct ostream *output; + ssize_t sret; + int ret; + + i_assert(req->state < HTTP_SERVER_REQUEST_STATE_SUBMITTED_RESPONSE || + req->state == HTTP_SERVER_REQUEST_STATE_PAYLOAD_OUT); + + http_server_response_ref(resp); + + if (resp->payload_stream == NULL) { + output = http_server_response_get_payload_output( + resp, IO_BLOCK_SIZE, TRUE); + } else { + output = http_server_ostream_get_output(resp->payload_stream); + } + + if (data != NULL) { + if ((sret = o_stream_send(output, data, size)) < 0) { + *_resp = NULL; + o_stream_destroy(&output); + http_server_response_unref(&resp); + return -1; + } + i_assert((size_t)sret == size); + } else { + if ((ret = o_stream_finish(output)) < 0) { + *_resp = NULL; + o_stream_destroy(&output); + http_server_response_unref(&resp); + return -1; + } + i_assert(ret > 0); + } + + switch (req->state) { + case HTTP_SERVER_REQUEST_STATE_FINISHED: + ret = 1; + break; + case HTTP_SERVER_REQUEST_STATE_ABORTED: + e_debug(resp->event, + "Request aborted while sending blocking payload"); + ret = -1; + break; + default: + ret = 0; + break; + } + + if (data == NULL) + o_stream_destroy(&output); + + /* Callback may have messed with our pointer, so unref using local + variable */ + if (!http_server_response_unref(&resp)) + *_resp = NULL; + + /* Return status */ + return ret; +} + +int http_server_response_send_payload(struct http_server_response **_resp, + const unsigned char *data, size_t size) +{ + struct http_server_response *resp = *_resp; + int ret; + + resp->payload_corked = TRUE; + + i_assert(data != NULL); + + ret = http_server_response_output_payload(&resp, data, size); + if (ret < 0) + *_resp = NULL; + else { + i_assert(ret == 0); + i_assert(resp != NULL); + } + return ret; +} + +int http_server_response_finish_payload(struct http_server_response **_resp) +{ + struct http_server_response *resp = *_resp; + int ret; + + *_resp = NULL; + ret = http_server_response_output_payload(&resp, NULL, 0); + i_assert(ret != 0); + return ret < 0 ? -1 : 0; +} + +void http_server_response_abort_payload(struct http_server_response **_resp) +{ + struct http_server_response *resp = *_resp; + struct http_server_request *req = resp->request; + + *_resp = NULL; + + http_server_request_abort(&req, "Aborted sending response payload"); +} + +static void +http_server_response_payload_input(struct http_server_response *resp) +{ + struct http_server_connection *conn = resp->request->conn; + + io_remove(&conn->io_resp_payload); + + (void)http_server_connection_output(conn); +} + +int http_server_response_send_more(struct http_server_response *resp) +{ + struct http_server_connection *conn = resp->request->conn; + struct ostream *output = resp->payload_output; + enum ostream_send_istream_result res; + + i_assert(resp->payload_output != NULL); + + if (resp->payload_finished) { + e_debug(resp->event, "Finish sending payload (more)"); + return http_server_response_finish_payload_out(resp); + } + + if (resp->payload_stream != NULL) { + conn->output_locked = TRUE; + return http_server_ostream_continue(resp->payload_stream); + } + + i_assert(resp->payload_input != NULL); + io_remove(&conn->io_resp_payload); + + /* Chunked ostream needs to write to the parent stream's buffer */ + o_stream_set_max_buffer_size(output, IO_BLOCK_SIZE); + res = o_stream_send_istream(output, resp->payload_input); + o_stream_set_max_buffer_size(output, SIZE_MAX); + + switch (res) { + case OSTREAM_SEND_ISTREAM_RESULT_FINISHED: + /* Finished sending */ + if (!resp->payload_chunked && + (resp->payload_input->v_offset - resp->payload_offset) != + resp->payload_size) { + e_error(resp->event, + "Payload stream %s size changed unexpectedly", + i_stream_get_name(resp->payload_input)); + http_server_connection_close( + &conn, "Payload read failure"); + return -1; + } + /* Finished sending payload */ + e_debug(resp->event, "Finish sending payload"); + return http_server_response_finish_payload_out(resp); + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT: + /* Input is blocking (server needs to act; disable timeout) */ + conn->output_locked = TRUE; + http_server_connection_stop_idle_timeout(conn); + conn->io_resp_payload = io_add_istream(resp->payload_input, + http_server_response_payload_input, resp); + return 1; + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT: + /* Output is blocking (client needs to act; enable timeout) */ + conn->output_locked = TRUE; + http_server_connection_start_idle_timeout(conn); + o_stream_set_flush_pending(output, TRUE); + //e_debug(resp->event, "Partially sent payload"); + return 0; + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT: + /* We're in the middle of sending a response, so the connection + will also have to be aborted */ + e_error(resp->event, "read(%s) failed: %s", + i_stream_get_name(resp->payload_input), + i_stream_get_error(resp->payload_input)); + http_server_connection_close(&conn, + "Payload read failure"); + return -1; + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT: + /* Failed to send response */ + http_server_connection_handle_output_error(conn); + return -1; + } + i_unreached(); +} + +static int http_server_response_send_real(struct http_server_response *resp) +{ + struct http_server_request *req = resp->request; + struct http_server_connection *conn = req->conn; + string_t *rtext = t_str_new(256); + struct const_iovec iov[3]; + uoff_t content_length = 0; + bool chunked = FALSE, send_content_length = FALSE, close = FALSE; + bool is_head = http_request_method_is(&req->req, "HEAD"); + int ret; + + i_assert(!conn->output_locked); + + /* Determine response payload to send */ + if (resp->payload_input != NULL) { + i_assert(resp->tunnel_callback == NULL && + resp->status / 100 != 1 && + resp->status != 204 && resp->status != 304); + if (resp->payload_chunked) { + if (http_server_request_version_equals(req, 1, 0)) { + /* Connection close marks end of payload + */ + close = TRUE; + } else { + /* Input stream with unknown size */ + chunked = TRUE; + } + } else { + /* Send Content-Length if we have specified a payload, + even if it's 0 bytes. */ + content_length = resp->payload_size; + send_content_length = TRUE; + } + } else if (resp->payload_stream != NULL) { + /* HTTP payload output stream */ + if (!http_server_ostream_get_size(resp->payload_stream, + &content_length)) { + /* size not known at this point */ + chunked = TRUE; + } else { + /* output stream already finished, so data is + pre-buffered */ + send_content_length = TRUE; + } + } else if (resp->tunnel_callback == NULL && resp->status / 100 != 1 && + resp->status != 204 && resp->status != 304 && !is_head) { + /* RFC 7230, Section 3.3: Message Body + + Responses to the HEAD request method (Section 4.3.2 of + [RFC7231]) never include a message body because the + associated response header fields (e.g., Transfer-Encoding, + Content-Length, etc.), if present, indicate only what their + values would have been if the request method had been GET + (Section 4.3.1 of [RFC7231]). 2xx (Successful) responses to a + CONNECT request method (Section 4.3.6 of [RFC7231]) switch to + tunnel mode instead of having a message body. All 1xx + (Informational), 204 (No Content), and 304 (Not Modified) + responses do not include a message body. All other responses + do include a message body, although the body might be of zero + length. + + RFC 7230, Section 3.3.2: Content-Length + + A server MUST NOT send a Content-Length header field in any + 2xx (Successful) response to a CONNECT request (Section 4.3.6 + of [RFC7231]). + + -> Create empty body if it is missing. + */ + send_content_length = TRUE; + } + + /* Initialize output payload stream if needed */ + if (is_head) { + e_debug(resp->event, "A HEAD response has no payload"); + } else if (chunked) { + i_assert(resp->payload_input != NULL || + resp->payload_stream != NULL); + + e_debug(resp->event, "Will send payload in chunks"); + + resp->payload_output = + http_transfer_chunked_ostream_create(conn->conn.output); + } else if (send_content_length) { + i_assert(resp->payload_input != NULL || content_length == 0 || + resp->payload_stream != NULL); + + e_debug(resp->event, + "Will send payload with explicit size %"PRIuUOFF_T, + content_length); + + if (content_length > 0) { + resp->payload_output = conn->conn.output; + o_stream_ref(conn->conn.output); + } + } else if (close) { + i_assert(resp->payload_input != NULL); + + e_debug(resp->event, + "Will close connection after sending payload " + "(HTTP/1.0)"); + + resp->payload_output = conn->conn.output; + o_stream_ref(conn->conn.output); + } else { + e_debug(resp->event, "Response has no payload"); + } + + /* Create status line */ + str_append(rtext, "HTTP/1.1 "); + str_printfa(rtext, "%u", resp->status); + str_append(rtext, " "); + str_append(rtext, resp->reason); + + /* Create special headers implicitly if not set explicitly using + http_server_response_add_header() */ + if (!resp->have_hdr_date) { + str_append(rtext, "\r\nDate: "); + str_append(rtext, http_date_create(resp->date)); + str_append(rtext, "\r\n"); + } + if (array_is_created(&resp->auth_challenges)) { + str_append(rtext, "WWW-Authenticate: "); + http_auth_create_challenges(rtext, &resp->auth_challenges); + str_append(rtext, "\r\n"); + } + if (chunked) { + if (!resp->have_hdr_body_spec) + str_append(rtext, "Transfer-Encoding: chunked\r\n"); + } else if (send_content_length) { + if (!resp->have_hdr_body_spec) { + str_printfa(rtext, "Content-Length: %"PRIuUOFF_T"\r\n", + content_length); + } + } + if (!resp->have_hdr_connection) { + close = (close || req->req.connection_close || + req->connection_close || req->conn->input_broken); + if (close && resp->tunnel_callback == NULL) + str_append(rtext, "Connection: close\r\n"); + else if (http_server_request_version_equals(req, 1, 0)) + str_append(rtext, "Connection: Keep-Alive\r\n"); + } + + /* Status line + implicit headers */ + iov[0].iov_base = str_data(rtext); + iov[0].iov_len = str_len(rtext); + /* Explicit headers */ + iov[1].iov_base = str_data(resp->headers); + iov[1].iov_len = str_len(resp->headers); + /* End of header */ + iov[2].iov_base = "\r\n"; + iov[2].iov_len = 2; + + req->state = HTTP_SERVER_REQUEST_STATE_PAYLOAD_OUT; + o_stream_cork(conn->conn.output); + + if (o_stream_sendv(conn->conn.output, iov, N_ELEMENTS(iov)) < 0) { + http_server_connection_handle_output_error(conn); + return -1; + } + + e_debug(resp->event, "Sent header"); + + if (resp->payload_stream != NULL) + http_server_ostream_output_available(resp->payload_stream); + if (resp->payload_output != NULL) { + /* Non-blocking payload */ + ret = http_server_response_send_more(resp); + if (ret < 0) + return -1; + } else { + /* No payload to send */ + e_debug(resp->event, "No payload to send"); + if (resp->payload_stream != NULL) { + ret = http_server_ostream_continue(resp->payload_stream); + if (ret < 0) + return -1; + } + conn->output_locked = FALSE; + ret = http_server_response_finish_payload_out(resp); + if (ret < 0) + return -1; + } + + if (conn->conn.output != NULL && !resp->payload_corked && + o_stream_uncork_flush(conn->conn.output) < 0) { + http_server_connection_handle_output_error(conn); + return -1; + } + return ret; +} + +int http_server_response_send(struct http_server_response *resp) +{ + int ret; + + T_BEGIN { + ret = http_server_response_send_real(resp); + } T_END; + return ret; +} + +void http_server_response_get_status(struct http_server_response *resp, + int *status_r, const char **reason_r) +{ + i_assert(resp != NULL); + *status_r = resp->status; + *reason_r = resp->reason; +} + +uoff_t http_server_response_get_total_size(struct http_server_response *resp) +{ + i_assert(resp != NULL); + return resp->payload_size + str_len(resp->headers); +} + +void http_server_response_add_permanent_header(struct http_server_response *resp, + const char *key, const char *value) +{ + http_server_response_add_header(resp, key, value); + + if (!array_is_created(&resp->perm_headers)) + i_array_init(&resp->perm_headers, 4); + char *key_dup = i_strdup(key); + char *value_dup = i_strdup(value); + array_push_back(&resp->perm_headers, &key_dup); + array_push_back(&resp->perm_headers, &value_dup); +} diff --git a/src/lib-http/http-server.c b/src/lib-http/http-server.c new file mode 100644 index 0000000..dd89a16 --- /dev/null +++ b/src/lib-http/http-server.c @@ -0,0 +1,132 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "net.h" +#include "str.h" +#include "hash.h" +#include "array.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "connection.h" +#include "dns-lookup.h" +#include "iostream-rawlog.h" +#include "iostream-ssl.h" +#include "http-url.h" + +#include "http-server-private.h" + +static struct event_category event_category_http_server = { + .name = "http-server" +}; + +/* + * Server + */ + +struct http_server *http_server_init(const struct http_server_settings *set) +{ + struct http_server *server; + pool_t pool; + size_t pool_size; + + pool_size = (set->ssl != NULL) ? 10240 : 1024; /* ca/cert/key will be >8K */ + pool = pool_alloconly_create("http server", pool_size); + server = p_new(pool, struct http_server, 1); + server->pool = pool; + + if (set->default_host != NULL && *set->default_host != '\0') + server->set.default_host = p_strdup(pool, set->default_host); + if (set->rawlog_dir != NULL && *set->rawlog_dir != '\0') + server->set.rawlog_dir = p_strdup(pool, set->rawlog_dir); + if (set->ssl != NULL) { + server->set.ssl = + ssl_iostream_settings_dup(server->pool, set->ssl); + } + server->set.max_client_idle_time_msecs = set->max_client_idle_time_msecs; + server->set.max_pipelined_requests = + (set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1); + server->set.request_limits = set->request_limits; + server->set.socket_send_buffer_size = set->socket_send_buffer_size; + server->set.socket_recv_buffer_size = set->socket_recv_buffer_size; + server->set.debug = set->debug; + + server->event = event_create(set->event); + event_add_category(server->event, &event_category_http_server); + event_set_forced_debug(server->event, set->debug); + event_set_append_log_prefix(server->event, "http-server: "); + + server->conn_list = http_server_connection_list_init(); + + p_array_init(&server->resources, pool, 4); + p_array_init(&server->locations, pool, 4); + + return server; +} + +void http_server_deinit(struct http_server **_server) +{ + struct http_server *server = *_server; + struct http_server_resource *res; + + *_server = NULL; + + connection_list_deinit(&server->conn_list); + + array_foreach_elem(&server->resources, res) + http_server_resource_free(&res); + i_assert(array_count(&server->locations) == 0); + + if (server->ssl_ctx != NULL) + ssl_iostream_context_unref(&server->ssl_ctx); + event_unref(&server->event); + pool_unref(&server->pool); +} + +void http_server_switch_ioloop(struct http_server *server) +{ + struct connection *_conn = server->conn_list->connections; + + /* move connections */ + /* FIXME: we wouldn't necessarily need to switch all of them + immediately, only those that have requests now. but also connections + that get new requests before ioloop is switched again.. */ + for (; _conn != NULL; _conn = _conn->next) { + struct http_server_connection *conn = + (struct http_server_connection *)_conn; + + http_server_connection_switch_ioloop(conn); + } +} + +void http_server_shut_down(struct http_server *server) +{ + struct connection *_conn, *_next; + + server->shutting_down = TRUE; + + for (_conn = server->conn_list->connections; + _conn != NULL; _conn = _next) { + struct http_server_connection *conn = + (struct http_server_connection *)_conn; + + _next = _conn->next; + (void)http_server_connection_shut_down(conn); + } +} + +int http_server_init_ssl_ctx(struct http_server *server, const char **error_r) +{ + const char *error; + + if (server->set.ssl == NULL || server->ssl_ctx != NULL) + return 0; + + if (ssl_iostream_server_context_cache_get(server->set.ssl, + &server->ssl_ctx, &error) < 0) { + *error_r = t_strdup_printf("Couldn't initialize SSL context: %s", + error); + return -1; + } + return 0; +} diff --git a/src/lib-http/http-server.h b/src/lib-http/http-server.h new file mode 100644 index 0000000..48708a3 --- /dev/null +++ b/src/lib-http/http-server.h @@ -0,0 +1,427 @@ +#ifndef HTTP_SERVER_H +#define HTTP_SERVER_H + +#include "http-common.h" +#include "http-auth.h" +#include "http-request.h" + +struct istream; +struct ostream; + +struct http_request; + +struct http_server; +struct http_server_resource; +struct http_server_request; +struct http_server_response; + +/* + * Server settings + */ + +struct http_server_settings { + const char *default_host; + + const char *rawlog_dir; + + /* SSL settings; if NULL, master_service_ssl_init() is used instead */ + const struct ssl_iostream_settings *ssl; + + /* The maximum time in milliseconds a client is allowed to be idle + before it is disconnected. */ + unsigned int max_client_idle_time_msecs; + + /* Maximum number of pipelined requests per connection (default = 1) */ + unsigned int max_pipelined_requests; + + /* Request limits */ + struct http_request_limits request_limits; + + /* The kernel send/receive buffer sizes used for the connection sockets. + Configuring this is mainly useful for the test suite. The kernel + defaults are used when these settings are 0. */ + size_t socket_send_buffer_size; + size_t socket_recv_buffer_size; + + /* Event to use for the http server. */ + struct event *event; + + /* Enable logging debug messages */ + bool debug; +}; + +/* + * Response + */ + +/* Connection data for an established HTTP tunnel */ +struct http_server_tunnel { + int fd_in, fd_out; + struct istream *input; + struct ostream *output; +}; + +typedef void +(*http_server_tunnel_callback_t)(void *context, + const struct http_server_tunnel *tunnel); + +/* Start creating the response for the request. This function can be called + only once for each request. */ +struct http_server_response * +http_server_response_create(struct http_server_request *req, + unsigned int status, const char *reason); + +/* Reference a server response */ +void http_server_response_ref(struct http_server_response *resp); +/* Unreference a server response. Returns TRUE if there are still more + references, FALSE if not. */ +bool http_server_response_unref(struct http_server_response **_resp); + +/* Add a custom header to the response. This can override headers that are + otherwise created implicitly. */ +void http_server_response_add_header(struct http_server_response *resp, + const char *key, const char *value); +/* Add a header permanently to the response. Even if another response is + created for the request, this header is kept. */ +void http_server_response_add_permanent_header(struct http_server_response *resp, + const char *key, const char *value); +/* Change the response code and text, cannot be used after submission */ +void http_server_response_update_status(struct http_server_response *resp, + unsigned int status, const char *reason); +/* Set the value of the "Date" header for the response using a time_t value. + Use this instead of setting it directly using + http_server_response_add_header() */ +void http_server_response_set_date(struct http_server_response *resp, + time_t date); +/* Assign an input stream for the outgoing payload of this response. The input + stream is read asynchronously while the response is sent to the client. */ +void http_server_response_set_payload(struct http_server_response *resp, + struct istream *input); +/* Assign payload data to the response. The data is copied to the request pool. + If your data is already durably allocated during the existence of the + response, you should consider using http_server_response_set_payload() with + a data input stream instead. This will avoid copying the data unnecessarily. + */ +void http_server_response_set_payload_data(struct http_server_response *resp, + const unsigned char *data, + size_t size); + +/* Get an output stream for the outgoing payload of this response. The output + stream operates asynchronously when blocking is FALSE. In that case the + flush callback is called once more data can be sent. When blocking is TRUE, + writing to the stream will block until all data is sent. In every respect, + it operates very similar to a normal file output stream. The response is + submitted implicitly when the stream is first used; e.g., when it is written, + flushed, or o_stream_set_flush_pending(ostream, TRUE) is called. */ +struct ostream * +http_server_response_get_payload_output(struct http_server_response *resp, + size_t max_buffer_size, bool blocking); + +/* Get the status code and reason string currently set for this response. */ +void http_server_response_get_status(struct http_server_response *resp, + int *status_r, const char **reason_r); +/* Get the total size of the response when sent over the connection. */ +uoff_t http_server_response_get_total_size(struct http_server_response *resp); +/* Add authentication challenge to the response. */ +void http_server_response_add_auth(struct http_server_response *resp, + const struct http_auth_challenge *chlng); +/* Add "Basic" authentication challenge to the response. */ +void http_server_response_add_auth_basic(struct http_server_response *resp, + const char *realm); + +/* Submit the response. It is queued for transmission to the client. */ +void http_server_response_submit(struct http_server_response *resp); +/* Submit the response and close the connection once it is sent. */ +void http_server_response_submit_close(struct http_server_response *resp); +/* Submit the response and turn the connection it is sent across into a tunnel + once it is sent successfully. The callback is called once that happens. */ +void http_server_response_submit_tunnel(struct http_server_response *resp, + http_server_tunnel_callback_t callback, + void *context); + +/* Submits response and blocks until provided payload is sent. Multiple calls + are allowed; payload is sent in chunks this way. Payload transmission is + finished with http_server_response_finish_payload(). If the sending fails, + returns -1 and sets resp=NULL to indicate that the response was freed, + otherwise returns 0 and resp is unchanged. + + An often more convenient ostream wrapper API is available as + http_server_response_get_payload_output() with blocking=TRUE. + */ +int http_server_response_send_payload(struct http_server_response **resp, + const unsigned char *data, size_t size); +/* Finish sending the payload. Always frees resp and sets it to NULL. + Returns 0 on success, -1 on error. */ +int http_server_response_finish_payload(struct http_server_response **resp); +/* Abort response payload transmission prematurely. This closes the associated + connection */ +void http_server_response_abort_payload(struct http_server_response **resp); + +/* + * Request + */ + +/* Get the parsed HTTP request information for this request. */ +const struct http_request * +http_server_request_get(struct http_server_request *req); + +/* Reference a server request */ +void http_server_request_ref(struct http_server_request *req); +/* Unreference a server request. Returns TRUE if there are still more + references, FALSE if not. */ +bool http_server_request_unref(struct http_server_request **_req); + +/* Set flag that determines whether the connection is closed after the + request is handled. */ +void http_server_request_connection_close(struct http_server_request *req, + bool close); + +/* Get the pool for this request. */ +pool_t http_server_request_get_pool(struct http_server_request *req); +/* Returns the response created for the request with + http_server_response_create(), or NULL if none. */ +struct http_server_response * +http_server_request_get_response(struct http_server_request *req); +/* Returns TRUE if request is finished either because a response was sent + or because the request was aborted. */ +bool http_server_request_is_finished(struct http_server_request *req); + +/* Add a header to any HTTP response created for the HTTP request. */ +void http_server_request_add_response_header(struct http_server_request *req, + const char *key, const char *value); + +/* Return input stream for the request's payload. Optionally, this stream + can be made blocking. Do *NOT* meddle with the FD of the http_request + payload to achieve the same, because protocol violations will result. + */ +struct istream * +http_server_request_get_payload_input(struct http_server_request *req, + bool blocking); + +/* Forward the incoming request payload to the provided output stream in the + background. Calls the provided callback once the payload was forwarded + successfully. If forwarding fails, the client is presented with an + appropriate error. If the payload size exceeds max_size, the client will + get a 413 error. Before the callback finishes, the application must either + have added a reference to the request or have submitted a response. */ +void http_server_request_forward_payload(struct http_server_request *req, + struct ostream *output, + uoff_t max_size, + void (*callback)(void *), + void *context); +#define http_server_request_forward_payload(req, output, max_size, \ + callback, context) \ + http_server_request_forward_payload(req, output, max_size, \ + (void(*)(void*))callback, TRUE ? context : \ + CALLBACK_TYPECHECK(callback, void (*)(typeof(context)))) +/* Forward the incoming request payload to the provided buffer in the + background. Behaves identical to http_server_request_forward_payload() + otherwise. */ +void http_server_request_buffer_payload(struct http_server_request *req, + buffer_t *buffer, uoff_t max_size, + void (*callback)(void *), + void *context); +#define http_server_request_buffer_payload(req, buffer, max_size, \ + callback, context) \ + http_server_request_buffer_payload(req, buffer, max_size, \ + (void(*)(void*))callback, TRUE ? context : \ + CALLBACK_TYPECHECK(callback, void (*)(typeof(context)))) +/* Handle the incoming request payload by calling the callback each time + more data is available. Payload reading automatically finishes when the + request payload is fully read. Before the final callback finishes, the + application must either have added a reference to the request or have + submitted a response. */ +void http_server_request_handle_payload(struct http_server_request *req, + void (*callback)(void *context), + void *context); +#define http_server_request_handle_payload(req, callback, context) \ + http_server_request_handle_payload(req,\ + (void(*)(void*))callback, TRUE ? context : \ + CALLBACK_TYPECHECK(callback, void (*)(typeof(context)))) + +/* Get the authentication credentials provided in this request. Returns 0 if + the Authorization header is absent, returns -1 when that header cannot be + parsed, and returns 1 otherwise */ +int http_server_request_get_auth(struct http_server_request *req, + struct http_auth_credentials *credentials); + +/* Send a failure response for the request with given status/reason. */ +void http_server_request_fail(struct http_server_request *req, + unsigned int status, const char *reason); +/* Send a failure response for the request with given status/reason + and close the connection. */ +void http_server_request_fail_close(struct http_server_request *req, + unsigned int status, const char *reason); +/* Send a failure response for the request with given status/reason/text. + The text is sent as the response payload, if appropriate. */ +void http_server_request_fail_text(struct http_server_request *req, + unsigned int status, const char *reason, + const char *format, ...) ATTR_FORMAT(4, 5); +/* Send an authentication failure response for the request with given reason. + The provided challenge is set in the WWW-Authenticate header of the + response. */ +void http_server_request_fail_auth(struct http_server_request *req, + const char *reason, + const struct http_auth_challenge *chlng) + ATTR_NULL(2); +/* Send a authentication failure response for the request with given reason. + The provided realm is used to construct an Basic challenge in the + WWW-Authenticate header of the response. */ +void http_server_request_fail_auth_basic(struct http_server_request *req, + const char *reason, const char *realm) + ATTR_NULL(2); + +/* Send a 405 failure response for a request with an unknown method. The allow + parameter is the value used for the mandatory "Allow" header in the response. + */ +void http_server_request_fail_bad_method(struct http_server_request *req, + const char *allow); + +/* Call the specified callback when HTTP request is destroyed. This happens + after one of the following: + + a) Response and its payload is fully sent, + b) Response was submitted, but it couldn't be sent due to disconnection or + some other error, + c) http_server_deinit() was called and the request was aborted + + Note client disconnection before response is submitted isn't visible to this. + The request payload reading is the responsibility of the caller, which also + must handle the read errors by submitting a failure response. */ +void http_server_request_set_destroy_callback(struct http_server_request *req, + void (*callback)(void *), + void *context); +#define http_server_request_set_destroy_callback(req, callback, context) \ + http_server_request_set_destroy_callback( \ + req, (void(*)(void*))callback, \ + (TRUE ? context : \ + CALLBACK_TYPECHECK(callback, void (*)(typeof(context))))) + +/* + * Connection + */ + +/* Connection statistics */ +struct http_server_stats { + /* The number of requests received and responses sent */ + unsigned int request_count, response_count; + /* Bytes sent and received accross the connection */ + uoff_t input, output; +}; + +/* Connection callbacks */ +struct http_server_callbacks { + /* Handle the server request. All requests must be sent back a response. + The response is sent either with http_server_request_fail*() or + http_server_response_submit*(). For simple requests you can send the + response back immediately. If you can't do that, you'll need to + reference the request (or the request payload input stream). Then the + code flow usually goes like this: + + - http_server_request_set_destroy_callback(destroy_callback) + - http_server_request_ref() + - <do whatever is needed to handle the request> + - http_server_response_create() + - http_server_response_set_payload() can be used especially with + istream-callback to create a large response without temp files. + - http_server_response_submit() triggers the destroy_callback + after it has finished sending the response and its payload. + - In destroy_callback: http_server_request_unref() and any other + necessary cleanup - the request handling is now fully finished. + */ + void (*handle_request)(void *context, struct http_server_request *req); + void (*handle_connect_request)(void *context, + struct http_server_request *req, + struct http_url *target); + + /* Called once the connection is destroyed. */ + void (*connection_destroy)(void *context, const char *reason); +}; + +/* Create a HTTP server connection object for the provided fd pair. The + callbacks structure is described above. */ +struct http_server_connection * +http_server_connection_create(struct http_server *server, + int fd_in, int fd_out, bool ssl, + const struct http_server_callbacks *callbacks, + void *context); +/* Reference the connection */ +void http_server_connection_ref(struct http_server_connection *conn); +/* Dereference the connection. Returns FALSE if unrefing destroyed the + connection entirely */ +bool http_server_connection_unref(struct http_server_connection **_conn); +/* Dereference and close the connection. The provided reason is passed to the + connection_destroy() callback. */ +void http_server_connection_close(struct http_server_connection **_conn, + const char *reason); +/* Get the current statistics for this connection */ +const struct http_server_stats * +http_server_connection_get_stats(struct http_server_connection *conn); + +/* Switch connection to a specific ioloop. */ +struct ioloop * +http_server_connection_switch_ioloop_to(struct http_server_connection *conn, + struct ioloop *ioloop); +/* Switch connection to the current ioloop. */ +struct ioloop * +http_server_connection_switch_ioloop(struct http_server_connection *conn); + +/* + * Resource + */ + +typedef void +(http_server_resource_callback_t)(void *context, + struct http_server_request *req, + const char *sub_path); + +struct http_server_resource * +http_server_resource_create(struct http_server *server, pool_t pool, + http_server_resource_callback_t *callback, + void *context); +#define http_server_resource_create(server, pool, callback, context) \ + http_server_resource_create(server, pool, \ + (http_server_resource_callback_t *)callback, \ + (TRUE ? context : \ + CALLBACK_TYPECHECK(callback, void (*)( \ + typeof(context), struct http_server_request *req, \ + const char *sub_path)))) +/* Resources are freed upon http_server_deinit(), so calling + http_server_resource_free() is only necessary when the resource needs to + disappear somewhere in the middle of the server lifetime. */ +void http_server_resource_free(struct http_server_resource **_res); + +pool_t http_server_resource_get_pool(struct http_server_resource *res) + ATTR_PURE; +const char * +http_server_resource_get_path(struct http_server_resource *res) ATTR_PURE; +struct event * +http_server_resource_get_event(struct http_server_resource *res) ATTR_PURE; + +void http_server_resource_add_location(struct http_server_resource *res, + const char *path); + +/* Call the specified callback when HTTP resource is destroyed. */ +void http_server_resource_set_destroy_callback(struct http_server_resource *res, + void (*callback)(void *), + void *context); +#define http_server_resource_set_destroy_callback(req, callback, context) \ + http_server_resource_set_destroy_callback(req, \ + (void(*)(void*))callback, context - \ + CALLBACK_TYPECHECK(callback, void (*)(typeof(context)))) + +/* + * Server + */ + +struct http_server *http_server_init(const struct http_server_settings *set); +void http_server_deinit(struct http_server **_server); + +/* Shut down the server; accept no new requests and drop connections once + they become idle */ +void http_server_shut_down(struct http_server *server); + +/* Switch this server to the current ioloop */ +void http_server_switch_ioloop(struct http_server *server); + +#endif diff --git a/src/lib-http/http-transfer-chunked.c b/src/lib-http/http-transfer-chunked.c new file mode 100644 index 0000000..0e90992 --- /dev/null +++ b/src/lib-http/http-transfer-chunked.c @@ -0,0 +1,749 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream-private.h" +#include "ostream-private.h" +#include "http-parser.h" +#include "http-header-parser.h" + +#include "http-transfer.h" + +#define MIN_CHUNK_SIZE_WITH_EXTRA 6 + +/* + * Chunked input stream + */ + +enum http_transfer_chunked_parse_state { + HTTP_CHUNKED_PARSE_STATE_INIT, + HTTP_CHUNKED_PARSE_STATE_SIZE, + HTTP_CHUNKED_PARSE_STATE_EXT, + HTTP_CHUNKED_PARSE_STATE_EXT_NAME, + HTTP_CHUNKED_PARSE_STATE_EXT_EQ, + HTTP_CHUNKED_PARSE_STATE_EXT_VALUE, + HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_STRING, + HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_ESCAPE, + HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_TOKEN, + HTTP_CHUNKED_PARSE_STATE_CR, + HTTP_CHUNKED_PARSE_STATE_LF, + HTTP_CHUNKED_PARSE_STATE_DATA, + HTTP_CHUNKED_PARSE_STATE_DATA_READY, + HTTP_CHUNKED_PARSE_STATE_DATA_CR, + HTTP_CHUNKED_PARSE_STATE_DATA_LF, + HTTP_CHUNKED_PARSE_STATE_TRAILER, + HTTP_CHUNKED_PARSE_STATE_FINISHED, +}; + +struct http_transfer_chunked_istream { + struct istream_private istream; + struct stat statbuf; + + const unsigned char *begin, *cur, *end; + enum http_transfer_chunked_parse_state state; + unsigned int parsed_chars; + + uoff_t chunk_size, chunk_v_offset, chunk_pos; + uoff_t size, max_size; + + struct http_header_parser *header_parser; + + bool finished:1; +}; + +/* Chunk parser */ + +static inline const char *_chr_sanitize(unsigned char c) +{ + if (c >= 0x20 && c < 0x7F) + return t_strdup_printf("'%c'", c); + return t_strdup_printf("0x%02x", c); +} + +static int +http_transfer_chunked_parse_size(struct http_transfer_chunked_istream *tcstream) +{ + uoff_t size = 0, prev; + + /* chunk-size = 1*HEXDIG */ + + while (tcstream->cur < tcstream->end) { + prev = tcstream->chunk_size; + + if (*tcstream->cur >= '0' && *tcstream->cur <= '9') + size = *tcstream->cur-'0'; + else if (*tcstream->cur >= 'A' && *tcstream->cur <= 'F') + size = *tcstream->cur-'A' + 10; + else if (*tcstream->cur >= 'a' && *tcstream->cur <= 'f') + size = *tcstream->cur-'a' + 10; + else { + if (tcstream->parsed_chars == 0) { + io_stream_set_error( + &tcstream->istream.iostream, + "Expected chunk size digit, " + "but found %s", + _chr_sanitize(*tcstream->cur)); + return -1; + } + tcstream->parsed_chars = 0; + return 1; + } + tcstream->chunk_size <<= 4; + tcstream->chunk_size += size; + if (tcstream->chunk_size < prev) { + io_stream_set_error(&tcstream->istream.iostream, + "Chunk size exceeds integer limit"); + return -1; + } + tcstream->parsed_chars++; + tcstream->cur++; + } + + return 0; +} + +static int +http_transfer_chunked_skip_token(struct http_transfer_chunked_istream *tcstream) +{ + const unsigned char *first = tcstream->cur; + + /* token = 1*tchar */ + while (tcstream->cur < tcstream->end && + http_char_is_token(*tcstream->cur)) + tcstream->cur++; + + tcstream->parsed_chars += (tcstream->cur-first); + if (tcstream->cur == tcstream->end) + return 0; + if (tcstream->parsed_chars == 0) + return -1; + return 1; +} + +static int +http_transfer_chunked_skip_qdtext( + struct http_transfer_chunked_istream *tcstream) +{ + /* qdtext = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text */ + while (tcstream->cur < tcstream->end && + http_char_is_qdtext(*tcstream->cur)) + tcstream->cur++; + if (tcstream->cur == tcstream->end) + return 0; + return 1; +} + +static int +http_transfer_chunked_parse(struct http_transfer_chunked_istream *tcstream) +{ + int ret; + + /* RFC 7230, Section 4.1: Chunked Transfer Encoding + + chunked-body = *chunk + last-chunk + trailer-part + CRLF + + chunk = chunk-size [ chunk-ext ] CRLF + chunk-data CRLF + chunk-size = 1*HEXDIG + last-chunk = 1*("0") [ chunk-ext ] CRLF + + chunk-ext = *( ";" chunk-ext-name [ "=" chunk-ext-val ] ) + chunk-ext-name = token + chunk-ext-val = token / quoted-string + chunk-data = 1*OCTET ; a sequence of chunk-size octets + trailer-part = *( header-field CRLF ) + */ + + for (;;) { + switch (tcstream->state) { + case HTTP_CHUNKED_PARSE_STATE_INIT: + tcstream->chunk_size = 0; + tcstream->chunk_pos = 0; + tcstream->parsed_chars = 0; + tcstream->state = HTTP_CHUNKED_PARSE_STATE_SIZE; + /* fall through */ + case HTTP_CHUNKED_PARSE_STATE_SIZE: + ret = http_transfer_chunked_parse_size(tcstream); + if (ret <= 0) + return ret; + tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT; + /* fall through */ + case HTTP_CHUNKED_PARSE_STATE_EXT: + if (*tcstream->cur != ';') { + tcstream->state = HTTP_CHUNKED_PARSE_STATE_CR; + break; + } + /* chunk-ext */ + tcstream->cur++; + tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_NAME; + if (tcstream->cur >= tcstream->end) + return 0; + /* fall through */ + case HTTP_CHUNKED_PARSE_STATE_EXT_NAME: + /* chunk-ext-name = token */ + ret = http_transfer_chunked_skip_token(tcstream); + if (ret <= 0) { + if (ret < 0) { + io_stream_set_error( + &tcstream->istream.iostream, + "Invalid chunked extension name"); + } + return ret; + } + tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_EQ; + /* fall through */ + case HTTP_CHUNKED_PARSE_STATE_EXT_EQ: + if (*tcstream->cur != '=') { + tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT; + break; + } + tcstream->cur++; + tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE; + if (tcstream->cur >= tcstream->end) + return 0; + /* fall through */ + case HTTP_CHUNKED_PARSE_STATE_EXT_VALUE: + /* chunk-ext-val = token / quoted-string */ + if (*tcstream->cur != '"') { + tcstream->state = + HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_TOKEN; + break; + } + tcstream->cur++; + tcstream->state = + HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_STRING; + if (tcstream->cur >= tcstream->end) + return 0; + /* fall through */ + case HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_STRING: + if (*tcstream->cur == '"') { + tcstream->cur++; + tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT; + if (tcstream->cur >= tcstream->end) + return 0; + } else if ((ret = http_transfer_chunked_skip_qdtext(tcstream)) <= 0) { + if (ret < 0) { + io_stream_set_error( + &tcstream->istream.iostream, + "Invalid chunked extension value"); + } + return ret; + } else if (*tcstream->cur == '\\') { + tcstream->cur++; + tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_ESCAPE; + if (tcstream->cur >= tcstream->end) + return 0; + } else { + io_stream_set_error( + &tcstream->istream.iostream, + "Invalid character %s in chunked extension value string", + _chr_sanitize(*tcstream->cur)); + return -1; + } + break; + case HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_ESCAPE: + /* ( HTAB / SP / VCHAR / obs-text ) */ + if (!http_char_is_text(*tcstream->cur)) { + io_stream_set_error( + &tcstream->istream.iostream, + "Escaped invalid character %s in chunked extension value string", + _chr_sanitize(*tcstream->cur)); + return -1; + } + tcstream->state = + HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_STRING; + if (tcstream->cur >= tcstream->end) + return 0; + break; + case HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_TOKEN: + ret = http_transfer_chunked_skip_token(tcstream); + if (ret <= 0) { + if (ret < 0) { + io_stream_set_error( + &tcstream->istream.iostream, + "Invalid chunked extension value"); + } + return ret; + } + tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT; + break; + case HTTP_CHUNKED_PARSE_STATE_CR: + tcstream->state = HTTP_CHUNKED_PARSE_STATE_LF; + if (*tcstream->cur == '\r') { + tcstream->cur++; + if (tcstream->cur >= tcstream->end) + return 0; + } + /* fall through */ + case HTTP_CHUNKED_PARSE_STATE_LF: + if (*tcstream->cur != '\n') { + io_stream_set_error( + &tcstream->istream.iostream, + "Expected new line after chunk size, " + "but found %s", + _chr_sanitize(*tcstream->cur)); + return -1; + } + tcstream->cur++; + if (tcstream->chunk_size > 0) + tcstream->state = HTTP_CHUNKED_PARSE_STATE_DATA; + else + tcstream->state = HTTP_CHUNKED_PARSE_STATE_TRAILER; + return 1; + case HTTP_CHUNKED_PARSE_STATE_DATA_READY: + /* fall through */ + case HTTP_CHUNKED_PARSE_STATE_DATA_CR: + tcstream->state = HTTP_CHUNKED_PARSE_STATE_DATA_LF; + if (*tcstream->cur == '\r') { + tcstream->cur++; + if (tcstream->cur >= tcstream->end) + return 0; + } + /* fall through */ + case HTTP_CHUNKED_PARSE_STATE_DATA_LF: + if (*tcstream->cur != '\n') { + io_stream_set_error( + &tcstream->istream.iostream, + "Expected new line after chunk data, but found %s", + _chr_sanitize(*tcstream->cur)); + return -1; + } + tcstream->cur++; + tcstream->state = HTTP_CHUNKED_PARSE_STATE_INIT; + break; + default: + i_unreached(); + } + } + + i_unreached(); + return -1; +} + +static int +http_transfer_chunked_parse_next(struct http_transfer_chunked_istream *tcstream) +{ + struct istream_private *stream = &tcstream->istream; + struct istream *input = tcstream->istream.parent; + size_t size; + int ret; + + while ((ret = i_stream_read_more(input, &tcstream->begin, &size)) > 0) { + tcstream->cur = tcstream->begin; + tcstream->end = tcstream->cur + size; + + if ((ret = http_transfer_chunked_parse(tcstream)) < 0) { + stream->istream.stream_errno = EIO; + return -1; + } + + i_stream_skip(input, tcstream->cur - tcstream->begin); + + if (ret > 0) { + if (tcstream->state == HTTP_CHUNKED_PARSE_STATE_DATA) { + tcstream->chunk_v_offset = input->v_offset; + + tcstream->size += tcstream->chunk_size; + if (tcstream->max_size > 0 && + tcstream->size > tcstream->max_size) { + io_stream_set_error( + &tcstream->istream.iostream, + "Total chunked payload size exceeds maximum"); + stream->istream.stream_errno = EMSGSIZE; + return -1; + } + } + return ret; + } + } + + i_assert(ret != -2); + + if (ret < 0) { + if (stream->parent->eof && + stream->parent->stream_errno == 0) { + /* unexpected EOF */ + io_stream_set_error(&tcstream->istream.iostream, + "Unexpected end of payload"); + stream->istream.stream_errno = EIO; + } else { + /* parent stream error */ + stream->istream.stream_errno = + stream->parent->stream_errno; + } + } + return ret; +} + +/* Input stream */ + +static ssize_t +http_transfer_chunked_istream_read_data( + struct http_transfer_chunked_istream *tcstream) +{ + struct istream_private *stream = &tcstream->istream; + const unsigned char *data; + size_t size, avail; + ssize_t ret = 0; + + i_assert(tcstream->chunk_pos <= tcstream->chunk_size); + if (tcstream->chunk_pos == tcstream->chunk_size) { + tcstream->state = HTTP_CHUNKED_PARSE_STATE_DATA_READY; + return 0; + } + + // FIXME: is this even necessary? + i_stream_seek(stream->parent, + tcstream->chunk_v_offset + tcstream->chunk_pos); + + /* read from parent if necessary */ + data = i_stream_get_data(stream->parent, &size); + if (size == 0) { + ret = i_stream_read_memarea(stream->parent); + if (ret <= 0) { + i_assert(ret != -2); /* 0 sized buffer can't be full */ + if (stream->parent->eof && + stream->parent->stream_errno == 0) { + /* unexpected EOF */ + io_stream_set_error( + &tcstream->istream.iostream, + "Unexpected end of payload"); + stream->istream.stream_errno = EIO; + } else { + /* parent stream error */ + stream->istream.stream_errno = + stream->parent->stream_errno; + } + return ret; + } + data = i_stream_get_data(stream->parent, &size); + i_assert(size != 0); + } + + size = (size > (tcstream->chunk_size - tcstream->chunk_pos) ? + (tcstream->chunk_size - tcstream->chunk_pos) : size); + + /* Allocate buffer space */ + if (!i_stream_try_alloc(stream, size, &avail)) + return -2; + + /* Copy payload */ + size = size > avail ? avail : size; + memcpy(&stream->w_buffer[stream->pos], data, size); + + i_stream_skip(stream->parent, size); + + tcstream->chunk_pos += size; + i_assert(tcstream->chunk_pos <= tcstream->chunk_size); + if (tcstream->chunk_pos == tcstream->chunk_size) + tcstream->state = HTTP_CHUNKED_PARSE_STATE_DATA_READY; + + ret = size; + stream->pos = stream->pos+size; + return ret; +} + +static int +http_transfer_chunked_parse_trailer( + struct http_transfer_chunked_istream *tcstream) +{ + struct istream_private *stream = &tcstream->istream; + const char *field_name, *error; + const unsigned char *field_data; + size_t field_size; + int ret; + + if (tcstream->header_parser == NULL) { + /* NOTE: trailer is currently ignored */ + /* FIXME: limit trailer size */ + tcstream->header_parser = + http_header_parser_init(tcstream->istream.parent, + NULL, 0); + } + + while ((ret = http_header_parse_next_field(tcstream->header_parser, + &field_name, &field_data, + &field_size, &error)) > 0) { + if (field_name == NULL) + break; + } + + if (ret <= 0) { + if (ret < 0) { + io_stream_set_error( + &stream->iostream, + "Failed to parse chunked trailer: %s", error); + stream->istream.stream_errno = EIO; + } + return ret; + } + return 1; +} + +static ssize_t +http_transfer_chunked_istream_read(struct istream_private *stream) +{ + struct http_transfer_chunked_istream *tcstream = + (struct http_transfer_chunked_istream *)stream; + ssize_t ret = 0; + + for (;;) { + switch (tcstream->state) { + case HTTP_CHUNKED_PARSE_STATE_FINISHED: + tcstream->istream.istream.eof = TRUE; + return -1; + case HTTP_CHUNKED_PARSE_STATE_DATA: + ret = http_transfer_chunked_istream_read_data(tcstream); + if (ret != 0) + return ret; + if (tcstream->state != + HTTP_CHUNKED_PARSE_STATE_DATA_READY) + return 0; + break; + case HTTP_CHUNKED_PARSE_STATE_TRAILER: + ret = http_transfer_chunked_parse_trailer(tcstream); + if (ret <= 0) + return ret; + tcstream->state = HTTP_CHUNKED_PARSE_STATE_FINISHED; + tcstream->istream.istream.eof = TRUE; + return -1; + default: + ret = http_transfer_chunked_parse_next(tcstream); + if (ret <= 0) + return ret; + } + } + + return -1; +} + +static void +http_transfer_chunked_istream_destroy(struct iostream_private *stream) +{ + struct http_transfer_chunked_istream *tcstream = + (struct http_transfer_chunked_istream *)stream; + + if (tcstream->header_parser != NULL) + http_header_parser_deinit(&tcstream->header_parser); + + // FIXME: copied from istream.c; there's got to be a better way. + i_stream_free_buffer(&tcstream->istream); +} + +struct istream * +http_transfer_chunked_istream_create(struct istream *input, uoff_t max_size) +{ + struct http_transfer_chunked_istream *tcstream; + + tcstream = i_new(struct http_transfer_chunked_istream, 1); + tcstream->max_size = max_size; + + tcstream->istream.max_buffer_size = + input->real_stream->max_buffer_size; + + tcstream->istream.iostream.destroy = + http_transfer_chunked_istream_destroy; + tcstream->istream.read = http_transfer_chunked_istream_read; + + tcstream->istream.istream.readable_fd = FALSE; + tcstream->istream.istream.blocking = input->blocking; + tcstream->istream.istream.seekable = FALSE; + return i_stream_create(&tcstream->istream, input, + i_stream_get_fd(input), 0); +} + +/* + * Chunked output stream + */ + +// FIXME: provide support for corking the stream. This means that we'll have +// to buffer sent data here rather than in the parent steam; we need to know +// the size of the chunks before we can send them. + +#define DEFAULT_MAX_BUFFER_SIZE (1024*32) + +struct http_transfer_chunked_ostream { + struct ostream_private ostream; + + size_t chunk_size, chunk_pos; + + bool chunk_active:1; + bool sent_trailer:1; +}; + +static size_t _log16(size_t in) +{ + size_t res = 0; + + while (in > 0) { + in >>= 4; + res++; + } + return res; +} + +static size_t _max_chunk_size(size_t avail) +{ + size_t chunk_extra = 2*2; + + /* Make sure we have room for both chunk data and overhead + + chunk = chunk-size [ chunk-ext ] CRLF + chunk-data CRLF + chunk-size = 1*HEXDIG + */ + chunk_extra += _log16(avail); + return (avail < chunk_extra ? 0 : avail - chunk_extra); +} + +static int +http_transfer_chunked_ostream_send_trailer( + struct http_transfer_chunked_ostream *tcstream) +{ + struct ostream_private *stream = &tcstream->ostream; + ssize_t sent; + + if (tcstream->sent_trailer) + return 1; + + if (o_stream_get_buffer_avail_size(stream->parent) < 5) { + if (o_stream_flush_parent(stream) < 0) + return -1; + if (o_stream_get_buffer_avail_size(stream->parent) < 5) + return 0; + } + + sent = o_stream_send(tcstream->ostream.parent, "0\r\n\r\n", 5); + if (sent < 0) { + o_stream_copy_error_from_parent(stream); + return -1; + } + i_assert(sent == 5); + + tcstream->sent_trailer = TRUE; + return 1; +} + +static void +http_transfer_chunked_ostream_close(struct iostream_private *stream, + bool close_parent) +{ + struct http_transfer_chunked_ostream *tcstream = + (struct http_transfer_chunked_ostream *)stream; + + i_assert(tcstream->ostream.finished || + tcstream->ostream.ostream.stream_errno != 0 || + tcstream->ostream.error_handling_disabled); + if (close_parent) + o_stream_close(tcstream->ostream.parent); +} + +static int +http_transfer_chunked_ostream_flush(struct ostream_private *stream) +{ + struct http_transfer_chunked_ostream *tcstream = + (struct http_transfer_chunked_ostream *)stream; + int ret; + + if (stream->finished && + (ret = http_transfer_chunked_ostream_send_trailer(tcstream)) <= 0) + return ret; + + return o_stream_flush_parent(stream); +} + +static ssize_t +http_transfer_chunked_ostream_sendv(struct ostream_private *stream, + const struct const_iovec *iov, + unsigned int iov_count) +{ + struct http_transfer_chunked_ostream *tcstream = + (struct http_transfer_chunked_ostream *)stream; + struct const_iovec *iov_new; + unsigned int iov_count_new, i; + size_t bytes = 0, max_bytes; + ssize_t ret; + const char *prefix; + + i_assert(stream->parent->real_stream->max_buffer_size >= + MIN_CHUNK_SIZE_WITH_EXTRA); + + if ((ret = o_stream_flush(stream->parent)) <= 0) { + /* error / we still couldn't flush existing data to + parent stream. */ + if (ret < 0) + o_stream_copy_error_from_parent(stream); + return ret; + } + + /* check how many bytes we want to send */ + bytes = 0; + for (i = 0; i < iov_count; i++) + bytes += iov[i].iov_len; + + /* check if we have room to send at least one byte */ + max_bytes = o_stream_get_buffer_avail_size(stream->parent); + max_bytes = _max_chunk_size(max_bytes); + if (max_bytes < MIN_CHUNK_SIZE_WITH_EXTRA) + return 0; + + tcstream->chunk_size = (bytes > max_bytes ? max_bytes : bytes); + + /* determine what to send */ + bytes = tcstream->chunk_size; + iov_count_new = 1; + for (i = 0; i < iov_count && bytes > 0; i++) { + if (bytes <= iov[i].iov_len) + break; + bytes -= iov[i].iov_len; + iov_count_new++; + } + + /* create new iovec */ + prefix = t_strdup_printf("%llx\r\n", + (unsigned long long)tcstream->chunk_size); + iov_count = iov_count_new + 2; + iov_new = t_new(struct const_iovec, iov_count); + iov_new[0].iov_base = prefix; + iov_new[0].iov_len = strlen(prefix); + memcpy(&iov_new[1], iov, sizeof(struct const_iovec) * iov_count_new); + iov_new[iov_count-2].iov_len = bytes; + iov_new[iov_count-1].iov_base = "\r\n"; + iov_new[iov_count-1].iov_len = 2; + + /* send */ + if ((ret = o_stream_sendv(stream->parent, iov_new, iov_count)) <= 0) { + i_assert(ret < 0); + o_stream_copy_error_from_parent(stream); + return -1; + } + + /* all must be sent */ + i_assert((size_t)ret == (tcstream->chunk_size + iov_new[0].iov_len + + iov_new[iov_count-1].iov_len)); + + stream->ostream.offset += tcstream->chunk_size; + return tcstream->chunk_size; +} + +struct ostream * +http_transfer_chunked_ostream_create(struct ostream *output) +{ + struct http_transfer_chunked_ostream *tcstream; + size_t max_size; + + tcstream = i_new(struct http_transfer_chunked_ostream, 1); + tcstream->ostream.sendv = http_transfer_chunked_ostream_sendv; + tcstream->ostream.flush = http_transfer_chunked_ostream_flush; + tcstream->ostream.iostream.close = http_transfer_chunked_ostream_close; + if (output->real_stream->max_buffer_size > 0) + max_size = output->real_stream->max_buffer_size; + else + max_size = DEFAULT_MAX_BUFFER_SIZE; + + tcstream->ostream.max_buffer_size = _max_chunk_size(max_size); + return o_stream_create(&tcstream->ostream, output, + o_stream_get_fd(output)); +} diff --git a/src/lib-http/http-transfer.h b/src/lib-http/http-transfer.h new file mode 100644 index 0000000..f3f6791 --- /dev/null +++ b/src/lib-http/http-transfer.h @@ -0,0 +1,26 @@ +#ifndef HTTP_TRANSFER_H +#define HTTP_TRANSFER_H + +struct http_transfer_param { + const char *attribute; + const char *value; +}; +ARRAY_DEFINE_TYPE(http_transfer_param, struct http_transfer_param); + +struct http_transfer_coding { + const char *name; + ARRAY_TYPE(http_transfer_param) parameters; + +}; +ARRAY_DEFINE_TYPE(http_transfer_coding, struct http_transfer_coding); + + +// FIXME: we currently lack a means to get error strings from the input stream + +struct istream * +http_transfer_chunked_istream_create(struct istream *input, uoff_t max_size); +struct ostream * + http_transfer_chunked_ostream_create(struct ostream *output); + +#endif + diff --git a/src/lib-http/http-url.c b/src/lib-http/http-url.c new file mode 100644 index 0000000..229a58a --- /dev/null +++ b/src/lib-http/http-url.c @@ -0,0 +1,678 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "strfuncs.h" +#include "net.h" +#include "uri-util.h" + +#include "http-url.h" +#include "http-request.h" + +/* + * HTTP URL parser + */ + +struct http_url_parser { + struct uri_parser parser; + + enum http_url_parse_flags flags; + + struct http_url *url; + struct http_url *base; + + enum http_request_target_format req_format; + + bool relative:1; + bool request_target:1; +}; + +static bool http_url_parse_authority_form(struct http_url_parser *url_parser); + +static bool +http_url_parse_scheme(struct http_url_parser *url_parser, const char **scheme_r) +{ + struct uri_parser *parser = &url_parser->parser; + + *scheme_r = NULL; + if ((url_parser->flags & HTTP_URL_PARSE_SCHEME_EXTERNAL) != 0) + return TRUE; + + if (uri_parse_scheme(parser, scheme_r) <= 0) { + parser->cur = parser->begin; + return FALSE; + } + + return TRUE; +} + +static bool http_url_parse_unknown_scheme(struct http_url_parser *url_parser) +{ + struct uri_parser *parser = &url_parser->parser; + + if (url_parser->request_target) { + /* Valid as non-HTTP scheme, but also try to parse as authority + */ + parser->cur = parser->begin; + if (!http_url_parse_authority_form(url_parser)) { + /* indicate non-http-url */ + url_parser->url = NULL; + url_parser->req_format = + HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE; + } + return TRUE; + } + parser->error = "Not an HTTP URL"; + return FALSE; +} + +static bool +http_url_parse_userinfo(struct http_url_parser *url_parser, + struct uri_authority *auth, + const char **user_r, const char **password_r) +{ + struct uri_parser *parser = &url_parser->parser; + const char *p; + + *user_r = *password_r = NULL; + + if (auth->enc_userinfo == NULL) + return TRUE; + + if ((url_parser->flags & HTTP_URL_ALLOW_USERINFO_PART) == 0) { + /* RFC 7230, Section 2.7.1: http URI Scheme + + A sender MUST NOT generate the userinfo subcomponent (and its + "@" delimiter) when an "http" URI reference is generated + within a message as a request target or header field value. + Before making use of an "http" URI reference received from an + untrusted source, a recipient SHOULD parse for userinfo and + treat its presence as an error; it is likely being used to + obscure the authority for the sake of phishing attacks. + */ + parser->error = "HTTP URL does not allow `userinfo@' part"; + return FALSE; + } + + p = strchr(auth->enc_userinfo, ':'); + if (p == NULL) { + if (!uri_data_decode(parser, auth->enc_userinfo, NULL, user_r)) + return FALSE; + } else { + if (!uri_data_decode(parser, auth->enc_userinfo, p, user_r)) + return FALSE; + if (!uri_data_decode(parser, p + 1, NULL, password_r)) + return FALSE; + } + return TRUE; +} + +static bool http_url_parse_authority(struct http_url_parser *url_parser) +{ + struct uri_parser *parser = &url_parser->parser; + struct http_url *url = url_parser->url; + struct uri_authority auth; + const char *user = NULL, *password = NULL; + int ret; + + if ((ret = uri_parse_host_authority(parser, &auth)) < 0) + return FALSE; + if (auth.host.name == NULL || *auth.host.name == '\0') { + /* RFC 7230, Section 2.7.1: http URI Scheme + + A sender MUST NOT generate an "http" URI with an empty host + identifier. A recipient that processes such a URI reference + MUST reject it as invalid. + */ + parser->error = "HTTP URL does not allow empty host identifier"; + return FALSE; + } + if (ret > 0) { + if (!http_url_parse_userinfo(url_parser, &auth, + &user, &password)) + return FALSE; + } + if (url != NULL) { + uri_host_copy(parser->pool, &url->host, &auth.host); + url->port = auth.port; + url->user = p_strdup(parser->pool, user); + url->password = p_strdup(parser->pool, password); + } + return TRUE; +} + +static bool http_url_parse_authority_form(struct http_url_parser *url_parser) +{ + struct uri_parser *parser = &url_parser->parser; + + if (!http_url_parse_authority(url_parser)) + return FALSE; + if (parser->cur != parser->end) + return FALSE; + url_parser->req_format = HTTP_REQUEST_TARGET_FORMAT_AUTHORITY; + return TRUE; +} + +static int +http_url_parse_path(struct http_url_parser *url_parser) +{ + struct uri_parser *parser = &url_parser->parser; + struct http_url *url = url_parser->url, *base = url_parser->base; + const char *const *path; + int path_relative; + string_t *fullpath = NULL; + int ret; + + /* path-abempty / path-absolute / path-noscheme / path-empty */ + if ((ret = uri_parse_path(parser, &path_relative, &path)) < 0) + return -1; + + /* Resolve path */ + if (ret == 0) { + if (url_parser->relative && url != NULL) + url->path = p_strdup(parser->pool, base->path); + return 0; + } + + if (url != NULL) + fullpath = t_str_new(256); + + if (url_parser->relative && path_relative > 0 && base->path != NULL) { + const char *pbegin = base->path; + const char *pend = base->path + strlen(base->path); + const char *p = pend - 1; + + i_assert(*pbegin == '/'); + + /* Discard trailing segments of base path based on how many + effective leading '..' segments were found in the relative + path. + */ + while (path_relative > 0 && p > pbegin) { + while (p > pbegin && *p != '/') p--; + if (p >= pbegin) { + pend = p; + path_relative--; + } + if (p > pbegin) p--; + } + + if (url != NULL && pend > pbegin) + str_append_data(fullpath, pbegin, pend - pbegin); + } + + /* Append relative path */ + while (*path != NULL) { + const char *part; + + if (!uri_data_decode(parser, *path, NULL, &part)) + return -1; + + if (url != NULL) { + str_append_c(fullpath, '/'); + str_append(fullpath, part); + } + path++; + } + + if (url != NULL) + url->path = p_strdup(parser->pool, str_c(fullpath)); + return 1; +} + +static bool +http_url_parse_query(struct http_url_parser *url_parser, bool have_path) +{ + struct uri_parser *parser = &url_parser->parser; + struct http_url *url = url_parser->url, *base = url_parser->base; + const char *query; + int ret; + + if ((ret = uri_parse_query(parser, &query)) < 0) + return FALSE; + if (url == NULL) + return TRUE; + + if (ret > 0) + url->enc_query = p_strdup(parser->pool, query); + else if (url_parser->relative && !have_path) + url->enc_query = p_strdup(parser->pool, base->enc_query); + return TRUE; +} + +static bool +http_url_parse_fragment(struct http_url_parser *url_parser, bool have_path) +{ + struct uri_parser *parser = &url_parser->parser; + struct http_url *url = url_parser->url, *base = url_parser->base; + const char *fragment; + int ret; + + if ((ret = uri_parse_fragment(parser, &fragment)) < 0) + return FALSE; + if (ret > 0 && + (url_parser->flags & HTTP_URL_ALLOW_FRAGMENT_PART) == 0) { + parser->error = + "URL fragment not allowed for HTTP URL in this context"; + return FALSE; + } + if (url == NULL) + return TRUE; + + if (ret > 0) + url->enc_fragment = p_strdup(parser->pool, fragment); + else if (url_parser->relative && !have_path) + url->enc_fragment = p_strdup(parser->pool, base->enc_fragment); + return TRUE; +} + +static bool http_url_do_parse(struct http_url_parser *url_parser) +{ + struct uri_parser *parser = &url_parser->parser; + struct http_url *url = url_parser->url, *base = url_parser->base; + bool relative = TRUE, have_scheme = FALSE, have_authority = FALSE, + have_path = FALSE; + const char *scheme; + int ret; + + /* RFC 7230, Appendix B: + + http-URI = "http://" authority path-abempty [ "?" query ] + [ "#" fragment ] + https-URI = "https://" authority path-abempty [ "?" query ] + [ "#" fragment ] + partial-URI = relative-part [ "?" query ] + + request-target = origin-form / absolute-form / authority-form / + asterisk-form + + origin-form = absolute-path [ "?" query ] + absolute-form = absolute-URI + authority-form = authority + asterisk-form = "*" + ; Not parsed here + + absolute-path = 1*( "/" segment ) + + RFC 3986, Appendix A: (implemented in uri-util.h) + + absolute-URI = scheme ":" hier-part [ "?" query ] + + hier-part = "//" authority path-abempty + / path-absolute + / path-rootless + / path-empty + + relative-part = "//" authority path-abempty + / path-absolute + / path-noscheme + / path-empty + + authority = [ userinfo "@" ] host [ ":" port ] + + path-abempty = *( "/" segment ) + path-absolute = "/" [ segment-nz *( "/" segment ) ] + path-noscheme = segment-nz-nc *( "/" segment ) + path-rootless = segment-nz *( "/" segment ) + path-empty = 0<pchar> + + segment = *pchar + segment-nz = 1*pchar + segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" ) + ; non-zero-length segment without any colon ":" + + query = *( pchar / "/" / "?" ) + fragment = *( pchar / "/" / "?" ) + */ + + /* "http:" / "https:" */ + if (http_url_parse_scheme(url_parser, &scheme)) { + if (scheme == NULL) { + /* Scheme externally parsed */ + } else if (strcasecmp(scheme, "https") == 0) { + if (url != NULL) + url->have_ssl = TRUE; + } else if (strcasecmp(scheme, "http") != 0) { + return http_url_parse_unknown_scheme(url_parser); + } + + relative = FALSE; + have_scheme = TRUE; + } + + /* "//" authority ; or + * ["//"] authority ; when parsing a request target + */ + if (parser->cur < parser->end && parser->cur[0] == '/') { + if ((have_scheme || !url_parser->request_target) && + (parser->cur + 1) < parser->end && parser->cur[1] == '/') { + parser->cur += 2; + relative = FALSE; + have_authority = TRUE; + } else { + /* start of absolute-path */ + } + } else if (url_parser->request_target && !have_scheme) { + if (!http_url_parse_authority_form(url_parser)) { + /* not non-HTTP scheme and invalid as authority-form */ + parser->error = "Request target is invalid"; + return FALSE; + } + return TRUE; + } + + if (have_scheme && !have_authority) { + parser->error = "Absolute HTTP URL requires `//' after `http:'"; + return FALSE; + } + + if (have_authority) { + if (!http_url_parse_authority(url_parser)) + return FALSE; + } + + /* Relative URLs are only valid when we have a base URL */ + if (relative) { + if (base == NULL) { + parser->error = "Relative HTTP URL not allowed"; + return FALSE; + } else if (!have_authority && url != NULL) { + uri_host_copy(parser->pool, &url->host, &base->host); + url->port = base->port; + url->have_ssl = base->have_ssl; + url->user = p_strdup_empty(parser->pool, base->user); + url->password = p_strdup_empty(parser->pool, + base->password); + } + + url_parser->relative = TRUE; + } + + /* path-abempty / path-absolute / path-noscheme / path-empty */ + ret = http_url_parse_path(url_parser); + if (ret < 0) + return FALSE; + have_path = (ret > 0); + + /* [ "?" query ] */ + if (!http_url_parse_query(url_parser, have_path)) + return FALSE; + + /* [ "#" fragment ] */ + if (!http_url_parse_fragment(url_parser, have_path)) + return FALSE; + + /* must be at end of URL now */ + i_assert(parser->cur == parser->end); + + if (have_scheme) + url_parser->req_format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE; + return TRUE; +} + +/* Public API */ + +int http_url_parse(const char *url, struct http_url *base, + enum http_url_parse_flags flags, pool_t pool, + struct http_url **url_r, const char **error_r) +{ + struct http_url_parser url_parser; + + /* base != NULL indicates whether relative URLs are allowed. However, + certain flags may also dictate whether relative URLs are + allowed/required. */ + i_assert((flags & HTTP_URL_PARSE_SCHEME_EXTERNAL) == 0 || base == NULL); + + i_zero(&url_parser); + uri_parser_init(&url_parser.parser, pool, url); + url_parser.parser.allow_pct_nul = (flags & HTTP_URL_ALLOW_PCT_NUL) != 0; + + url_parser.url = p_new(pool, struct http_url, 1); + url_parser.base = base; + url_parser.flags = flags; + + if (!http_url_do_parse(&url_parser)) { + *error_r = url_parser.parser.error; + return -1; + } + *url_r = url_parser.url; + return 0; +} + +int http_url_request_target_parse(const char *request_target, + const char *host_header, + const struct http_url *default_base, + pool_t pool, + struct http_request_target *target, + const char **error_r) +{ + struct http_url_parser url_parser; + struct uri_authority auth; + struct http_url base; + + i_zero(&base); + if (host_header != NULL && *host_header != '\0') { + struct uri_parser *parser; + + i_zero(&url_parser); + parser = &url_parser.parser; + uri_parser_init(parser, pool, host_header); + + if (uri_parse_host_authority(parser, &auth) <= 0) { + *error_r = t_strdup_printf("Invalid Host header: %s", + parser->error); + return -1; + } + + if (parser->cur != parser->end || auth.enc_userinfo != NULL) { + *error_r = "Invalid Host header: " + "Contains invalid character"; + return -1; + } + + base.host = auth.host; + base.port = auth.port; + } else if (default_base == NULL) { + *error_r = "Empty Host header"; + return -1; + } else { + i_assert(default_base != NULL); + base = *default_base; + } + + if (request_target[0] == '*' && request_target[1] == '\0') { + struct http_url *url = p_new(pool, struct http_url, 1); + + uri_host_copy(pool, &url->host, &base.host); + url->port = base.port; + target->url = url; + target->format = HTTP_REQUEST_TARGET_FORMAT_ASTERISK; + return 0; + } + + i_zero(&url_parser); + uri_parser_init(&url_parser.parser, pool, request_target); + + url_parser.url = p_new(pool, struct http_url, 1); + url_parser.request_target = TRUE; + url_parser.req_format = HTTP_REQUEST_TARGET_FORMAT_ORIGIN; + url_parser.base = &base; + url_parser.flags = 0; + + if (!http_url_do_parse(&url_parser)) { + *error_r = url_parser.parser.error; + return -1; + } + + target->url = url_parser.url; + target->format = url_parser.req_format; + return 0; +} + +/* + * HTTP URL manipulation + */ + +void http_url_init_authority_from(struct http_url *dest, + const struct http_url *src) +{ + i_zero(dest); + dest->host = src->host; + dest->port = src->port; + dest->have_ssl = src->have_ssl; +} + +void http_url_copy_authority(pool_t pool, struct http_url *dest, + const struct http_url *src) +{ + i_zero(dest); + uri_host_copy(pool, &dest->host, &src->host); + dest->port = src->port; + dest->have_ssl = src->have_ssl; +} + +struct http_url * +http_url_clone_authority(pool_t pool, const struct http_url *src) +{ + struct http_url *new_url; + + new_url = p_new(pool, struct http_url, 1); + http_url_copy_authority(pool, new_url, src); + + return new_url; +} + +void http_url_copy(pool_t pool, struct http_url *dest, + const struct http_url *src) +{ + http_url_copy_authority(pool, dest, src); + dest->path = p_strdup(pool, src->path); + dest->enc_query = p_strdup(pool, src->enc_query); + dest->enc_fragment = p_strdup(pool, src->enc_fragment); +} + +void http_url_copy_with_userinfo(pool_t pool, struct http_url *dest, + const struct http_url *src) +{ + http_url_copy(pool, dest, src); + dest->user = p_strdup(pool, src->user); + dest->password = p_strdup(pool, src->password); +} + +struct http_url *http_url_clone(pool_t pool, const struct http_url *src) +{ + struct http_url *new_url; + + new_url = p_new(pool, struct http_url, 1); + http_url_copy(pool, new_url, src); + + return new_url; +} + +struct http_url * +http_url_clone_with_userinfo(pool_t pool, const struct http_url *src) +{ + struct http_url *new_url; + + new_url = p_new(pool, struct http_url, 1); + http_url_copy_with_userinfo(pool, new_url, src); + + return new_url; +} + +/* + * HTTP URL construction + */ + +static void +http_url_add_scheme(string_t *urlstr, const struct http_url *url) +{ + /* scheme */ + if (!url->have_ssl) + uri_append_scheme(urlstr, "http"); + else + uri_append_scheme(urlstr, "https"); + str_append(urlstr, "//"); +} + +static void +http_url_add_authority(string_t *urlstr, const struct http_url *url) +{ + /* host */ + uri_append_host(urlstr, &url->host); + /* port */ + uri_append_port(urlstr, url->port); +} + +static void +http_url_add_target(string_t *urlstr, const struct http_url *url) +{ + if (url->path == NULL || *url->path == '\0') { + /* Older syntax of RFC 2616 requires this slash at all times for + an absolute URL. */ + str_append_c(urlstr, '/'); + } else { + uri_append_path_data(urlstr, "", url->path); + } + + /* query (pre-encoded) */ + if (url->enc_query != NULL) { + str_append_c(urlstr, '?'); + str_append(urlstr, url->enc_query); + } +} + +const char *http_url_create(const struct http_url *url) +{ + string_t *urlstr = t_str_new(512); + + http_url_add_scheme(urlstr, url); + http_url_add_authority(urlstr, url); + http_url_add_target(urlstr, url); + + /* fragment */ + if (url->enc_fragment != NULL) { + str_append_c(urlstr, '#'); + str_append(urlstr, url->enc_fragment); + } + + return str_c(urlstr); +} + +const char *http_url_create_host(const struct http_url *url) +{ + string_t *urlstr = t_str_new(512); + + http_url_add_scheme(urlstr, url); + http_url_add_authority(urlstr, url); + + return str_c(urlstr); +} + +const char *http_url_create_authority(const struct http_url *url) +{ + string_t *urlstr = t_str_new(256); + + http_url_add_authority(urlstr, url); + + return str_c(urlstr); +} + +const char *http_url_create_target(const struct http_url *url) +{ + string_t *urlstr = t_str_new(256); + + http_url_add_target(urlstr, url); + + return str_c(urlstr); +} + +void http_url_escape_path(string_t *out, const char *data) +{ + uri_append_query_data(out, "&;?=+", data); +} + +void http_url_escape_param(string_t *out, const char *data) +{ + uri_append_query_data(out, "&;/?=+", data); +} diff --git a/src/lib-http/http-url.h b/src/lib-http/http-url.h new file mode 100644 index 0000000..62d8922 --- /dev/null +++ b/src/lib-http/http-url.h @@ -0,0 +1,108 @@ +#ifndef HTTP_URL_H +#define HTTP_URL_H + +#include "net.h" +#include "uri-util.h" + +#include "http-common.h" + +struct http_request_target; + +struct http_url { + /* server */ + struct uri_host host; + in_port_t port; + + /* userinfo (not parsed by default) */ + const char *user; + const char *password; + + /* path */ + const char *path; + + /* ?query (still encoded) */ + const char *enc_query; + + /* #fragment (still encoded) */ + const char *enc_fragment; + + bool have_ssl:1; +}; + +/* + * HTTP URL parsing + */ + +enum http_url_parse_flags { + /* Scheme part 'http:' is already parsed externally. This implies that + this is an absolute HTTP URL. */ + HTTP_URL_PARSE_SCHEME_EXTERNAL = 0x01, + /* Allow '#fragment' part in HTTP URL */ + HTTP_URL_ALLOW_FRAGMENT_PART = 0x02, + /* Allow 'user:password@' part in HTTP URL */ + HTTP_URL_ALLOW_USERINFO_PART = 0x04, + /* Allow URL to contain %00 */ + HTTP_URL_ALLOW_PCT_NUL = 0x08, +}; + +int http_url_parse(const char *url, struct http_url *base, + enum http_url_parse_flags flags, pool_t pool, + struct http_url **url_r, const char **error_r); + +int http_url_request_target_parse(const char *request_target, + const char *host_header, + const struct http_url *default_base, + pool_t pool, + struct http_request_target *target, + const char **error_r) ATTR_NULL(3); + +/* + * HTTP URL evaluation + */ + +static inline in_port_t +http_url_get_port_default(const struct http_url *url, in_port_t default_port) +{ + return (url->port != 0 ? url->port : default_port); +} + +static inline in_port_t http_url_get_port(const struct http_url *url) +{ + return http_url_get_port_default( + url, (url->have_ssl ? HTTPS_DEFAULT_PORT : HTTP_DEFAULT_PORT)); +} + +/* + * HTTP URL manipulation + */ + +void http_url_init_authority_from(struct http_url *dest, + const struct http_url *src); +void http_url_copy_authority(pool_t pool, struct http_url *dest, + const struct http_url *src); +struct http_url * +http_url_clone_authority(pool_t pool, const struct http_url *src); + +void http_url_copy(pool_t pool, struct http_url *dest, + const struct http_url *src); +void http_url_copy_with_userinfo(pool_t pool, struct http_url *dest, + const struct http_url *src); + +struct http_url *http_url_clone(pool_t pool,const struct http_url *src); +struct http_url * +http_url_clone_with_userinfo(pool_t pool, const struct http_url *src); + +/* + * HTTP URL construction + */ + +const char *http_url_create(const struct http_url *url); + +const char *http_url_create_host(const struct http_url *url); +const char *http_url_create_authority(const struct http_url *url); +const char *http_url_create_target(const struct http_url *url); + +void http_url_escape_path(string_t *out, const char *data); +void http_url_escape_param(string_t *out, const char *data); + +#endif diff --git a/src/lib-http/test-http-auth.c b/src/lib-http/test-http-auth.c new file mode 100644 index 0000000..49ac7ee --- /dev/null +++ b/src/lib-http/test-http-auth.c @@ -0,0 +1,274 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "test-common.h" +#include "array.h" +#include "str-sanitize.h" +#include "http-auth.h" + +struct http_auth_challenge_test { + const char *scheme; + const char *data; + struct http_auth_param *params; +}; + +struct http_auth_challenges_test { + const char *challenges_in; + + struct http_auth_challenge_test *challenges; +}; + +/* Valid auth challenges tests */ +static const struct http_auth_challenges_test +valid_auth_challenges_tests[] = { + { + .challenges_in = "Basic realm=\"WallyWorld\"", + .challenges = (struct http_auth_challenge_test []) { + { .scheme = "Basic", + .data = NULL, + .params = (struct http_auth_param []) { + { "realm", "WallyWorld" }, { NULL, NULL } + } + },{ + .scheme = NULL + } + } + },{ + .challenges_in = "Digest " + "realm=\"testrealm@host.com\", " + "qop=\"auth,auth-int\", " + "nonce=\"dcd98b7102dd2f0e8b11d0f600bfb0c093\", " + "opaque=\"5ccc069c403ebaf9f0171e9517f40e41\"", + .challenges = (struct http_auth_challenge_test []) { + { .scheme = "Digest", + .data = NULL, + .params = (struct http_auth_param []) { + { "realm", "testrealm@host.com" }, + { "qop", "auth,auth-int" }, + { "nonce", "dcd98b7102dd2f0e8b11d0f600bfb0c093" }, + { "opaque", "5ccc069c403ebaf9f0171e9517f40e41" }, + { NULL, NULL } + } + },{ + .scheme = NULL + } + } + },{ + .challenges_in = "Newauth realm=\"apps\", type=1, " + "title=\"Login to \\\"apps\\\"\", Basic realm=\"simple\"", + .challenges = (struct http_auth_challenge_test []) { + { .scheme = "Newauth", + .data = NULL, + .params = (struct http_auth_param []) { + { "realm", "apps" }, + { "type", "1" }, + { "title", "Login to \"apps\"" }, + { NULL, NULL } + } + },{ + .scheme = "Basic", + .data = NULL, + .params = (struct http_auth_param []) { + { "realm", "simple" }, + { NULL, NULL } + } + },{ + .scheme = NULL + } + } + } +}; + +static const unsigned int valid_auth_challenges_test_count = + N_ELEMENTS(valid_auth_challenges_tests); + +static void test_http_auth_challenges_valid(void) +{ + unsigned int i; + + for (i = 0; i < valid_auth_challenges_test_count; i++) T_BEGIN { + const char *challenges_in; + ARRAY_TYPE(http_auth_challenge) out; + const struct http_auth_challenges_test *test; + bool result; + + test = &valid_auth_challenges_tests[i]; + challenges_in = test->challenges_in; + + test_begin(t_strdup_printf("http auth challenges valid [%d]", i)); + + i_zero(&out); + result = (http_auth_parse_challenges + ((const unsigned char *)challenges_in, strlen(challenges_in), + &out) > 0); + test_out(t_strdup_printf("parse `%s'", challenges_in), result); + if (result) { + const struct http_auth_challenge *chalo; + const struct http_auth_challenge_test *chalt; + unsigned int index; + + index = 0; + chalt = test->challenges; + array_foreach(&out, chalo) { + const struct http_auth_param *paramo, *paramt; + unsigned int pindex; + + if (chalt != NULL && chalt->scheme != NULL) { + i_assert(chalo->scheme != NULL); + test_out(t_strdup_printf("[%d]->scheme = %s", + index, str_sanitize(chalo->scheme, 80)), + strcmp(chalo->scheme, chalt->scheme) == 0); + if (chalo->data == NULL || chalt->data == NULL) { + test_out(t_strdup_printf("[%d]->data = %s", + index, str_sanitize(chalo->data, 80)), + chalo->data == chalt->data); + } else { + test_out(t_strdup_printf("[%d]->data = %s", + index, str_sanitize(chalo->data, 80)), + strcmp(chalo->data, chalt->data) == 0); + } + paramt = chalt->params; + pindex = 0; + array_foreach(&chalo->params, paramo) { + if (paramt->name == NULL) { + test_out(t_strdup_printf("[%d]->params[%d]: %s = %s", + index, pindex, str_sanitize(paramo->name, 80), + str_sanitize(paramo->value, 80)), FALSE); + break; + } else { + test_out(t_strdup_printf("[%d]->params[%d]: %s = %s", + index, pindex, str_sanitize(paramo->name, 80), + str_sanitize(paramo->value, 80)), + strcmp(paramo->name, paramt->name) == 0 && + strcmp(paramo->value, paramt->value) == 0); + paramt++; + } + pindex++; + } + chalt++; + } + index++; + } + } + + test_end(); + } T_END; +} + +struct http_auth_credentials_test { + const char *credentials_in; + + const char *scheme; + const char *data; + struct http_auth_param *params; +}; + +/* Valid auth credentials tests */ +static const struct http_auth_credentials_test +valid_auth_credentials_tests[] = { + { + .credentials_in = "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==", + .scheme = "Basic", + .data = "QWxhZGRpbjpvcGVuIHNlc2FtZQ==", + .params = NULL + },{ + .credentials_in = "Digest username=\"Mufasa\", " + "realm=\"testrealm@host.com\", " + "nonce=\"dcd98b7102dd2f0e8b11d0f600bfb0c093\", " + "uri=\"/dir/index.html\", " + "qop=auth, " + "nc=00000001, " + "cnonce=\"0a4f113b\", " + "response=\"6629fae49393a05397450978507c4ef1\", " + "opaque=\"5ccc069c403ebaf9f0171e9517f40e41\"", + .scheme = "Digest", + .data = NULL, + .params = (struct http_auth_param []) { + { "username", "Mufasa" }, + { "realm", "testrealm@host.com" }, + { "nonce", "dcd98b7102dd2f0e8b11d0f600bfb0c093" }, + { "uri", "/dir/index.html" }, + { "qop", "auth" }, + { "nc", "00000001" }, + { "cnonce", "0a4f113b" }, + { "response", "6629fae49393a05397450978507c4ef1" }, + { "opaque", "5ccc069c403ebaf9f0171e9517f40e41" }, + { NULL, NULL } + } + } +}; + +static const unsigned int valid_auth_credentials_test_count = + N_ELEMENTS(valid_auth_credentials_tests); + +static void test_http_auth_credentials_valid(void) +{ + unsigned int i; + + for (i = 0; i < valid_auth_credentials_test_count; i++) T_BEGIN { + const char *credentials_in; + struct http_auth_credentials out; + const struct http_auth_credentials_test *test; + bool result; + + test = &valid_auth_credentials_tests[i]; + credentials_in = test->credentials_in; + + test_begin(t_strdup_printf("http auth credentials valid [%d]", i)); + + result = (http_auth_parse_credentials + ((const unsigned char *)credentials_in, strlen(credentials_in), + &out) > 0); + test_out(t_strdup_printf("parse `%s'", credentials_in), result); + if (result) { + const struct http_auth_param *paramo, *paramt; + unsigned int index; + + i_assert(out.scheme != NULL); + test_out(t_strdup_printf("->scheme = %s", + str_sanitize(out.scheme, 80)), + strcmp(out.scheme, test->scheme) == 0); + if (out.data == NULL || test->data == NULL) { + test_out(t_strdup_printf("->data = %s", + str_sanitize(out.data, 80)), + out.data == test->data); + } else { + test_out(t_strdup_printf("->data = %s", + str_sanitize(out.data, 80)), + strcmp(out.data, test->data) == 0); + } + paramt = test->params; + index = 0; + if (array_is_created(&out.params)) { + array_foreach(&out.params, paramo) { + if (paramt == NULL || paramt->name == NULL) { + test_out(t_strdup_printf("->params[%d]: %s = %s", + index++, str_sanitize(paramo->name, 80), + str_sanitize(paramo->value, 80)), FALSE); + break; + } else { + test_out(t_strdup_printf("->params[%d]: %s = %s", + index++, str_sanitize(paramo->name, 80), + str_sanitize(paramo->value, 80)), + strcmp(paramo->name, paramt->name) == 0 && + strcmp(paramo->value, paramt->value) == 0); + paramt++; + } + } + } + } + + test_end(); + } T_END; +} + + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_http_auth_challenges_valid, + test_http_auth_credentials_valid, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-http/test-http-client-errors.c b/src/lib-http/test-http-client-errors.c new file mode 100644 index 0000000..e127041 --- /dev/null +++ b/src/lib-http/test-http-client-errors.c @@ -0,0 +1,3944 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "str-sanitize.h" +#include "hostpid.h" +#include "ioloop.h" +#include "istream.h" +#include "istream-chain.h" +#include "ostream.h" +#include "time-util.h" +#include "sleep.h" +#include "connection.h" +#include "test-common.h" +#include "test-subprocess.h" +#include "http-url.h" +#include "http-request.h" +#include "http-client.h" + +#include <unistd.h> +#include <sys/signal.h> + +#define CLIENT_PROGRESS_TIMEOUT 10 +#define SERVER_KILL_TIMEOUT_SECS 20 + +static void main_deinit(void); + +/* + * Types + */ + +struct server_connection { + struct connection conn; + void *context; + + pool_t pool; + bool version_sent:1; +}; + +typedef void (*test_server_init_t)(unsigned int index); +typedef bool +(*test_client_init_t)(const struct http_client_settings *client_set); +typedef void (*test_dns_init_t)(void); + +/* + * State + */ + +/* common */ +static struct ip_addr bind_ip; +static in_port_t *bind_ports = 0; +static struct ioloop *ioloop; +static bool debug = FALSE; + +/* server */ +static struct io *io_listen; +static int fd_listen = -1; +static struct connection_list *server_conn_list; +static size_t server_read_max = 0; +static unsigned int server_index; +static int (*test_server_init)(struct server_connection *conn); +static void (*test_server_deinit)(struct server_connection *conn); +static void (*test_server_input)(struct server_connection *conn); + +/* client */ +static struct timeout *to_client_progress = NULL; +static struct http_client *http_client = NULL; + +/* + * Forward declarations + */ + +/* server */ +static void test_server_run(unsigned int index); +static void server_connection_deinit(struct server_connection **_conn); + +/* client */ +static void test_client_defaults(struct http_client_settings *http_set); +static void test_client_deinit(void); + +/* test*/ +static void +test_run_client_server(const struct http_client_settings *client_set, + test_client_init_t client_test, + test_server_init_t server_test, + unsigned int server_tests_count, + test_dns_init_t dns_test) ATTR_NULL(3); + +/* + * Utility + */ + +static void +test_client_assert_response(const struct http_response *resp, + bool condition) +{ + const char *reason = (resp->reason != NULL ? resp->reason : "<NULL>"); + + test_assert(resp->reason != NULL && *resp->reason != '\0'); + + if (!condition) + i_error("BAD RESPONSE: %u %s", resp->status, reason); + else if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); +} + +/* + * Unconfigured SSL + */ + +/* client */ + +struct _unconfigured_ssl { + unsigned int count; +}; + +static void +test_client_unconfigured_ssl_response(const struct http_response *resp, + struct _unconfigured_ssl *ctx) +{ + test_client_assert_response( + resp, resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_unconfigured_ssl(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _unconfigured_ssl *ctx; + + ctx = i_new(struct _unconfigured_ssl, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", "127.0.0.1", "/unconfigured-ssl.txt", + test_client_unconfigured_ssl_response, ctx); + http_client_request_set_ssl(hreq, TRUE); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", "127.0.0.1", "/unconfigured-ssl2.txt", + test_client_unconfigured_ssl_response, ctx); + http_client_request_set_ssl(hreq, TRUE); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_unconfigured_ssl(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("unconfigured ssl"); + test_run_client_server(&http_client_set, + test_client_unconfigured_ssl, NULL, 0, NULL); + test_end(); +} + +/* + * Unconfigured SSL abort + */ + +/* client */ + +struct _unconfigured_ssl_abort { + unsigned int count; +}; + +static void +test_client_unconfigured_ssl_abort_response1( + const struct http_response *resp, + struct _unconfigured_ssl_abort *ctx ATTR_UNUSED) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_out_quiet("inappropriate callback", FALSE); +} + +static void +test_client_unconfigured_ssl_abort_response2( + const struct http_response *resp, struct _unconfigured_ssl_abort *ctx) +{ + test_client_assert_response( + resp, resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED); + + i_free(ctx); + io_loop_stop(ioloop); +} + +static bool +test_client_unconfigured_ssl_abort( + const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _unconfigured_ssl_abort *ctx; + + ctx = i_new(struct _unconfigured_ssl_abort, 1); + ctx->count = 1; + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", "127.0.0.1", "/unconfigured-ssl.txt", + test_client_unconfigured_ssl_abort_response1, ctx); + http_client_request_set_ssl(hreq, TRUE); + http_client_request_submit(hreq); + http_client_request_abort(&hreq); + + hreq = http_client_request( + http_client, "GET", "127.0.0.1", "/unconfigured-ssl2.txt", + test_client_unconfigured_ssl_abort_response2, ctx); + http_client_request_set_ssl(hreq, TRUE); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_unconfigured_ssl_abort(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("unconfigured ssl abort"); + test_run_client_server(&http_client_set, + test_client_unconfigured_ssl_abort, + NULL, 0, NULL); + test_end(); +} + +/* + * Invalid URL + */ + +/* client */ + +struct _invalid_url { + unsigned int count; +}; + +static void +test_client_invalid_url_response(const struct http_response *resp, + struct _invalid_url *ctx) +{ + test_client_assert_response( + resp, resp->status == HTTP_CLIENT_REQUEST_ERROR_INVALID_URL); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_invalid_url(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _invalid_url *ctx; + + ctx = i_new(struct _invalid_url, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request_url_str( + http_client, "GET", "imap://example.com/INBOX", + test_client_invalid_url_response, ctx); + http_client_request_submit(hreq); + + hreq = http_client_request_url_str( + http_client, "GET", "http:/www.example.com", + test_client_invalid_url_response, ctx); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_invalid_url(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("invalid url"); + test_run_client_server(&http_client_set, + test_client_invalid_url, NULL, 0, NULL); + test_end(); +} + +/* + * Host lookup failed + */ + +/* client */ + +struct _host_lookup_failed { + unsigned int count; +}; + +static void +test_client_host_lookup_failed_response(const struct http_response *resp, + struct _host_lookup_failed *ctx) +{ + test_client_assert_response( + resp, + resp->status == HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_host_lookup_failed(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _host_lookup_failed *ctx; + + ctx = i_new(struct _host_lookup_failed, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", "host.in-addr.arpa", + "/host-lookup-failed.txt", + test_client_host_lookup_failed_response, ctx); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", "host.in-addr.arpa", + "/host-lookup-failed2.txt", + test_client_host_lookup_failed_response, ctx); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_host_lookup_failed(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("host lookup failed"); + test_run_client_server(&http_client_set, + test_client_host_lookup_failed, NULL, 0, NULL); + test_end(); +} + +/* + * Connection refused + */ + +/* server */ + +static void +test_server_connection_refused(unsigned int index ATTR_UNUSED) +{ + i_close_fd(&fd_listen); + + test_subprocess_notify_signal_send_parent(SIGUSR1); +} + +/* client */ + +struct _connection_refused { + unsigned int count; + struct timeout *to; +}; + +static void +test_client_connection_refused_response(const struct http_response *resp, + struct _connection_refused *ctx) +{ + test_assert(ctx->to == NULL); + timeout_remove(&ctx->to); + + test_client_assert_response( + resp, resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static void +test_client_connection_refused_timeout(struct _connection_refused *ctx) +{ + if (debug) + i_debug("TIMEOUT (ok)"); + timeout_remove(&ctx->to); +} + +static bool +test_client_connection_refused(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _connection_refused *ctx; + + /* wait for the server side to close the socket */ + test_subprocess_notify_signal_wait(SIGUSR1, 10000); + + ctx = i_new(struct _connection_refused, 1); + ctx->count = 2; + + if (client_set->max_connect_attempts > 0) { + ctx->to = timeout_add_short(250, + test_client_connection_refused_timeout, ctx); + } + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/connection-refused.txt", + test_client_connection_refused_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/connection-refused2.txt", + test_client_connection_refused_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_connection_refused(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("connection refused"); + test_subprocess_notify_signal_reset(SIGUSR1); + test_run_client_server(&http_client_set, + test_client_connection_refused, + test_server_connection_refused, 1, NULL); + test_end(); + + http_client_set.max_connect_attempts = 3; + + test_begin("connection refused backoff"); + test_subprocess_notify_signal_reset(SIGUSR1); + test_run_client_server(&http_client_set, + test_client_connection_refused, + test_server_connection_refused, 1, NULL); + test_end(); +} + +/* + * Connection lost prematurely + */ + +/* server */ + +static void +test_server_connection_lost_prematurely_input(struct server_connection *conn) +{ + server_connection_deinit(&conn); +} + +static void test_server_connection_lost_prematurely(unsigned int index) +{ + test_server_input = test_server_connection_lost_prematurely_input; + test_server_run(index); +} + +/* client */ + +struct _connection_lost_prematurely { + unsigned int count; + struct timeout *to; +}; + +static void +test_client_connection_lost_prematurely_response( + const struct http_response *resp, + struct _connection_lost_prematurely *ctx) +{ + test_assert(ctx->to == NULL); + timeout_remove(&ctx->to); + + test_client_assert_response( + resp, + resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static void +test_client_connection_lost_prematurely_timeout( + struct _connection_lost_prematurely *ctx) +{ + if (debug) + i_debug("TIMEOUT (ok)"); + timeout_remove(&ctx->to); +} + +static bool +test_client_connection_lost_prematurely( + const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _connection_lost_prematurely *ctx; + + ctx = i_new(struct _connection_lost_prematurely, 1); + ctx->count = 2; + + ctx->to = timeout_add_short( + 250, test_client_connection_lost_prematurely_timeout, ctx); + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/connection-refused-retry.txt", + test_client_connection_lost_prematurely_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/connection-refused-retry2.txt", + test_client_connection_lost_prematurely_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_connection_lost_prematurely(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.max_connect_attempts = 3; + http_client_set.max_attempts = 3; + + test_begin("connection lost prematurely"); + test_run_client_server(&http_client_set, + test_client_connection_lost_prematurely, + test_server_connection_lost_prematurely, 1, + NULL); + test_end(); +} + +/* + * Connection timed out + */ + +/* client */ + +struct _connection_timed_out { + unsigned int count; +}; + +static void +test_client_connection_timed_out_response(const struct http_response *resp, + struct _connection_timed_out *ctx) +{ + test_client_assert_response( + resp, resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_connection_timed_out(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _connection_timed_out *ctx; + + ctx = i_new(struct _connection_timed_out, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", "192.168.0.0", "/connection-timed-out.txt", + test_client_connection_timed_out_response, ctx); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", "192.168.0.0", "/connection-timed-out2.txt", + test_client_connection_timed_out_response, ctx); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_connection_timed_out(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.connect_timeout_msecs = 1000; + http_client_set.max_attempts = 1; + + test_begin("connection timed out"); + test_run_client_server(&http_client_set, + test_client_connection_timed_out, NULL, 0, NULL); + test_end(); +} + +/* + * Invalid redirect + */ + +/* server */ + +/* -> not accepted */ + +static void test_invalid_redirect_input1(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 302 Redirect\r\n" + "Location: http://localhost:4444\r\n" + "\r\n"; + + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_invalid_redirect1(unsigned int index) +{ + test_server_input = test_invalid_redirect_input1; + test_server_run(index); +} + +/* -> bad location */ + +static void test_invalid_redirect_input2(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 302 Redirect\r\n" + "Location: unix:/var/run/dovecot/auth-master\r\n" + "\r\n"; + + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_invalid_redirect2(unsigned int index) +{ + test_server_input = test_invalid_redirect_input2; + test_server_run(index); +} + +/* -> too many */ + +static void test_invalid_redirect_input3(struct server_connection *conn) +{ + string_t *resp; + + resp = t_str_new(512); + str_printfa(resp, + "HTTP/1.1 302 Redirect\r\n" + "Location: http://%s:%u/friep.txt\r\n" + "\r\n", + net_ip2addr(&bind_ip), bind_ports[server_index+1]); + o_stream_nsend(conn->conn.output, str_data(resp), str_len(resp)); + server_connection_deinit(&conn); +} + +static void test_server_invalid_redirect3(unsigned int index) +{ + test_server_input = test_invalid_redirect_input3; + test_server_run(index); +} + +/* client */ + +static void +test_client_invalid_redirect_response(const struct http_response *resp, + void *context ATTR_UNUSED) +{ + test_client_assert_response( + resp, + resp->status == HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT); + + io_loop_stop(ioloop); +} + +static bool +test_client_invalid_redirect(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/invalid-redirect.txt", + test_client_invalid_redirect_response, NULL); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_invalid_redirect(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("invalid redirect: not accepted"); + http_client_set.max_redirects = 0; + test_run_client_server(&http_client_set, + test_client_invalid_redirect, + test_server_invalid_redirect1, 1, NULL); + test_end(); + + test_begin("invalid redirect: bad location"); + http_client_set.max_redirects = 1; + test_run_client_server(&http_client_set, + test_client_invalid_redirect, + test_server_invalid_redirect2, 1, NULL); + test_end(); + + test_begin("invalid redirect: too many"); + http_client_set.max_redirects = 1; + test_run_client_server(&http_client_set, + test_client_invalid_redirect, + test_server_invalid_redirect3, 3, NULL); + test_end(); +} + +/* + * Unseekable redirect + */ + +/* server */ + +static void test_unseekable_redirect_input(struct server_connection *conn) +{ + string_t *resp; + + resp = t_str_new(512); + str_printfa(resp, + "HTTP/1.1 302 Redirect\r\n" + "Location: http://%s:%u/frml.txt\r\n" + "\r\n", + net_ip2addr(&bind_ip), bind_ports[server_index+1]); + o_stream_nsend(conn->conn.output, str_data(resp), str_len(resp)); + server_connection_deinit(&conn); +} + +static void test_server_unseekable_redirect(unsigned int index) +{ + test_server_input = test_unseekable_redirect_input; + test_server_run(index); +} + +/* client */ + +static void +test_client_unseekable_redirect_response(const struct http_response *resp, + void *context ATTR_UNUSED) +{ + test_client_assert_response( + resp, resp->status == HTTP_CLIENT_REQUEST_ERROR_ABORTED); + + io_loop_stop(ioloop); +} + +static bool +test_client_unseekable_redirect(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct istream *input; + + http_client = http_client_init(client_set); + + input = i_stream_create_from_data("FROP", 4); + input->seekable = FALSE; + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/unseekable-redirect.txt", + test_client_unseekable_redirect_response, NULL); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, FALSE); + http_client_request_submit(hreq); + + i_stream_unref(&input); + return TRUE; +} + +/* test */ + +static void test_unseekable_redirect(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.max_redirects = 1; + + test_begin("unseekable redirect"); + test_run_client_server(&http_client_set, + test_client_unseekable_redirect, + test_server_unseekable_redirect, 2, NULL); + test_end(); +} + +/* + * Unseekable retry + */ + +/* server */ + +static void test_unseekable_retry_input(struct server_connection *conn) +{ + server_connection_deinit(&conn); +} + +static void test_server_unseekable_retry(unsigned int index) +{ + test_server_input = test_unseekable_retry_input; + test_server_run(index); +} + +/* client */ + +static void +test_client_unseekable_retry_response(const struct http_response *resp, + void *context ATTR_UNUSED) +{ + test_client_assert_response( + resp, resp->status == HTTP_CLIENT_REQUEST_ERROR_ABORTED); + + io_loop_stop(ioloop); +} + +static bool +test_client_unseekable_retry(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct istream *input; + + http_client = http_client_init(client_set); + + input = i_stream_create_from_data("FROP", 4); + input->seekable = FALSE; + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/unseekable-retry.txt", + test_client_unseekable_retry_response, NULL); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, FALSE); + http_client_request_submit(hreq); + + i_stream_unref(&input); + return TRUE; +} + +/* test */ + +static void test_unseekable_retry(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.max_attempts = 3; + + test_begin("unseekable retry"); + test_run_client_server(&http_client_set, + test_client_unseekable_retry, + test_server_unseekable_retry, 2, NULL); + test_end(); +} + +/* + * Broken payload + */ + +/* server */ + +static void test_broken_payload_input(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 18\r\n" + "\r\n" + "Everything is OK\r\n"; + + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_broken_payload(unsigned int index) +{ + test_server_input = test_broken_payload_input; + test_server_run(index); +} + +/* client */ + +static void +test_client_broken_payload_response(const struct http_response *resp, + void *context ATTR_UNUSED) +{ + test_client_assert_response( + resp, resp->status == HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD); + + io_loop_stop(ioloop); +} + +static bool +test_client_broken_payload(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct istream *input; + + test_expect_errors(1); + + http_client = http_client_init(client_set); + + input = i_stream_create_error_str(EIO, "Moehahahaha!!"); + i_stream_set_name(input, "PURE EVIL"); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/broken-payload.txt", + test_client_broken_payload_response, NULL); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, FALSE); + http_client_request_submit(hreq); + + i_stream_unref(&input); + return TRUE; +} + +/* test */ + +static void test_broken_payload(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("broken payload"); + test_run_client_server(&http_client_set, + test_client_broken_payload, + test_server_broken_payload, 1, NULL); + test_end(); +} + +/* + * Retry payload + */ + +/* server */ + +struct _retry_payload_sctx { + bool eoh; +}; + +static int test_retry_payload_init(struct server_connection *conn) +{ + struct _retry_payload_sctx *ctx; + + ctx = p_new(conn->pool, struct _retry_payload_sctx, 1); + conn->context = ctx; + return 0; +} + +static void test_retry_payload_input(struct server_connection *conn) +{ + struct _retry_payload_sctx *ctx = conn->context; + const char *line; + + while ((line = i_stream_read_next_line(conn->conn.input)) != NULL) { + if (*line == '\0') { + ctx->eoh = TRUE; + continue; + } + if (ctx->eoh) + break; + } + + if (conn->conn.input->stream_errno != 0) { + i_fatal("server: Stream error: %s", + i_stream_get_error(conn->conn.input)); + } + if (line == NULL) { + if (conn->conn.input->eof) + i_fatal("server: Client stream ended prematurely"); + return; + } + + i_assert(ctx->eoh); + + if (strcmp(line, "This is the payload we expect.") == 0) { + if (debug) + i_debug("Expected payload received"); + o_stream_nsend_str(conn->conn.output, + "HTTP/1.1 500 Oh no!\r\n" + "Connection: close\r\n" + "Content-Length: 17\r\n" + "\r\n" + "Expected result\r\n"); + } else { + i_error("Unexpected payload received: `%s'", + str_sanitize(line, 128)); + o_stream_nsend_str(conn->conn.output, + "HTTP/1.1 501 Oh no!\r\n" + "Connection: close\r\n" + "Content-Length: 19\r\n" + "\r\n" + "Unexpected result\r\n"); + } + server_connection_deinit(&conn); +} + +static void test_server_retry_payload(unsigned int index) +{ + test_server_init = test_retry_payload_init; + test_server_input = test_retry_payload_input; + test_server_run(index); +} + +/* client */ + +struct _retry_payload_ctx { + unsigned int count; +}; + +struct _retry_payload_request_ctx { + struct _retry_payload_ctx *ctx; + struct http_client_request *req; +}; + +static void +test_client_retry_payload_response(const struct http_response *resp, + struct _retry_payload_request_ctx *rctx) +{ + struct _retry_payload_ctx *ctx = rctx->ctx; + + test_client_assert_response(resp, resp->status == 500); + + if (http_client_request_try_retry(rctx->req)) { + if (debug) + i_debug("retrying"); + return; + } + i_free(rctx); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_retry_payload(const struct http_client_settings *client_set) +{ + static const char payload[] = "This is the payload we expect.\r\n"; + struct _retry_payload_ctx *ctx; + struct _retry_payload_request_ctx *rctx; + struct http_client_request *hreq; + struct istream *input; + + ctx = i_new(struct _retry_payload_ctx, 1); + ctx->count = 1; + + http_client = http_client_init(client_set); + + input = i_stream_create_from_data(payload, sizeof(payload)-1); + + rctx = i_new(struct _retry_payload_request_ctx, 1); + rctx->ctx = ctx; + + rctx->req = hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), "/retry-payload.txt", + test_client_retry_payload_response, rctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, FALSE); + http_client_request_submit(hreq); + + i_stream_unref(&input); + return TRUE; +} + +/* test */ + +static void test_retry_payload(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.max_attempts = 2; + + server_read_max = 0; + + test_begin("retry payload"); + test_run_client_server(&http_client_set, + test_client_retry_payload, + test_server_retry_payload, 1, NULL); + test_end(); +} + +/* + * Connection lost + */ + +/* server */ + +static void test_connection_lost_input(struct server_connection *conn) +{ + ssize_t ret; + + if (server_read_max == 0) { + server_connection_deinit(&conn); + return; + } + + i_stream_set_max_buffer_size(conn->conn.input, server_read_max); + ret = i_stream_read(conn->conn.input); + if (ret == -2) { + server_connection_deinit(&conn); + return; + } + if (ret < 0) { + i_assert(conn->conn.input->eof); + if (conn->conn.input->stream_errno == 0) + i_fatal("server: Client stream ended prematurely"); + else + i_fatal("server: Stream error: %s", + i_stream_get_error(conn->conn.input)); + } +} + +static void test_server_connection_lost(unsigned int index) +{ + test_server_input = test_connection_lost_input; + test_server_run(index); +} + +/* client */ + +struct _connection_lost_ctx { + unsigned int count; +}; + +struct _connection_lost_request_ctx { + struct _connection_lost_ctx *ctx; + struct http_client_request *req; +}; + +static void +test_client_connection_lost_response(const struct http_response *resp, + struct _connection_lost_request_ctx *rctx) +{ + struct _connection_lost_ctx *ctx = rctx->ctx; + + test_client_assert_response( + resp, + resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST); + + if (http_client_request_try_retry(rctx->req)) { + if (debug) + i_debug("retrying"); + return; + } + i_free(rctx); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_connection_lost(const struct http_client_settings *client_set) +{ + static const char payload[] = + "This is a useless payload that only serves as a means to give " + "the server the opportunity to close the connection before the " + "payload is finished."; + struct _connection_lost_ctx *ctx; + struct _connection_lost_request_ctx *rctx; + struct http_client_request *hreq; + struct istream *input; + + ctx = i_new(struct _connection_lost_ctx, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + input = i_stream_create_from_data(payload, sizeof(payload)-1); + + rctx = i_new(struct _connection_lost_request_ctx, 1); + rctx->ctx = ctx; + + rctx->req = hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/connection-lost.txt", + test_client_connection_lost_response, rctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, FALSE); + http_client_request_submit(hreq); + + rctx = i_new(struct _connection_lost_request_ctx, 1); + rctx->ctx = ctx; + + rctx->req = hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/connection-lost2.txt", + test_client_connection_lost_response, rctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + i_stream_unref(&input); + return TRUE; +} + +/* test */ + +static void test_connection_lost(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + server_read_max = 0; + + test_begin("connection lost: one attempt"); + http_client_set.max_attempts = 1; + test_run_client_server(&http_client_set, + test_client_connection_lost, + test_server_connection_lost, 1, NULL); + test_end(); + + test_begin("connection lost: two attempts"); + http_client_set.max_attempts = 2; + test_run_client_server(&http_client_set, + test_client_connection_lost, + test_server_connection_lost, 1, NULL); + test_end(); + + test_begin("connection lost: three attempts"); + http_client_set.max_attempts = 3; + test_run_client_server(&http_client_set, + test_client_connection_lost, + test_server_connection_lost, 1, NULL); + test_end(); + + test_begin("connection lost: manual retry"); + http_client_set.max_attempts = 3; + http_client_set.no_auto_retry = TRUE; + test_run_client_server(&http_client_set, + test_client_connection_lost, + test_server_connection_lost, 1, NULL); + test_end(); +} + +/* + * Connection lost after 100-continue + */ + +/* server */ + +static void test_connection_lost_100_input(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 100 Continue\r\n" + "\r\n"; + + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_connection_lost_100(unsigned int index) +{ + test_server_input = test_connection_lost_100_input; + test_server_run(index); +} + +/* client */ + +struct _connection_lost_100_ctx { + unsigned int count; +}; + +static void +test_client_connection_lost_100_response(const struct http_response *resp, + struct _connection_lost_100_ctx *ctx) +{ + test_client_assert_response( + resp, + resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_connection_lost_100( + const struct http_client_settings *client_set) +{ + static const char payload[] = + "This is a useless payload that only serves as a means to give " + "the server the opportunity to close the connection before the " + "payload is finished."; + struct _connection_lost_100_ctx *ctx; + struct http_client_request *hreq; + struct istream *input; + + ctx = i_new(struct _connection_lost_100_ctx, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + input = i_stream_create_from_data(payload, sizeof(payload)-1); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/connection-lost.txt", + test_client_connection_lost_100_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, TRUE); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/connection-lost2.txt", + test_client_connection_lost_100_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, TRUE); + http_client_request_submit(hreq); + + i_stream_unref(&input); + return TRUE; +} + +/* test */ + +static void test_connection_lost_100(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + server_read_max = 0; + + test_begin("connection lost after 100-continue"); + http_client_set.max_attempts = 1; + test_run_client_server(&http_client_set, + test_client_connection_lost_100, + test_server_connection_lost_100, 1, NULL); + test_end(); +} + +/* + * Connection lost in sub-ioloop + */ + +/* server */ + +static void +test_connection_lost_sub_ioloop_input(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 0\r\n" + "\r\n"; + + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_connection_lost_sub_ioloop(unsigned int index) +{ + test_server_input = test_connection_lost_sub_ioloop_input; + test_server_run(index); +} + +/* client */ + +struct _connection_lost_sub_ioloop_ctx { + unsigned int count; +}; + +static void +test_client_connection_lost_sub_ioloop_response2( + const struct http_response *resp, struct ioloop *sub_ioloop) +{ + test_client_assert_response( + resp, + (resp->status == 200 || + resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST)); + + io_loop_stop(sub_ioloop); +} + +static void +test_client_connection_lost_sub_ioloop_response( + const struct http_response *resp, + struct _connection_lost_sub_ioloop_ctx *ctx) +{ + struct http_client_request *hreq; + struct ioloop *sub_ioloop; + + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_assert(resp->status == 200 || + resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + + sub_ioloop = io_loop_create(); + http_client_switch_ioloop(http_client); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/connection-lost-sub-ioloop3.txt", + test_client_connection_lost_sub_ioloop_response2, sub_ioloop); + http_client_request_set_port(hreq, bind_ports[1]); + http_client_request_submit(hreq); + + io_loop_run(sub_ioloop); + io_loop_set_current(ioloop); + http_client_switch_ioloop(http_client); + io_loop_set_current(sub_ioloop); + io_loop_destroy(&sub_ioloop); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_connection_lost_sub_ioloop( + const struct http_client_settings *client_set) +{ + static const char payload[] = + "This is a useless payload that only serves as a means to give " + "the server the opportunity to close the connection before the " + "payload is finished."; + struct _connection_lost_sub_ioloop_ctx *ctx; + struct http_client_request *hreq; + struct istream *input; + + ctx = i_new(struct _connection_lost_sub_ioloop_ctx, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + input = i_stream_create_from_data(payload, sizeof(payload)-1); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/connection-lost-sub-ioloop.txt", + test_client_connection_lost_sub_ioloop_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, TRUE); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/connection-lost-sub-ioloop2.txt", + test_client_connection_lost_sub_ioloop_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, TRUE); + http_client_request_submit(hreq); + + i_stream_unref(&input); + return TRUE; +} + +/* test */ + +static void test_connection_lost_sub_ioloop(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + server_read_max = 0; + + test_begin("connection lost while running sub-ioloop"); + http_client_set.max_attempts = 1; + test_run_client_server(&http_client_set, + test_client_connection_lost_sub_ioloop, + test_server_connection_lost_sub_ioloop, 2, NULL); + test_end(); +} + +/* + * Early success + */ + +/* server */ + +static void test_early_success_input(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 18\r\n" + "\r\n" + "Everything is OK\r\n"; + + i_sleep_msecs(200); + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_early_success(unsigned int index) +{ + test_server_input = test_early_success_input; + test_server_run(index); +} + +/* client */ + +struct _early_success_ctx { + unsigned int count; +}; + +static void +test_client_early_success_response(const struct http_response *resp, + struct _early_success_ctx *ctx) +{ + if (ctx->count == 2) { + test_client_assert_response( + resp, + resp->status == HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE); + } else { + test_client_assert_response(resp, resp->status == 200); + } + + if (--ctx->count == 0) { + io_loop_stop(ioloop); + i_free(ctx); + } +} + +static bool +test_client_early_success(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _early_success_ctx *ctx; + string_t *payload; + unsigned int i; + + ctx = i_new(struct _early_success_ctx, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/early-success.txt", + test_client_early_success_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + + T_BEGIN { + struct istream_chain *chain; + struct istream *input, *chain_input; + + payload = t_str_new(64*3000); + for (i = 0; i < 3000; i++) { + str_append(payload, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n"); + } + + chain_input = i_stream_create_chain(&chain, IO_BLOCK_SIZE); + + input = i_stream_create_copy_from_data(str_data(payload), + str_len(payload)); + i_stream_chain_append(chain, input); + i_stream_unref(&input); + + http_client_request_set_payload(hreq, chain_input, FALSE); + i_stream_unref(&chain_input); + } T_END; + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/early-success2.txt", + test_client_early_success_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_early_success(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.socket_send_buffer_size = 4096; + + test_begin("early succes"); + test_run_client_server(&http_client_set, + test_client_early_success, + test_server_early_success, 1, NULL); + test_end(); +} + +/* + * Bad response + */ + +/* server */ + +static void test_bad_response_input(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 666 Really bad response\r\n" + "\r\n"; + + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_bad_response(unsigned int index) +{ + test_server_input = test_bad_response_input; + test_server_run(index); +} + +/* client */ + +struct _bad_response_ctx { + unsigned int count; +}; + +static void +test_client_bad_response_response(const struct http_response *resp, + struct _bad_response_ctx *ctx) +{ + test_client_assert_response( + resp, resp->status == HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_bad_response(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _bad_response_ctx *ctx; + + ctx = i_new(struct _bad_response_ctx, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/bad-response.txt", + test_client_bad_response_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/bad-response2.txt", + test_client_bad_response_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_bad_response(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("bad response"); + test_run_client_server(&http_client_set, + test_client_bad_response, + test_server_bad_response, 1, NULL); + test_end(); +} + +/* + * Request timed out + */ + +/* server */ + +static void +test_request_timed_out_input(struct server_connection *conn ATTR_UNUSED) +{ + /* do nothing */ +} + +static void test_server_request_timed_out(unsigned int index) +{ + test_server_input = test_request_timed_out_input; + test_server_run(index); +} + +/* client */ + +struct _request_timed_out1_ctx { + unsigned int count; +}; + +static void +test_client_request_timed_out1_response(const struct http_response *resp, + struct _request_timed_out1_ctx *ctx) +{ + test_client_assert_response( + resp, resp->status == HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_request_timed_out1(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _request_timed_out1_ctx *ctx; + + ctx = i_new(struct _request_timed_out1_ctx, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/request-timed-out1-1.txt", + test_client_request_timed_out1_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/request-timed-out1-2.txt", + test_client_request_timed_out1_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + return TRUE; +} + +struct _request_timed_out2_ctx { + struct timeout *to; + unsigned int count; + unsigned int max_parallel_connections; +}; + +static void +test_client_request_timed_out2_timeout(struct _request_timed_out2_ctx *ctx) +{ + timeout_remove(&ctx->to); + i_debug("TIMEOUT"); +} + +static void +test_client_request_timed_out2_response(const struct http_response *resp, + struct _request_timed_out2_ctx *ctx) +{ + test_client_assert_response( + resp, resp->status == HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT); + test_assert(ctx->to != NULL); + + if (--ctx->count > 0) { + if (ctx->to != NULL && ctx->max_parallel_connections <= 1) + timeout_reset(ctx->to); + } else { + timeout_remove(&ctx->to); + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_request_timed_out2(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _request_timed_out2_ctx *ctx; + + ctx = i_new(struct _request_timed_out2_ctx, 1); + ctx->count = 2; + ctx->max_parallel_connections = + client_set->max_parallel_connections; + + ctx->to = timeout_add(2000, + test_client_request_timed_out2_timeout, ctx); + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/request-timed-out2-1.txt", + test_client_request_timed_out2_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_attempt_timeout_msecs(hreq, 1000); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/request-timed-out2-2.txt", + test_client_request_timed_out2_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_attempt_timeout_msecs(hreq, 1000); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_request_timed_out(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("request timed out: one attempt"); + http_client_set.request_timeout_msecs = 1000; + http_client_set.max_attempts = 1; + test_run_client_server(&http_client_set, + test_client_request_timed_out1, + test_server_request_timed_out, 1, NULL); + test_end(); + + test_begin("request timed out: two attempts"); + http_client_set.request_timeout_msecs = 1000; + http_client_set.max_attempts = 1; + test_run_client_server(&http_client_set, + test_client_request_timed_out1, + test_server_request_timed_out, 1, NULL); + test_end(); + + test_begin("request absolutely timed out"); + http_client_set.request_timeout_msecs = 0; + http_client_set.request_absolute_timeout_msecs = 2000; + http_client_set.max_attempts = 3; + test_run_client_server(&http_client_set, + test_client_request_timed_out1, + test_server_request_timed_out, 1, NULL); + test_end(); + + test_begin("request double timed out"); + http_client_set.request_timeout_msecs = 500; + http_client_set.request_absolute_timeout_msecs = 2000; + http_client_set.max_attempts = 3; + test_run_client_server(&http_client_set, + test_client_request_timed_out1, + test_server_request_timed_out, 1, NULL); + test_end(); + + test_begin("request timed out: specific timeout"); + http_client_set.request_timeout_msecs = 3000; + http_client_set.request_absolute_timeout_msecs = 0; + http_client_set.max_attempts = 1; + http_client_set.max_parallel_connections = 1; + test_run_client_server(&http_client_set, + test_client_request_timed_out2, + test_server_request_timed_out, 1, NULL); + test_end(); + + test_begin("request timed out: specific timeout (parallel)"); + http_client_set.request_timeout_msecs = 3000; + http_client_set.request_absolute_timeout_msecs = 0; + http_client_set.max_attempts = 1; + http_client_set.max_parallel_connections = 4; + test_run_client_server(&http_client_set, + test_client_request_timed_out2, + test_server_request_timed_out, 1, NULL); + test_end(); +} + +/* + * Request aborted early + */ + +/* server */ + +static void +test_request_aborted_early_input(struct server_connection *conn ATTR_UNUSED) +{ + static const char *resp = + "HTTP/1.1 404 Not Found\r\n" + "\r\n"; + + /* wait one second to respond */ + i_sleep_intr_secs(1); + + /* respond */ + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_request_aborted_early(unsigned int index) +{ + test_server_input = test_request_aborted_early_input; + test_server_run(index); +} + +/* client */ + +struct _request_aborted_early_ctx { + struct http_client_request *req1, *req2; + struct timeout *to; +}; + +static void +test_client_request_aborted_early_response( + const struct http_response *resp, + struct _request_aborted_early_ctx *ctx ATTR_UNUSED) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + /* abort does not trigger callback */ + test_assert(FALSE); +} + +static void +test_client_request_aborted_early_timeout( + struct _request_aborted_early_ctx *ctx) +{ + timeout_remove(&ctx->to); + + if (ctx->req1 != NULL) { + /* abort early */ + http_client_request_abort(&ctx->req1); /* sent */ + http_client_request_abort(&ctx->req2); /* only queued */ + + /* wait a little for server to actually respond to an + already aborted request */ + ctx->to = timeout_add_short( + 1000, test_client_request_aborted_early_timeout, ctx); + } else { + /* all done */ + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_request_aborted_early(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _request_aborted_early_ctx *ctx; + + ctx = i_new(struct _request_aborted_early_ctx, 1); + + http_client = http_client_init(client_set); + + hreq = ctx->req1 = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/request-aborted-early.txt", + test_client_request_aborted_early_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = ctx->req2 = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/request-aborted-early2.txt", + test_client_request_aborted_early_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + ctx->to = timeout_add_short( + 500, test_client_request_aborted_early_timeout, ctx); + return TRUE; +} + +/* test */ + +static void test_request_aborted_early(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("request aborted early"); + test_run_client_server(&http_client_set, + test_client_request_aborted_early, + test_server_request_aborted_early, 1, NULL); + test_end(); +} + +/* + * Request failed blocking + */ + +/* server */ + +static void +test_request_failed_blocking_input(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 500 Internal Server Error\r\n" + "\r\n"; + + /* respond */ + o_stream_nsend_str(conn->conn.output, resp); + i_sleep_intr_secs(10); + server_connection_deinit(&conn); +} + +static void test_server_request_failed_blocking(unsigned int index) +{ + test_server_input = test_request_failed_blocking_input; + test_server_run(index); +} + +/* client */ + +struct _request_failed_blocking_ctx { + struct http_client_request *req; +}; + +static void +test_client_request_failed_blocking_response( + const struct http_response *resp, + struct _request_failed_blocking_ctx *ctx ATTR_UNUSED) +{ + test_client_assert_response(resp, resp->status == 500); +} + +static bool +test_client_request_failed_blocking( + const struct http_client_settings *client_set) +{ + static const char *payload = "This a test payload!"; + struct http_client_request *hreq; + struct _request_failed_blocking_ctx *ctx; + unsigned int n; + string_t *data; + + data = str_new(default_pool, 1000000); + for (n = 0; n < 50000; n++) + str_append(data, payload); + + ctx = i_new(struct _request_failed_blocking_ctx, 1); + + http_client = http_client_init(client_set); + + hreq = ctx->req = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/request-failed-blocking.txt", + test_client_request_failed_blocking_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + + test_assert(http_client_request_send_payload(&hreq, + str_data(data), str_len(data)) < 0); + i_assert(hreq == NULL); + + str_free(&data); + i_free(ctx); + return FALSE; +} + +/* test */ + +static void test_request_failed_blocking(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.socket_send_buffer_size = 4096; + + test_begin("request failed blocking"); + test_run_client_server(&http_client_set, + test_client_request_failed_blocking, + test_server_request_failed_blocking, 1, NULL); + test_end(); +} + +/* + * Client deinit early + */ + +/* server */ + +static void +test_client_deinit_early_input(struct server_connection *conn ATTR_UNUSED) +{ + static const char *resp = + "HTTP/1.1 404 Not Found\r\n" + "\r\n"; + + /* wait one second to respond */ + i_sleep_intr_secs(1); + + /* respond */ + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_client_deinit_early(unsigned int index) +{ + test_server_input = test_client_deinit_early_input; + test_server_run(index); +} + +/* client */ + +struct _client_deinit_early_ctx { + struct timeout *to; +}; + +static void +test_client_client_deinit_early_response( + const struct http_response *resp, + struct _client_deinit_early_ctx *ctx ATTR_UNUSED) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + /* abort does not trigger callback */ + test_assert(FALSE); +} + +static void +test_client_client_deinit_early_timeout(struct _client_deinit_early_ctx *ctx) +{ + timeout_remove(&ctx->to); + + /* deinit early */ + http_client_deinit(&http_client); + + /* all done */ + i_free(ctx); + io_loop_stop(ioloop); +} + +static bool +test_client_client_deinit_early(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _client_deinit_early_ctx *ctx; + + ctx = i_new(struct _client_deinit_early_ctx, 1); + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/client-deinit-early.txt", + test_client_client_deinit_early_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/client-deinit-early2.txt", + test_client_client_deinit_early_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + ctx->to = timeout_add_short( + 500, test_client_client_deinit_early_timeout, ctx); + return TRUE; +} + +/* test */ + +static void test_client_deinit_early(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("client deinit early"); + test_run_client_server(&http_client_set, + test_client_client_deinit_early, + test_server_client_deinit_early, 1, NULL); + test_end(); +} + +/* + * Retry with delay + */ + +/* server */ + +static void test_retry_with_delay_input(struct server_connection *conn) +{ + string_t *resp; + + resp = t_str_new(512); + str_printfa(resp, + "HTTP/1.1 500 Internal Server Error\r\n" + "\r\n"); + o_stream_nsend(conn->conn.output, str_data(resp), str_len(resp)); + server_connection_deinit(&conn); +} + +static void test_server_retry_with_delay(unsigned int index) +{ + test_server_input = test_retry_with_delay_input; + test_server_run(index); +} + +/* client */ + +struct _client_retry_with_delay_ctx { + struct http_client_request *req; + unsigned int retries; + struct timeval time; +}; + +static void +test_client_retry_with_delay_response( + const struct http_response *resp, + struct _client_retry_with_delay_ctx *ctx) +{ + int real_delay, exp_delay; + + test_client_assert_response(resp, resp->status == 500); + + if (ctx->retries > 0) { + /* check delay */ + real_delay = timeval_diff_msecs(&ioloop_timeval, &ctx->time); + exp_delay = (1 << (ctx->retries-1)) * 50; + if (real_delay < exp_delay-2) { + i_fatal("Retry delay is too short %d < %d", + real_delay, exp_delay); + } + } + + http_client_request_delay_msecs(ctx->req, (1 << ctx->retries) * 50); + ctx->time = ioloop_timeval; + if (http_client_request_try_retry(ctx->req)) { + ctx->retries++; + if (debug) + i_debug("retrying"); + return; + } + + i_free(ctx); + io_loop_stop(ioloop); +} + +static bool +test_client_retry_with_delay(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _client_retry_with_delay_ctx *ctx; + + ctx = i_new(struct _client_retry_with_delay_ctx, 1); + ctx->time = ioloop_timeval; + + http_client = http_client_init(client_set); + + ctx->req = hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/retry-with-delay.txt", + test_client_retry_with_delay_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_retry_with_delay(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.max_attempts = 3; + + test_begin("retry with delay"); + test_run_client_server(&http_client_set, + test_client_retry_with_delay, + test_server_retry_with_delay, 1, NULL); + test_end(); +} + +/* + * DNS service failure + */ + +/* client */ + +struct _dns_service_failure { + unsigned int count; +}; + +static void +test_client_dns_service_failure_response( + const struct http_response *resp, + struct _dns_service_failure *ctx) +{ + test_client_assert_response( + resp, + resp->status == HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_dns_service_failure(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _dns_service_failure *ctx; + + ctx = i_new(struct _dns_service_failure, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", "example.com", "/dns-service-failure.txt", + test_client_dns_service_failure_response, ctx); + http_client_request_set_port(hreq, 80); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", "example.com", "/dns-service-failure2.txt", + test_client_dns_service_failure_response, ctx); + http_client_request_set_port(hreq, 80); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_dns_service_failure(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.dns_client_socket_path = "./frop"; + + test_begin("dns service failure"); + test_run_client_server(&http_client_set, + test_client_dns_service_failure, + NULL, 0, NULL); + test_end(); +} + +/* + * DNS timeout + */ + +/* dns */ + +static void test_dns_timeout_input(struct server_connection *conn ATTR_UNUSED) +{ + /* hang */ + i_sleep_intr_secs(100); + server_connection_deinit(&conn); +} + +static void test_dns_dns_timeout(void) +{ + test_server_input = test_dns_timeout_input; + test_server_run(0); +} + +/* client */ + +struct _dns_timeout { + unsigned int count; +}; + +static void +test_client_dns_timeout_response( + const struct http_response *resp, + struct _dns_timeout *ctx) +{ + test_client_assert_response( + resp, + resp->status == HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_dns_timeout(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _dns_timeout *ctx; + + ctx = i_new(struct _dns_timeout, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", "example.com", "/dns-timeout.txt", + test_client_dns_timeout_response, ctx); + http_client_request_set_port(hreq, 80); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", "example.com", "/dns-timeout2.txt", + test_client_dns_timeout_response, ctx); + http_client_request_set_port(hreq, 80); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_dns_timeout(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.request_timeout_msecs = 2000; + http_client_set.connect_timeout_msecs = 2000; + http_client_set.dns_client_socket_path = "./dns-test"; + + test_begin("dns timeout"); + test_run_client_server(&http_client_set, + test_client_dns_timeout, NULL, 0, + test_dns_dns_timeout); + test_end(); +} + +/* + * DNS lookup failure + */ + +/* dns */ + +static void +test_dns_lookup_failure_input(struct server_connection *conn) +{ + if (!conn->version_sent) { + conn->version_sent = TRUE; + o_stream_nsend_str(conn->conn.output, "VERSION\tdns\t1\t0\n"); + } + + o_stream_nsend_str(conn->conn.output, + t_strdup_printf("%d\tFAIL\n", EAI_FAIL)); + server_connection_deinit(&conn); +} + +static void test_dns_dns_lookup_failure(void) +{ + test_server_input = test_dns_lookup_failure_input; + test_server_run(0); +} + +/* client */ + +struct _dns_lookup_failure { + unsigned int count; +}; + +static void +test_client_dns_lookup_failure_response(const struct http_response *resp, + struct _dns_lookup_failure *ctx) +{ + test_client_assert_response( + resp, + resp->status == HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_dns_lookup_failure(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _dns_lookup_failure *ctx; + + ctx = i_new(struct _dns_lookup_failure, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", "example.com", "/dns-lookup-failure.txt", + test_client_dns_lookup_failure_response, ctx); + http_client_request_set_port(hreq, 80); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", "example.com", "/dns-lookup-failure2.txt", + test_client_dns_lookup_failure_response, ctx); + http_client_request_set_port(hreq, 80); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_dns_lookup_failure(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.dns_client_socket_path = "./dns-test"; + + test_begin("dns lookup failure"); + test_run_client_server(&http_client_set, + test_client_dns_lookup_failure, NULL, 0, + test_dns_dns_lookup_failure); + test_end(); +} + +/* + * DNS lookup ttl + */ + +/* dns */ + +static void +test_dns_lookup_ttl_input(struct server_connection *conn) +{ + static unsigned int count = 0; + const char *line; + + if (!conn->version_sent) { + conn->version_sent = TRUE; + o_stream_nsend_str(conn->conn.output, "VERSION\tdns\t1\t0\n"); + } + + while ((line = i_stream_read_next_line(conn->conn.input)) != NULL) { + if (str_begins(line, "VERSION")) + continue; + if (debug) + i_debug("DNS REQUEST %u: %s", count, line); + + if (count == 0) { + o_stream_nsend_str(conn->conn.output, + "0\t127.0.0.1\n"); + } else { + o_stream_nsend_str( + conn->conn.output, + t_strdup_printf("%d\tFAIL\n", EAI_FAIL)); + if (count > 4) { + server_connection_deinit(&conn); + return; + } + } + count++; + } +} + +static void test_dns_dns_lookup_ttl(void) +{ + test_server_input = test_dns_lookup_ttl_input; + test_server_run(0); +} + +/* server */ + +static void +test_server_dns_lookup_ttl_input(struct server_connection *conn) +{ + string_t *resp; + + resp = t_str_new(512); + str_printfa(resp, + "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "\r\n"); + o_stream_nsend(conn->conn.output, str_data(resp), str_len(resp)); + server_connection_deinit(&conn); +} + +static void test_server_dns_lookup_ttl(unsigned int index) +{ + test_server_input = test_server_dns_lookup_ttl_input; + test_server_run(index); +} + +/* client */ + +struct _dns_lookup_ttl { + struct http_client *client; + unsigned int count; + struct timeout *to; +}; + +static void +test_client_dns_lookup_ttl_response_stage2(const struct http_response *resp, + struct _dns_lookup_ttl *ctx) +{ + test_client_assert_response( + resp, + resp->status == HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static void test_client_dns_lookup_ttl_stage2_start(struct _dns_lookup_ttl *ctx) +{ + struct http_client_request *hreq; + + timeout_remove(&ctx->to); + + ctx->count = 2; + + hreq = http_client_request( + ctx->client, "GET", "example.com", + "/dns-lookup-ttl-stage2.txt", + test_client_dns_lookup_ttl_response_stage2, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request( + ctx->client, "GET", "example.com", + "/dns-lookup-ttl2-stage2.txt", + test_client_dns_lookup_ttl_response_stage2, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); +} + +static void +test_client_dns_lookup_ttl_response_stage1(const struct http_response *resp, + struct _dns_lookup_ttl *ctx) +{ + test_client_assert_response(resp, resp->status == 200); + + if (--ctx->count == 0) { + ctx->to = timeout_add(2000, + test_client_dns_lookup_ttl_stage2_start, ctx); + } +} + +static bool +test_client_dns_lookup_ttl(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _dns_lookup_ttl *ctx; + + ctx = i_new(struct _dns_lookup_ttl, 1); + ctx->count = 2; + + ctx->client = http_client = http_client_init(client_set); + + hreq = http_client_request( + ctx->client, "GET", "example.com", + "/dns-lookup-ttl-stage1.txt", + test_client_dns_lookup_ttl_response_stage1, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request( + ctx->client, "GET", "example.com", + "/dns-lookup-ttl2-stage1.txt", + test_client_dns_lookup_ttl_response_stage1, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_dns_lookup_ttl(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.dns_client_socket_path = "./dns-test"; + http_client_set.dns_ttl_msecs = 1000; + + test_begin("dns lookup ttl"); + test_run_client_server(&http_client_set, + test_client_dns_lookup_ttl, + test_server_dns_lookup_ttl, 1, + test_dns_dns_lookup_ttl); + test_end(); +} + +/* + * Peer reuse failure + */ + +/* server */ + +static void test_peer_reuse_failure_input(struct server_connection *conn) +{ + static unsigned int seq = 0; + static const char *resp = + "HTTP/1.1 200 OK\r\n" + "\r\n"; + + o_stream_nsend_str(conn->conn.output, resp); + if (seq++ > 2) { + server_connection_deinit(&conn); + io_loop_stop(current_ioloop); + } +} + +static void test_server_peer_reuse_failure(unsigned int index) +{ + test_server_input = test_peer_reuse_failure_input; + test_server_run(index); +} + +/* client */ + +struct _peer_reuse_failure { + struct timeout *to; + bool first:1; +}; + +static void +test_client_peer_reuse_failure_response2(const struct http_response *resp, + struct _peer_reuse_failure *ctx) +{ + test_client_assert_response( + resp, http_response_is_internal_error(resp)); + + i_free(ctx); + io_loop_stop(ioloop); +} + +static void +test_client_peer_reuse_failure_next(struct _peer_reuse_failure *ctx) +{ + struct http_client_request *hreq; + + timeout_remove(&ctx->to); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + "/peer-reuse-next.txt", + test_client_peer_reuse_failure_response2, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); +} + +static void +test_client_peer_reuse_failure_response1(const struct http_response *resp, + struct _peer_reuse_failure *ctx) +{ + if (ctx->first) { + test_client_assert_response(resp, resp->status == 200); + + ctx->first = FALSE; + ctx->to = timeout_add_short( + 500, test_client_peer_reuse_failure_next, ctx); + } else { + test_client_assert_response( + resp, http_response_is_internal_error(resp)); + } + + test_assert(resp->reason != NULL && *resp->reason != '\0'); +} + +static bool +test_client_peer_reuse_failure(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _peer_reuse_failure *ctx; + + ctx = i_new(struct _peer_reuse_failure, 1); + ctx->first = TRUE; + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), "/peer-reuse.txt", + test_client_peer_reuse_failure_response1, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), "/peer-reuse.txt", + test_client_peer_reuse_failure_response1, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), "/peer-reuse.txt", + test_client_peer_reuse_failure_response1, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_peer_reuse_failure(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.max_connect_attempts = 1; + http_client_set.max_idle_time_msecs = 500; + + test_begin("peer reuse failure"); + test_run_client_server(&http_client_set, + test_client_peer_reuse_failure, + test_server_peer_reuse_failure, 1, NULL); + test_end(); +} + +/* + * Reconnect failure + */ + +/* dns */ + +static void test_dns_reconnect_failure_input(struct server_connection *conn) +{ + static unsigned int count = 0; + const char *line; + + if (!conn->version_sent) { + conn->version_sent = TRUE; + o_stream_nsend_str(conn->conn.output, "VERSION\tdns\t1\t0\n"); + } + + while ((line = i_stream_read_next_line(conn->conn.input)) != NULL) { + if (str_begins(line, "VERSION")) + continue; + if (debug) + i_debug("DNS REQUEST %u: %s", count, line); + + if (count == 0) { + o_stream_nsend_str(conn->conn.output, + "0\t127.0.0.1\n"); + } else { + o_stream_nsend_str( + conn->conn.output, + t_strdup_printf("%d\tFAIL\n", EAI_FAIL)); + if (count > 4) { + server_connection_deinit(&conn); + return; + } + } + count++; + } +} + +static void test_dns_reconnect_failure(void) +{ + test_server_input = test_dns_reconnect_failure_input; + test_server_run(0); +} + +/* server */ + +static void test_reconnect_failure_input(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 18\r\n" + "\r\n" + "Everything is OK\r\n"; + + o_stream_nsend_str(conn->conn.output, resp); + io_loop_stop(current_ioloop); + io_remove(&io_listen); + i_close_fd(&fd_listen); + server_connection_deinit(&conn); +} + +static void test_server_reconnect_failure(unsigned int index) +{ + test_server_input = test_reconnect_failure_input; + test_server_run(index); +} + +/* client */ + +struct _reconnect_failure_ctx { + struct timeout *to; +}; + +static void +test_client_reconnect_failure_response2(const struct http_response *resp, + struct _reconnect_failure_ctx *ctx) +{ + test_client_assert_response( + resp, resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED); + + io_loop_stop(ioloop); + i_free(ctx); +} + +static void +test_client_reconnect_failure_next(struct _reconnect_failure_ctx *ctx) +{ + struct http_client_request *hreq; + + if (debug) + i_debug("NEXT REQUEST"); + + timeout_remove(&ctx->to); + + hreq = http_client_request( + http_client, "GET", "example.com", "/reconnect-failure-2.txt", + test_client_reconnect_failure_response2, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); +} + +static void +test_client_reconnect_failure_response1(const struct http_response *resp, + struct _reconnect_failure_ctx *ctx) +{ + test_client_assert_response(resp, resp->status == 200); + + ctx->to = timeout_add_short( + 5000, test_client_reconnect_failure_next, ctx); +} + +static bool +test_client_reconnect_failure(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _reconnect_failure_ctx *ctx; + + ctx = i_new(struct _reconnect_failure_ctx, 1); + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", "example.com", "/reconnect-failure-1.txt", + test_client_reconnect_failure_response1, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_reconnect_failure(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.dns_client_socket_path = "./dns-test"; + http_client_set.dns_ttl_msecs = 10000; + http_client_set.max_idle_time_msecs = 1000; + http_client_set.max_attempts = 1; + http_client_set.request_timeout_msecs = 1000; + + test_begin("reconnect failure"); + test_run_client_server(&http_client_set, + test_client_reconnect_failure, + test_server_reconnect_failure, 1, + test_dns_reconnect_failure); + test_end(); +} + +/* + * Multi IP attempts + */ + +/* dns */ + +static void test_multi_ip_attempts_input(struct server_connection *conn) +{ + unsigned int count = 0; + const char *line; + + if (!conn->version_sent) { + conn->version_sent = TRUE; + o_stream_nsend_str(conn->conn.output, "VERSION\tdns\t1\t0\n"); + } + + while ((line = i_stream_read_next_line(conn->conn.input)) != NULL) { + if (str_begins(line, "VERSION")) + continue; + if (debug) + i_debug("DNS REQUEST %u: %s", count, line); + + if (strcmp(line, "IP\ttest1.local") == 0) { + o_stream_nsend_str(conn->conn.output, + "0\t127.0.0.4\t127.0.0.3\t" + "127.0.0.2\t127.0.0.1\n"); + continue; + } + + o_stream_nsend_str(conn->conn.output, + "0\t10.255.255.1\t192.168.0.0\t" + "192.168.255.255\t127.0.0.1\n"); + } +} + +static void test_dns_multi_ip_attempts(void) +{ + test_server_input = test_multi_ip_attempts_input; + test_server_run(0); +} + +/* server */ + +static void test_server_multi_ip_attempts_input(struct server_connection *conn) +{ + string_t *resp; + + resp = t_str_new(512); + str_printfa(resp, + "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "\r\n"); + o_stream_nsend(conn->conn.output, str_data(resp), str_len(resp)); + server_connection_deinit(&conn); +} + +static void test_server_multi_ip_attempts(unsigned int index) +{ + test_server_input = test_server_multi_ip_attempts_input; + test_server_run(index); +} + +/* client */ + +struct _multi_ip_attempts { + unsigned int count; +}; + +static void +test_client_multi_ip_attempts_response(const struct http_response *resp, + struct _multi_ip_attempts *ctx) +{ + test_client_assert_response(resp, resp->status == 200); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static bool +test_client_multi_ip_attempts1(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _multi_ip_attempts *ctx; + + ctx = i_new(struct _multi_ip_attempts, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", "test1.local", "/multi-ip-attempts.txt", + test_client_multi_ip_attempts_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", "test1.local", "/multi-ip-attempts2.txt", + test_client_multi_ip_attempts_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + return TRUE; +} + +static bool +test_client_multi_ip_attempts2(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _multi_ip_attempts *ctx; + + ctx = i_new(struct _multi_ip_attempts, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request( + http_client, "GET", "test2.local", "/multi-ip-attempts.txt", + test_client_multi_ip_attempts_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request( + http_client, "GET", "test2.local", "/multi-ip-attempts2.txt", + test_client_multi_ip_attempts_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_multi_ip_attempts(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.connect_timeout_msecs = 1000; + http_client_set.request_timeout_msecs = 1000; + http_client_set.dns_client_socket_path = "./dns-test"; + http_client_set.max_connect_attempts = 4; + + test_begin("multi IP attempts (connection refused)"); + test_run_client_server(&http_client_set, + test_client_multi_ip_attempts1, + test_server_multi_ip_attempts, 1, + test_dns_multi_ip_attempts); + test_end(); + + test_begin("multi IP attempts (connect timeout)"); + test_run_client_server(&http_client_set, + test_client_multi_ip_attempts2, + test_server_multi_ip_attempts, 1, + test_dns_multi_ip_attempts); + test_end(); + + http_client_set.soft_connect_timeout_msecs = 100; + + test_begin("multi IP attempts (soft connect timeout)"); + test_run_client_server(&http_client_set, + test_client_multi_ip_attempts2, + test_server_multi_ip_attempts, 1, + test_dns_multi_ip_attempts); + test_end(); +} + +/* + * Idle connections + */ + +/* server */ + +struct _idle_connections_sctx { + bool eoh; +}; + +static int test_idle_connections_init(struct server_connection *conn) +{ + struct _idle_connections_sctx *ctx; + + ctx = p_new(conn->pool, struct _idle_connections_sctx, 1); + conn->context = ctx; + return 0; +} + +static void test_idle_connections_input(struct server_connection *conn) +{ + struct _idle_connections_sctx *ctx = conn->context; + const char *line; + + while ((line = i_stream_read_next_line(conn->conn.input)) != NULL) { + if (*line == '\0') { + ctx->eoh = TRUE; + break; + } + } + + if (conn->conn.input->stream_errno != 0) { + i_fatal("server: Stream error: %s", + i_stream_get_error(conn->conn.input)); + } + if (line == NULL) { + if (conn->conn.input->eof) + server_connection_deinit(&conn); + return; + } + + i_assert(ctx->eoh); + ctx->eoh = FALSE; + + string_t *resp; + + resp = t_str_new(512); + str_printfa(resp, + "HTTP/1.1 200 OK\r\n" + "Content-Length: 0\r\n" + "\r\n"); + o_stream_nsend(conn->conn.output, str_data(resp), str_len(resp)); + if (o_stream_flush(conn->conn.output) < 0) { + i_fatal("server: Flush error: %s", + o_stream_get_error(conn->conn.output)); + } +} + +static void test_server_idle_connections(unsigned int index) +{ + test_server_init = test_idle_connections_init; + test_server_input = test_idle_connections_input; + test_server_run(index); +} + +/* client */ + +struct _idle_connections { + struct http_client *client; + unsigned int max, count; + struct timeout *to; +}; + +static void +test_client_idle_connections_response_stage2(const struct http_response *resp, + struct _idle_connections *ctx) +{ + test_client_assert_response( + resp, resp->status == 200); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static void test_client_idle_connections_stage2_start(struct _idle_connections *ctx) +{ + struct http_client_request *hreq; + unsigned int i; + + if (debug) + i_debug("STAGE 2"); + + timeout_remove(&ctx->to); + + ctx->count = ctx->max; + + for (i = 0; i < ctx->count; i++) { + hreq = http_client_request( + ctx->client, "GET", net_ip2addr(&bind_ip), + t_strdup_printf("/idle-connections-stage2-%d.txt", i), + test_client_idle_connections_response_stage2, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + } +} + +static void +test_client_idle_connections_response_stage1(const struct http_response *resp, + struct _idle_connections *ctx) +{ + test_client_assert_response(resp, resp->status == 200); + + if (--ctx->count == 0) { + if (debug) + i_debug("START STAGE 2"); + ctx->to = timeout_add_short( + 550, test_client_idle_connections_stage2_start, ctx); + } +} + +static bool +test_client_idle_connections(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _idle_connections *ctx; + unsigned int i; + + if (debug) + i_debug("STAGE 1"); + + ctx = i_new(struct _idle_connections, 1); + ctx->max = client_set->max_parallel_connections; + ctx->count = client_set->max_parallel_connections; + + ctx->client = http_client = http_client_init(client_set); + + for (i = 0; i < ctx->count; i++) { + hreq = http_client_request( + ctx->client, "GET", net_ip2addr(&bind_ip), + t_strdup_printf("/idle-connections-stage1-%d.txt", i), + test_client_idle_connections_response_stage1, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + } + + return TRUE; +} + +/* test */ + +static void test_idle_connections(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.max_idle_time_msecs = 1000; + + test_begin("idle connections (max 1)"); + http_client_set.max_parallel_connections = 1; + test_run_client_server(&http_client_set, + test_client_idle_connections, + test_server_idle_connections, 1, NULL); + test_end(); + + test_begin("idle connections (max 2)"); + http_client_set.max_parallel_connections = 2; + test_run_client_server(&http_client_set, + test_client_idle_connections, + test_server_idle_connections, 1, NULL); + test_end(); + + test_begin("idle connections (max 4)"); + http_client_set.max_parallel_connections = 4; + test_run_client_server(&http_client_set, + test_client_idle_connections, + test_server_idle_connections, 1, NULL); + test_end(); + + test_begin("idle connections (max 8)"); + http_client_set.max_parallel_connections = 8; + test_run_client_server(&http_client_set, + test_client_idle_connections, + test_server_idle_connections, 1, NULL); + test_end(); +} + +/* + * Idle hosts + */ + +/* dns */ + +static void +test_dns_idle_hosts_input(struct server_connection *conn) +{ + const char *line; + + if (!conn->version_sent) { + conn->version_sent = TRUE; + o_stream_nsend_str(conn->conn.output, "VERSION\tdns\t1\t0\n"); + } + + while ((line = i_stream_read_next_line(conn->conn.input)) != NULL) { + if (str_begins(line, "VERSION")) + continue; + if (debug) + i_debug("DNS REQUEST: %s", line); + + if (strcmp(line, "IP\thosta") == 0) { + o_stream_nsend_str(conn->conn.output, + "0\t127.0.0.1\n"); + } else { + i_sleep_msecs(300); + o_stream_nsend_str( + conn->conn.output, + t_strdup_printf("%d\tFAIL\n", EAI_FAIL)); + } + } +} + +static void test_dns_idle_hosts(void) +{ + test_server_input = test_dns_idle_hosts_input; + test_server_run(0); +} + +/* server */ + +struct _idle_hosts_sctx { + bool eoh; +}; + +static int test_idle_hosts_init(struct server_connection *conn) +{ + struct _idle_hosts_sctx *ctx; + + ctx = p_new(conn->pool, struct _idle_hosts_sctx, 1); + conn->context = ctx; + return 0; +} + +static void test_idle_hosts_input(struct server_connection *conn) +{ + struct _idle_hosts_sctx *ctx = conn->context; + const char *line; + + while ((line = i_stream_read_next_line(conn->conn.input)) != NULL) { + if (*line == '\0') { + ctx->eoh = TRUE; + break; + } + } + + if (conn->conn.input->stream_errno != 0) { + i_fatal("server: Stream error: %s", + i_stream_get_error(conn->conn.input)); + } + if (line == NULL) { + if (conn->conn.input->eof) + server_connection_deinit(&conn); + return; + } + + i_assert(ctx->eoh); + ctx->eoh = FALSE; + + string_t *resp; + + resp = t_str_new(512); + str_printfa(resp, + "HTTP/1.1 200 OK\r\n" + "Content-Length: 0\r\n" + "\r\n"); + o_stream_nsend(conn->conn.output, str_data(resp), str_len(resp)); + if (o_stream_flush(conn->conn.output) < 0) { + i_fatal("server: Flush error: %s", + o_stream_get_error(conn->conn.output)); + } +} + +static void test_server_idle_hosts(unsigned int index) +{ + test_server_init = test_idle_hosts_init; + test_server_input = test_idle_hosts_input; + test_server_run(index); +} + +/* client */ + +struct _idle_hosts { + struct http_client *client; + struct http_client_request *hostb_req; + unsigned int count; +}; + +static void +test_client_idle_hosts_response_hosta(const struct http_response *resp, + struct _idle_hosts *ctx) +{ + test_client_assert_response(resp, resp->status == 200); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static void +test_client_idle_hosts_response_hostb(const struct http_response *resp, + struct _idle_hosts *ctx) +{ + test_client_assert_response(resp, + resp->status == HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED); + + if (http_client_request_try_retry(ctx->hostb_req)) { + if (debug) + i_debug("retrying"); + return; + } + + ctx->hostb_req = NULL; +} + +static bool +test_client_idle_hosts(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _idle_hosts *ctx; + + ctx = i_new(struct _idle_hosts, 1); + ctx->count = 2; + + ctx->client = http_client = http_client_init(client_set); + + hreq = http_client_request( + ctx->client, "GET", "hosta", + t_strdup_printf("/idle-hosts-a1.txt"), + test_client_idle_hosts_response_hosta, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = ctx->hostb_req = http_client_request( + ctx->client, "GET", "hostb", + t_strdup_printf("/idle-hosts-b.txt"), + test_client_idle_hosts_response_hostb, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request( + ctx->client, "GET", "hosta", + t_strdup_printf("/idle-hosts-a2.txt"), + test_client_idle_hosts_response_hosta, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_delay_msecs(hreq, 600); + http_client_request_submit(hreq); + + return TRUE; +} + +/* test */ + +static void test_idle_hosts(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.dns_client_socket_path = "./dns-test"; + http_client_set.dns_ttl_msecs = 400; + http_client_set.max_parallel_connections = 1; + http_client_set.max_idle_time_msecs = 100; + http_client_set.max_attempts = 2; + + test_begin("idle hosts"); + test_run_client_server(&http_client_set, + test_client_idle_hosts, + test_server_idle_hosts, 1, + test_dns_idle_hosts); + test_end(); +} + +/* + * All tests + */ + +static void (*const test_functions[])(void) = { + test_unconfigured_ssl, + test_unconfigured_ssl_abort, + test_invalid_url, + test_host_lookup_failed, + test_connection_refused, + test_connection_lost_prematurely, + test_connection_timed_out, + test_invalid_redirect, + test_unseekable_redirect, + test_unseekable_retry, + test_broken_payload, + test_retry_payload, + test_connection_lost, + test_connection_lost_100, + test_connection_lost_sub_ioloop, + test_early_success, + test_bad_response, + test_request_timed_out, + test_request_aborted_early, + test_request_failed_blocking, + test_client_deinit_early, + test_retry_with_delay, + test_dns_service_failure, + test_dns_timeout, + test_dns_lookup_failure, + test_dns_lookup_ttl, + test_peer_reuse_failure, + test_reconnect_failure, + test_multi_ip_attempts, + test_idle_connections, + test_idle_hosts, + NULL +}; + +/* + * Test client + */ + +static void test_client_defaults(struct http_client_settings *http_set) +{ + /* client settings */ + i_zero(http_set); + http_set->max_idle_time_msecs = 5*1000; + http_set->max_parallel_connections = 1; + http_set->max_pipelined_requests = 1; + http_set->max_redirects = 0; + http_set->max_attempts = 1; + http_set->debug = debug; +} + +static void test_client_progress_timeout(void *context ATTR_UNUSED) +{ + /* Terminate test due to lack of progress */ + test_assert(FALSE); + timeout_remove(&to_client_progress); + io_loop_stop(current_ioloop); +} + +static bool +test_client_init(test_client_init_t client_test, + const struct http_client_settings *client_set) +{ + i_assert(client_test != NULL); + if (!client_test(client_set)) + return FALSE; + + to_client_progress = timeout_add(CLIENT_PROGRESS_TIMEOUT*1000, + test_client_progress_timeout, NULL); + return TRUE; +} + +static void test_client_deinit(void) +{ + timeout_remove(&to_client_progress); + + if (http_client != NULL) + http_client_deinit(&http_client); +} + +static void +test_client_run(test_client_init_t client_test, + const struct http_client_settings *client_set) +{ + if (test_client_init(client_test, client_set)) + io_loop_run(ioloop); + test_client_deinit(); +} + +/* + * Test server + */ + +/* client connection */ + +static void server_connection_input(struct connection *_conn) +{ + struct server_connection *conn = (struct server_connection *)_conn; + + test_server_input(conn); +} + +static void server_connection_init(int fd) +{ + struct server_connection *conn; + pool_t pool; + + net_set_nonblock(fd, TRUE); + + pool = pool_alloconly_create("server connection", 512); + conn = p_new(pool, struct server_connection, 1); + conn->pool = pool; + + connection_init_server(server_conn_list, &conn->conn, + "server connection", fd, fd); + + if (test_server_init != NULL) { + if (test_server_init(conn) != 0) + return; + } +} + +static void server_connection_deinit(struct server_connection **_conn) +{ + struct server_connection *conn = *_conn; + + *_conn = NULL; + + if (test_server_deinit != NULL) + test_server_deinit(conn); + + connection_deinit(&conn->conn); + pool_unref(&conn->pool); +} + +static void server_connection_destroy(struct connection *_conn) +{ + struct server_connection *conn = (struct server_connection *)_conn; + + server_connection_deinit(&conn); +} + +static void server_connection_accept(void *context ATTR_UNUSED) +{ + int fd; + + /* accept new client */ + fd = net_accept(fd_listen, NULL, NULL); + if (fd == -1) + return; + if (fd == -2) { + i_fatal("test server: accept() failed: %m"); + } + + server_connection_init(fd); +} + +/* */ + +static struct connection_settings server_connection_set = { + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .client = FALSE +}; + +static const struct connection_vfuncs server_connection_vfuncs = { + .destroy = server_connection_destroy, + .input = server_connection_input +}; + +static void test_server_run(unsigned int index) +{ + server_index = index; + + /* open server socket */ + io_listen = io_add(fd_listen, IO_READ, server_connection_accept, NULL); + + server_conn_list = connection_list_init(&server_connection_set, + &server_connection_vfuncs); + + io_loop_run(ioloop); + + /* close server socket */ + io_remove(&io_listen); + + connection_list_deinit(&server_conn_list); +} + +/* + * Tests + */ + +struct test_server_data { + unsigned int index; + test_server_init_t server_test; +}; + +static int test_open_server_fd(in_port_t *bind_port) +{ + int fd = net_listen(&bind_ip, bind_port, 128); + if (debug) + i_debug("server listening on %u", *bind_port); + if (fd == -1) { + i_fatal("listen(%s:%u) failed: %m", + net_ip2addr(&bind_ip), *bind_port); + } + return fd; +} + +static int test_run_server(struct test_server_data *data) +{ + i_set_failure_prefix("SERVER[%u]: ", data->index + 1); + + if (debug) + i_debug("PID=%s", my_pid); + + test_subprocess_notify_signal_send_parent(SIGHUP); + ioloop = io_loop_create(); + data->server_test(data->index); + io_loop_destroy(&ioloop); + + if (debug) + i_debug("Terminated"); + + i_close_fd(&fd_listen); + i_free(bind_ports); + main_deinit(); + return 0; +} + +static int test_run_dns(test_dns_init_t dns_test) +{ + i_set_failure_prefix("DNS: "); + + if (debug) + i_debug("PID=%s", my_pid); + + ioloop = io_loop_create(); + dns_test(); + io_loop_destroy(&ioloop); + + if (debug) + i_debug("Terminated"); + + i_close_fd(&fd_listen); + i_free(bind_ports); + main_deinit(); + return 0; +} + +static void +test_run_client(const struct http_client_settings *client_set, + test_client_init_t client_test) +{ + i_set_failure_prefix("CLIENT: "); + + if (debug) + i_debug("PID=%s", my_pid); + + ioloop = io_loop_create(); + test_client_run(client_test, client_set); + io_loop_destroy(&ioloop); + + if (debug) + i_debug("Terminated"); +} + +static void +test_run_client_server(const struct http_client_settings *client_set, + test_client_init_t client_test, + test_server_init_t server_test, + unsigned int server_tests_count, + test_dns_init_t dns_test) +{ + unsigned int i; + + test_subprocess_notify_signal_reset(SIGHUP); + test_server_init = NULL; + test_server_deinit = NULL; + test_server_input = NULL; + + if (server_tests_count > 0) { + int fds[server_tests_count]; + + bind_ports = i_new(in_port_t, server_tests_count); + for (i = 0; i < server_tests_count; i++) + fds[i] = test_open_server_fd(&bind_ports[i]); + + for (i = 0; i < server_tests_count; i++) { + struct test_server_data data; + + i_zero(&data); + data.index = i; + data.server_test = server_test; + + /* Fork server */ + fd_listen = fds[i]; + test_subprocess_fork(test_run_server, &data, FALSE); + i_close_fd(&fd_listen); + test_subprocess_notify_signal_wait(SIGHUP, 10000); + test_subprocess_notify_signal_reset(SIGHUP); + } + } + + if (dns_test != NULL) { + int fd; + + i_unlink_if_exists("./dns-test"); + fd = net_listen_unix("./dns-test", 128); + if (fd == -1) { + i_fatal("listen(./dns-test) failed: %m"); + } + + /* Fork DNS service */ + fd_listen = fd; + test_subprocess_fork(test_run_dns, dns_test, FALSE); + i_close_fd(&fd_listen); + } + + /* Run client */ + test_run_client(client_set, client_test); + + i_unset_failure_prefix(); + test_subprocess_kill_all(SERVER_KILL_TIMEOUT_SECS); + i_free(bind_ports); + + i_unlink_if_exists("./dns-test"); +} + +/* + * Main + */ + +static void main_init(void) +{ + /* nothing yet */ +} + +static void main_deinit(void) +{ + /* nothing yet; also called from sub-processes */ +} + +int main(int argc, char *argv[]) +{ + int c; + int ret; + + lib_init(); + main_init(); + + while ((c = getopt(argc, argv, "D")) > 0) { + switch (c) { + case 'D': + debug = TRUE; + break; + default: + i_fatal("Usage: %s [-D]", argv[0]); + } + } + + test_subprocesses_init(debug); + + /* listen on localhost */ + i_zero(&bind_ip); + bind_ip.family = AF_INET; + bind_ip.u.ip4.s_addr = htonl(INADDR_LOOPBACK); + + ret = test_run(test_functions); + + test_subprocesses_deinit(); + main_deinit(); + lib_deinit(); + + return ret; +} diff --git a/src/lib-http/test-http-client-request.c b/src/lib-http/test-http-client-request.c new file mode 100644 index 0000000..5b6fa93 --- /dev/null +++ b/src/lib-http/test-http-client-request.c @@ -0,0 +1,95 @@ +/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "test-common.h" +#include "str.h" +#include "http-client-private.h" + +static void +test_http_client_request_callback(const struct http_response *response ATTR_UNUSED, + void *context ATTR_UNUSED) +{ +} + +static void test_http_client_request_headers(void) +{ + struct http_client_settings set; + struct http_client *client; + struct http_client_request *req; + + test_begin("http client request headers"); + i_zero(&set); + client = http_client_init(&set); + req = http_client_request(client, "GET", "host", "target", + test_http_client_request_callback, NULL); + + test_assert(http_client_request_lookup_header(req, "qwe") == NULL); + + /* add the first */ + http_client_request_add_header(req, "qwe", "value1"); + test_assert_strcmp(http_client_request_lookup_header(req, "qwe"), "value1"); + test_assert_strcmp(str_c(req->headers), "qwe: value1\r\n"); + + /* replace the first with the same length */ + http_client_request_add_header(req, "qwe", "234567"); + test_assert_strcmp(http_client_request_lookup_header(req, "qwe"), "234567"); + test_assert_strcmp(str_c(req->headers), "qwe: 234567\r\n"); + + /* replace the first with smaller length */ + http_client_request_add_header(req, "qwe", "xyz"); + test_assert_strcmp(http_client_request_lookup_header(req, "qwe"), "xyz"); + test_assert_strcmp(str_c(req->headers), "qwe: xyz\r\n"); + + /* replace the first with longer length */ + http_client_request_add_header(req, "qwe", "abcdefg"); + test_assert_strcmp(http_client_request_lookup_header(req, "qwe"), "abcdefg"); + test_assert_strcmp(str_c(req->headers), "qwe: abcdefg\r\n"); + + /* add the second */ + http_client_request_add_header(req, "xyz", "1234"); + test_assert_strcmp(http_client_request_lookup_header(req, "qwe"), "abcdefg"); + test_assert_strcmp(http_client_request_lookup_header(req, "xyz"), "1234"); + test_assert_strcmp(str_c(req->headers), "qwe: abcdefg\r\nxyz: 1234\r\n"); + + /* replace second */ + http_client_request_add_header(req, "xyz", "yuiop"); + test_assert_strcmp(http_client_request_lookup_header(req, "qwe"), "abcdefg"); + test_assert_strcmp(http_client_request_lookup_header(req, "xyz"), "yuiop"); + test_assert_strcmp(str_c(req->headers), "qwe: abcdefg\r\nxyz: yuiop\r\n"); + + /* replace the first again */ + http_client_request_add_header(req, "qwe", "1234"); + test_assert_strcmp(http_client_request_lookup_header(req, "qwe"), "1234"); + test_assert_strcmp(http_client_request_lookup_header(req, "xyz"), "yuiop"); + test_assert_strcmp(str_c(req->headers), "qwe: 1234\r\nxyz: yuiop\r\n"); + + /* remove the headers */ + http_client_request_remove_header(req, "qwe"); + test_assert(http_client_request_lookup_header(req, "qwe") == NULL); + test_assert_strcmp(http_client_request_lookup_header(req, "xyz"), "yuiop"); + test_assert_strcmp(str_c(req->headers), "xyz: yuiop\r\n"); + + http_client_request_remove_header(req, "xyz"); + test_assert(http_client_request_lookup_header(req, "qwe") == NULL); + test_assert(http_client_request_lookup_header(req, "xyz") == NULL); + test_assert_strcmp(str_c(req->headers), ""); + + /* test _add_missing_header() */ + http_client_request_add_missing_header(req, "foo", "bar"); + test_assert_strcmp(str_c(req->headers), "foo: bar\r\n"); + http_client_request_add_missing_header(req, "foo", "123"); + test_assert_strcmp(str_c(req->headers), "foo: bar\r\n"); + + http_client_request_abort(&req); + http_client_deinit(&client); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_http_client_request_headers, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-http/test-http-client.c b/src/lib-http/test-http-client.c new file mode 100644 index 0000000..fc24bfa --- /dev/null +++ b/src/lib-http/test-http-client.c @@ -0,0 +1,472 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "safe-memset.h" +#include "ioloop.h" +#include "istream.h" +#include "write-full.h" +#include "http-url.h" +#include "http-client.h" +#include "dns-lookup.h" +#include "iostream-ssl.h" +#ifdef HAVE_OPENSSL +#include "iostream-openssl.h" +#endif + +#include <fcntl.h> +#include <unistd.h> + +struct http_test_request { + struct io *io; + struct istream *payload; + bool write_output; +}; + +static struct ioloop *ioloop; + +static void payload_input(struct http_test_request *req) +{ + const unsigned char *data; + size_t size; + int ret; + + /* read payload */ + while ((ret=i_stream_read_more(req->payload, &data, &size)) > 0) { + i_info("DEBUG: got data (size=%d)", (int)size); + if (req->write_output) + if (write_full(1, data, size) < 0) + i_error("REQUEST PAYLOAD WRITE ERROR: %m"); + i_stream_skip(req->payload, size); + } + + if (ret == 0) { + i_info("DEBUG: REQUEST: NEED MORE DATA"); + /* we will be called again for this request */ + } else { + if (req->payload->stream_errno != 0) { + i_error("REQUEST PAYLOAD READ ERROR: %s", + i_stream_get_error(req->payload)); + } else + i_info("DEBUG: REQUEST: Finished"); + io_remove(&req->io); + i_stream_unref(&req->payload); + i_free(req); + } +} + +static void +got_request_response(const struct http_response *response, + struct http_test_request *req) +{ + io_loop_stop(ioloop); + + if (response->status / 100 != 2) { + i_error("HTTP Request failed: %s", response->reason); + i_free(req); + /* payload (if any) is skipped implicitly */ + return; + } + + i_info("DEBUG: REQUEST SUCCEEDED: %s", response->reason); + + if (response->payload == NULL) { + i_free(req); + return; + } + + i_info("DEBUG: REQUEST: Got payload"); + i_stream_ref(response->payload); + req->payload = response->payload; + req->io = io_add_istream(response->payload, payload_input, req); + payload_input(req); +} + +static const char *test_query1 = "data=Frop&submit=Submit"; +static const char *test_query2 = "data=This%20is%20a%20test&submit=Submit"; +static const char *test_query3 = "foo=bar"; + +static void run_tests(struct http_client *http_client) +{ + struct http_client_request *http_req; + struct http_test_request *test_req; + struct istream *post_payload; + + // JigSAW is useful for testing: http://jigsaw.w3.org/HTTP/ + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "GET", "pigeonhole.dovecot.org", "/", + got_request_response, test_req); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "GET", "pigeonhole.dovecot.org", "/download.html", + got_request_response, test_req); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "GET", "jigsaw.w3.org", "/HTTP/300/301.html", + got_request_response, test_req); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "GET", "pigeonhole.dovecot.org", "/frop.html", + got_request_response, test_req); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "GET", "jigsaw.w3.org", "/HTTP/300/307.html", + got_request_response, test_req); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "GET", "pigeonhole.dovecot.org", "/documentation.html", + got_request_response, test_req); + http_client_request_set_urgent(http_req); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "GET", "jigsaw.w3.org", "/HTTP/300/302.html", + got_request_response, test_req); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "POST", "test.dovecot.org", "/http/post/index.php", + got_request_response, test_req); + post_payload = i_stream_create_from_data + ((const unsigned char *)test_query1, strlen(test_query1)); + http_client_request_set_payload(http_req, post_payload, FALSE); + i_stream_unref(&post_payload); + http_client_request_add_header(http_req, + "Content-Type", "application/x-www-form-urlencoded"); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "POST", "test.dovecot.org", "/http/post/index.php", + got_request_response, test_req); + post_payload = i_stream_create_from_data + ((const unsigned char *)test_query2, strlen(test_query2)); + http_client_request_set_payload(http_req, post_payload, TRUE); + i_stream_unref(&post_payload); + http_client_request_add_header(http_req, + "Content-Type", "application/x-www-form-urlencoded"); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "GET", "pigeonhole.dovecot.org", "/", + got_request_response, test_req); + http_client_request_set_port(http_req, 81); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "HEAD", "pigeonhole.dovecot.org", "/download.html", + got_request_response, test_req); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "GET", "pigeonhole.dovecot.org", "/", + got_request_response, test_req); + http_client_request_set_ssl(http_req, TRUE); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "GET", "pigeonhole.dovecot.org", "/download.html", + got_request_response, test_req); + http_client_request_set_ssl(http_req, TRUE); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "GET", "pigeonhole.dovecot.org", "/documentation.html", + got_request_response, test_req); + http_client_request_set_ssl(http_req, TRUE); + http_client_request_submit(http_req); + http_client_request_abort(&http_req); + i_free(test_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "POST", "posttestserver.com", "/post.php", + got_request_response, test_req); + post_payload = i_stream_create_from_data + ((const unsigned char *)test_query1, strlen(test_query1)); + http_client_request_set_payload(http_req, post_payload, TRUE); + i_stream_unref(&post_payload); + http_client_request_set_ssl(http_req, TRUE); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "POST", "posttestserver.com", "/post.php", + got_request_response, test_req); + post_payload = i_stream_create_from_data + ((const unsigned char *)test_query1, strlen(test_query1)); + http_client_request_set_payload(http_req, post_payload, TRUE); + i_stream_unref(&post_payload); + http_client_request_set_ssl(http_req, TRUE); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "POST", "posttestserver.com", "/post.php", + got_request_response, test_req); + post_payload = i_stream_create_from_data + ((const unsigned char *)test_query1, strlen(test_query1)); + http_client_request_set_payload(http_req, post_payload, TRUE); + i_stream_unref(&post_payload); + http_client_request_set_ssl(http_req, TRUE); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "GET", "wiki2.dovecot.org", "/Pigeonhole", + got_request_response, test_req); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "GET", "jigsaw.w3.org", "/HTTP/ChunkedScript", + got_request_response, test_req); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "POST", "jigsaw.w3.org", "/HTTP/300/Go_307", + got_request_response, test_req); + post_payload = i_stream_create_from_data + ((const unsigned char *)test_query3, strlen(test_query3)); + http_client_request_set_payload(http_req, post_payload, FALSE); + i_stream_unref(&post_payload); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "POST", "jigsaw.w3.org", "/HTTP/300/Go_307", + got_request_response, test_req); + post_payload = i_stream_create_from_data + ((const unsigned char *)test_query3, strlen(test_query3)); + http_client_request_set_payload(http_req, post_payload, FALSE); + i_stream_unref(&post_payload); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "POST", "jigsaw.w3.org", "/HTTP/300/Go_307", + got_request_response, test_req); + post_payload = i_stream_create_from_data + ((const unsigned char *)test_query3, strlen(test_query3)); + http_client_request_set_payload(http_req, post_payload, FALSE); + i_stream_unref(&post_payload); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "GET", "jigsaw.w3.org", "/HTTP/Basic/", + got_request_response, test_req); + http_client_request_set_auth_simple + (http_req, "guest", "guest"); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, + "PUT", "test.dovecot.org", "/http/put/put.php", + got_request_response, test_req); + post_payload = i_stream_create_file("Makefile.am", 10); + http_client_request_set_payload(http_req, post_payload, TRUE); + i_stream_unref(&post_payload); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request_url_str(http_client, + "GET", "https://invalid.dovecot.org/", + got_request_response, test_req); + http_client_request_submit(http_req); +} + +static void +test_http_request_init(struct http_client *http_client, + const char *method, const char *url_str, + struct http_client_request **http_req_r, + struct http_test_request **test_req_r) +{ + struct http_client_request *http_req; + struct http_test_request *test_req; + struct http_url *url; + const char *error; + + if (http_url_parse(url_str, NULL, 0, pool_datastack_create(), + &url, &error) < 0) + i_fatal("Invalid URL %s: %s", url_str, error); + + test_req = i_new(struct http_test_request, 1); + test_req->write_output = TRUE; + http_req = http_client_request(http_client, + method, url->host.name, + t_strconcat("/", url->path, url->enc_query, NULL), + got_request_response, test_req); + if (url->port != 0) + http_client_request_set_port(http_req, url->port); + if (url->have_ssl) + http_client_request_set_ssl(http_req, TRUE); + + *http_req_r = http_req; + *test_req_r = test_req; +} + +static void run_http_get(struct http_client *http_client, const char *url_str) +{ + struct http_client_request *http_req; + struct http_test_request *test_req; + + test_http_request_init(http_client, "GET", url_str, &http_req, &test_req); + http_client_request_submit(http_req); +} + +static void run_http_post(struct http_client *http_client, const char *url_str, + const char *path) +{ + struct http_client_request *http_req; + struct http_test_request *test_req; + struct istream *input; + + test_http_request_init(http_client, "POST", url_str, &http_req, &test_req); + input = i_stream_create_file(path, IO_BLOCK_SIZE); + http_client_request_set_payload(http_req, input, FALSE); + i_stream_unref(&input); + http_client_request_submit(http_req); +} + +int main(int argc, char *argv[]) +{ + struct dns_client *dns_client; + struct dns_lookup_settings dns_set; + struct http_client_settings http_set; + struct http_client_context *http_cctx; + struct http_client *http_client1, *http_client2, *http_client3, *http_client4; + struct ssl_iostream_settings ssl_set; + struct stat st; + const char *error; + + lib_init(); +#ifdef HAVE_OPENSSL + ssl_iostream_openssl_init(); +#endif + ioloop = io_loop_create(); + io_loop_set_running(ioloop); + + /* kludge: use safe_memset() here since otherwise it's not included in + the binary in all systems (but is in others! so linking + safe-memset.lo directly causes them to fail.) If safe_memset() isn't + included, libssl-iostream plugin loading fails. */ + i_zero_safe(&dns_set); + dns_set.dns_client_socket_path = PKG_RUNDIR"/dns-client"; + dns_set.timeout_msecs = 30*1000; + dns_set.idle_timeout_msecs = UINT_MAX; + + /* check if there is a DNS client */ + if (access(dns_set.dns_client_socket_path, R_OK|W_OK) == 0) { + dns_client = dns_client_init(&dns_set); + + if (dns_client_connect(dns_client, &error) < 0) + i_fatal("Couldn't initialize DNS client: %s", error); + } else { + dns_client = NULL; + } + i_zero(&ssl_set); + ssl_set.allow_invalid_cert = TRUE; + if (stat("/etc/ssl/certs", &st) == 0 && S_ISDIR(st.st_mode)) + ssl_set.ca_dir = "/etc/ssl/certs"; /* debian */ + if (stat("/etc/ssl/certs", &st) == 0 && S_ISREG(st.st_mode)) + ssl_set.ca_file = "/etc/pki/tls/cert.pem"; /* redhat */ + + i_zero(&http_set); + http_set.ssl = &ssl_set; + http_set.dns_client = dns_client; + http_set.max_idle_time_msecs = 5*1000; + http_set.max_parallel_connections = 4; + http_set.max_pipelined_requests = 4; + http_set.max_redirects = 2; + http_set.request_timeout_msecs = 10*1000; + http_set.max_attempts = 1; + http_set.debug = TRUE; + http_set.rawlog_dir = "/tmp/http-test"; + + http_cctx = http_client_context_create(&http_set); + + http_client1 = http_client_init_shared(http_cctx, NULL); + http_client2 = http_client_init_shared(http_cctx, NULL); + http_client3 = http_client_init_shared(http_cctx, NULL); + http_client4 = http_client_init_shared(http_cctx, NULL); + + switch (argc) { + case 1: + run_tests(http_client1); + run_tests(http_client2); + run_tests(http_client3); + run_tests(http_client4); + break; + case 2: + run_http_get(http_client1, argv[1]); + run_http_get(http_client2, argv[1]); + run_http_get(http_client3, argv[1]); + run_http_get(http_client4, argv[1]); + break; + case 3: + run_http_post(http_client1, argv[1], argv[2]); + run_http_post(http_client2, argv[1], argv[2]); + run_http_post(http_client3, argv[1], argv[2]); + run_http_post(http_client4, argv[1], argv[2]); + break; + default: + i_fatal("Too many parameters"); + } + + for (;;) { + bool pending = FALSE; + + if (http_client_get_pending_request_count(http_client1) > 0) { + i_debug("Requests still pending in client 1"); + pending = TRUE; + } else if (http_client_get_pending_request_count(http_client2) > 0) { + i_debug("Requests still pending in client 2"); + pending = TRUE; + } else if (http_client_get_pending_request_count(http_client3) > 0) { + i_debug("Requests still pending in client 3"); + pending = TRUE; + } else if (http_client_get_pending_request_count(http_client4) > 0) { + i_debug("Requests still pending in client 4"); + pending = TRUE; + } + if (!pending) + break; + io_loop_run(ioloop); + } + http_client_deinit(&http_client1); + http_client_deinit(&http_client2); + http_client_deinit(&http_client3); + http_client_deinit(&http_client4); + + http_client_context_unref(&http_cctx); + + if (dns_client != NULL) + dns_client_deinit(&dns_client); + + io_loop_destroy(&ioloop); + ssl_iostream_context_cache_free(); +#ifdef HAVE_OPENSSL + ssl_iostream_openssl_deinit(); +#endif + lib_deinit(); +} diff --git a/src/lib-http/test-http-date.c b/src/lib-http/test-http-date.c new file mode 100644 index 0000000..1d8150a --- /dev/null +++ b/src/lib-http/test-http-date.c @@ -0,0 +1,223 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "test-common.h" +#include "http-date.h" + +#include <time.h> + +struct http_date_test { + const char *date_in; + const char *date_out; + + struct tm tm; +}; + +/* Valid date tests */ +static const struct http_date_test valid_date_tests[] = { + /* Preferred format: */ + { + .date_in = "Sun, 11 Nov 2007 09:42:43 GMT", + .tm = { + .tm_year = 107, .tm_mon = 10, .tm_mday = 11, + .tm_hour = 9, .tm_min = 42, .tm_sec = 43 }, + },{ + .date_in = "Mon, 17 Aug 1992 13:06:27 GMT", + .tm = { + .tm_year = 92, .tm_mon = 7, .tm_mday = 17, + .tm_hour = 13, .tm_min = 06, .tm_sec = 27 }, + },{ + .date_in = "Tue, 03 Sep 1974 04:38:08 GMT", + .tm = { + .tm_year = 74, .tm_mon = 8, .tm_mday = 3, + .tm_hour = 4, .tm_min = 38, .tm_sec = 8 }, + },{ + .date_in = "Wed, 07 May 1980 06:20:42 GMT", + .tm = { + .tm_year = 80, .tm_mon = 4, .tm_mday = 7, + .tm_hour = 6, .tm_min = 20, .tm_sec = 42 }, + },{ + .date_in = "Thu, 15 Oct 1987 18:30:14 GMT", + .tm = { + .tm_year = 87, .tm_mon = 9, .tm_mday = 15, + .tm_hour = 18, .tm_min = 30, .tm_sec = 14 }, + },{ + .date_in = "Fri, 20 Dec 1996 00:20:07 GMT", + .tm = { + .tm_year = 96, .tm_mon = 11, .tm_mday = 20, + .tm_hour = 0, .tm_min = 20, .tm_sec = 7 }, + },{ + .date_in = "Sat, 19 Jan 2036 19:52:18 GMT", + .tm = { + .tm_year = 136, .tm_mon = 0, .tm_mday = 19, + .tm_hour = 19, .tm_min = 52, .tm_sec = 18 }, + },{ + .date_in = "Mon, 17 Apr 2006 14:41:45 GMT", + .tm = { + .tm_year = 106, .tm_mon = 3, .tm_mday = 17, + .tm_hour = 14, .tm_min = 41, .tm_sec = 45 }, + },{ + .date_in = "Sun, 06 Mar 2011 16:18:41 GMT", + .tm = { + .tm_year = 111, .tm_mon = 2, .tm_mday = 6, + .tm_hour = 16, .tm_min = 18, .tm_sec = 41 }, + },{ + .date_in = "Sat, 14 Jun 1975 16:09:30 GMT", + .tm = { + .tm_year = 75, .tm_mon = 5, .tm_mday = 14, + .tm_hour = 16, .tm_min = 9, .tm_sec = 30 }, + },{ + .date_in = "Fri, 05 Feb 2027 06:53:58 GMT", + .tm = { + .tm_year = 127, .tm_mon = 1, .tm_mday = 5, + .tm_hour = 6, .tm_min = 53, .tm_sec = 58 }, + },{ + .date_in = "Mon, 09 Jul 2018 02:24:29 GMT", + .tm = { + .tm_year = 118, .tm_mon = 6, .tm_mday = 9, + .tm_hour = 2, .tm_min = 24, .tm_sec = 29 }, + + /* Obsolete formats: */ + },{ + .date_in = "Wednesday, 02-Jun-82 16:06:23 GMT", + .date_out = "Wed, 02 Jun 1982 16:06:23 GMT", + .tm = { + .tm_year = 82, .tm_mon = 5, .tm_mday = 2, + .tm_hour = 16, .tm_min = 6, .tm_sec = 23 }, + },{ + .date_in = "Thursday, 23-May-02 12:16:24 GMT", + .date_out = "Thu, 23 May 2002 12:16:24 GMT", + .tm = { + .tm_year = 102, .tm_mon = 4, .tm_mday = 23, + .tm_hour = 12, .tm_min = 16, .tm_sec = 24 }, + },{ + .date_in = "Sun Nov 6 08:49:37 1994", + .date_out = "Sun, 06 Nov 1994 08:49:37 GMT", + .tm = { + .tm_year = 94, .tm_mon = 10, .tm_mday = 6, + .tm_hour = 8, .tm_min = 49, .tm_sec = 37 }, + },{ + .date_in = "Mon Apr 30 02:45:01 2012", + .date_out = "Mon, 30 Apr 2012 02:45:01 GMT", + .tm = { + .tm_year = 112, .tm_mon = 3, .tm_mday = 30, + .tm_hour = 2, .tm_min = 45, .tm_sec = 01 }, + } +}; + +static const unsigned int valid_date_test_count = N_ELEMENTS(valid_date_tests); + +static void test_http_date_valid(void) +{ + unsigned int i; + + for (i = 0; i < valid_date_test_count; i++) T_BEGIN { + const char *date_in, *date_out, *pdate_out; + const struct tm *tm = &valid_date_tests[i].tm; + struct tm ptm; + bool result; + + date_in = valid_date_tests[i].date_in; + date_out = valid_date_tests[i].date_out == NULL ? + date_in : valid_date_tests[i].date_out; + + test_begin(t_strdup_printf("http date valid [%d]", i)); + + result = http_date_parse_tm + ((const unsigned char *)date_in, strlen(date_in), &ptm); + test_out(t_strdup_printf("parse %s", date_in), result); + if (result) { + bool equal = tm->tm_year == ptm.tm_year && tm->tm_mon == ptm.tm_mon && + tm->tm_mday == ptm.tm_mday && tm->tm_hour == ptm.tm_hour && + tm->tm_min == ptm.tm_min && tm->tm_sec == ptm.tm_sec; + + test_out("valid timestamp", equal); + + pdate_out = http_date_create_tm(&ptm); + test_out_reason("valid create", strcmp(date_out, pdate_out) == 0, + pdate_out); + } + + test_end(); + } T_END; +} + +/* Invalid date tests */ +static const char *invalid_date_tests[] = { + "Mom, 09 Jul 2018 02:24:29 GMT", + "Mon; 09 Jul 2018 02:24:29 GMT", + "Mon, 09 Jul 2018 02:24:29 GMT", + "Mon, 90 Jul 2018 02:24:29 GMT", + "Mon, 090 Jul 2018 02:24:29 GMT", + "Mon, 09 Jul 2018 02:24:29 GMT", + "Mon, 09 Lul 2018 02:24:29 GMT", + "Mon, 09 July 2018 02:24:29 GMT", + "Mon, 09 Jul 2018 02:24:29 GMT", + "Mon, 09 Jul 22018 02:24:29 GMT", + "Mon, 09 Jul 2018 02:24:29 GMT", + "Mon, 09 Jul 2018 032:24:29 GMT", + "Mon, 09 Jul 2018 02:224:29 GMT", + "Mon, 09 Jul 2018 02:24:239 GMT", + "Mon, 09 Jul 2018 02;24:29 GMT", + "Mon, 09 Jul 2018 02:24;29 GMT", + "Mon, 09 Jul 2018 45:24:29 GMT", + "Mon, 09 Jul 2018 02:90:29 GMT", + "Mon, 09 Jul 2018 02:24:84 GMT", + "Mon, 09 Jul 2018 02:24:29 GMT", + "Mon, 09 Jul 2018 02:24:29 UTC", + "Mon, 09 Jul 2018 02:24:29 GM", + "Mon, 09 Jul 2018 02:24:29 GMTREE", + "Thu, 23-May-02 12:16:24 GMT", + "Thursday; 23-May-02 12:16:24 GMT", + "Thursday, 223-May-02 12:16:24 GMT", + "Thursday, 23-Mays-02 12:16:24 GMT", + "Thursday, 23-May-2002 12:16:24 GMT", + "Thursday, 23-May-02 122:16:24 GMT", + "Thursday, 23-May-02 12:164:24 GMT", + "Thursday, 23-May-02 12:16:244 GMT", + "Thursday, 23-May-02 12:16:24 EET", + "Sunday Nov 6 08:49:37 1994", + "Sun Nov 6 08:49:37 1994", + "Sun November 6 08:49:37 1994", + "Sun Nov 6 08:49:37 1994", + "Sun Nov 16 08:49:37 1994", + "Sun Nov 16 08:49:37 1994", + "Sun Nov 6 082:49:37 1994", + "Sun Nov 6 08:492:37 1994", + "Sun Nov 6 08:49:137 1994", + "Sun Nov 6 08:49:37 19914", + "Sun Nov 6 08:49:37 0000", +}; + +static const unsigned int invalid_date_test_count = N_ELEMENTS(invalid_date_tests); + +static void test_http_date_invalid(void) +{ + unsigned int i; + + for (i = 0; i < invalid_date_test_count; i++) T_BEGIN { + const char *date_in; + struct tm tm; + bool result; + + date_in = invalid_date_tests[i]; + + test_begin(t_strdup_printf("http date invalid [%d]", i)); + + result = http_date_parse_tm + ((const unsigned char *)date_in, strlen(date_in), &tm); + test_out(t_strdup_printf("parse %s", date_in), !result); + + test_end(); + } T_END; +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_http_date_valid, + test_http_date_invalid, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-http/test-http-header-parser.c b/src/lib-http/test-http-header-parser.c new file mode 100644 index 0000000..0550039 --- /dev/null +++ b/src/lib-http/test-http-header-parser.c @@ -0,0 +1,381 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "str-sanitize.h" +#include "istream.h" +#include "test-common.h" +#include "http-response.h" +#include "http-header-parser.h" + +#include <time.h> + +struct http_header_parse_result { + const char *name; + const char *value; +}; + +struct http_header_parse_test { + const char *header; + struct http_header_limits limits; + enum http_header_parse_flags flags; + const struct http_header_parse_result *fields; +}; + +/* Valid header tests */ + +static const struct http_header_parse_result valid_header_parse_result1[] = { + { "Date", "Sat, 06 Oct 2012 16:01:44 GMT" }, + { "Server", "Apache/2.2.16 (Debian)" }, + { "Last-Modified", "Mon, 30 Jul 2012 11:09:28 GMT" }, + { "Etag", "\"3d24677-3261-4c60a1863aa00\"" }, + { "Accept-Ranges", "bytes" }, + { "Vary", "Accept-Encoding" }, + { "Content-Encoding", "gzip" }, + { "Content-Length", "4092" }, + { "Keep-Alive", "timeout=15, max=100" }, + { "Connection", "Keep-Alive" }, + { "Content-Type", "text/html" }, + { NULL, NULL } +}; + +static const struct http_header_parse_result valid_header_parse_result2[] = { + { "Host", "p5-lrqzb4yavu4l7nagydw-428649-i2-v6exp3-ds.metric.example.com" }, + { "User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:15.0)" }, + { "Accept", "image/png,image/*;q=0.8,*/*;q=0.5" }, + { "Accept-Language", "en-us,en;q=0.5" }, + { "Accept-Encoding", "gzip, deflate" }, + { "DNT", "1" }, + { "Connection", "keep-alive" }, + { "Referer", "http://www.example.nl/" }, + { NULL, NULL } +}; + +static const struct http_header_parse_result valid_header_parse_result3[] = { + { "Date", "Sat, 06 Oct 2012 17:12:37 GMT" }, + { "Server", "Apache/2.2.16 (Debian) PHP/5.3.3-7+squeeze14 with" + " Suhosin-Patch proxy_html/3.0.1 mod_python/3.3.1 Python/2.6.6" + " mod_ssl/2.2.16 OpenSSL/0.9.8o mod_perl/2.0.4 Perl/v5.10.1" }, + { "WWW-Authenticate", "Basic realm=\"Munin\"" }, + { "Vary", "Accept-Encoding" }, + { "Content-Encoding", "gzip" }, + { "Content-Length", "445" }, + { "Keep-Alive", "timeout=15, max=98" }, + { "Connection", "Keep-Alive" }, + { "Content-Type", "text/html; charset=iso-8859-1" }, + { NULL, NULL } +}; + +static const struct http_header_parse_result valid_header_parse_result4[] = { + { "Age", "58" }, + { "Date", "Sun, 04 Aug 2013 09:33:09 GMT" }, + { "Expires", "Sun, 04 Aug 2013 09:34:08 GMT" }, + { "Cache-Control", "max-age=60" }, + { "Content-Length", "17336" }, + { "Connection", "Keep-Alive" }, + { "Via", "NS-CACHE-9.3" }, + { "Server", "Apache" }, + { "Vary", "Host" }, + { "Last-Modified", "Sun, 04 Aug 2013 09:33:07 GMT" }, + { "Content-Type", "text/html; charset=utf-8" }, + { "Content-Encoding", "gzip" }, + { NULL, NULL } +}; + +static const struct http_header_parse_result valid_header_parse_result5[] = { + { NULL, NULL } +}; + +static const struct http_header_parse_result valid_header_parse_result6[] = { + { "X-Frop", "This text\x80 contains obs-text\x81 characters" }, + { NULL, NULL } +}; + +static const struct http_header_parse_result valid_header_parse_result7[] = { + { "X-Frop", "This text contains invalid characters" }, + { NULL, NULL } +}; + +static const struct http_header_parse_test valid_header_parse_tests[] = { + { .header = + "Date: Sat, 06 Oct 2012 16:01:44 GMT\r\n" + "Server: Apache/2.2.16 (Debian)\r\n" + "Last-Modified: Mon, 30 Jul 2012 11:09:28 GMT\r\n" + "Etag: \"3d24677-3261-4c60a1863aa00\"\r\n" + "Accept-Ranges: bytes\r\n" + "Vary: Accept-Encoding\r\n" + "Content-Encoding: gzip\r\n" + "Content-Length: 4092\r\n" + "Keep-Alive: timeout=15, max=100\r\n" + "Connection: Keep-Alive\r\n" + "Content-Type: text/html\r\n" + "\r\n", + .fields = valid_header_parse_result1 + },{ + .header = + "Host: p5-lrqzb4yavu4l7nagydw-428649-i2-v6exp3-ds.metric.example.com\n" + "User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64; rv:15.0)\n" + "Accept:\t\timage/png,image/*;q=0.8,*/*;q=0.5\n" + "Accept-Language:\ten-us,en;q=0.5\n" + "Accept-Encoding: \t\tgzip, deflate\n" + "DNT: 1\n" + "Connection: \t\tkeep-alive\n" + "Referer: http://www.example.nl/\n" + "\n", + .fields = valid_header_parse_result2 + },{ + .header = + "Date: Sat, 06 Oct 2012 17:12:37 GMT\r\n" + "Server: Apache/2.2.16 (Debian) PHP/5.3.3-7+squeeze14 with\r\n" + " Suhosin-Patch proxy_html/3.0.1 mod_python/3.3.1 Python/2.6.6\r\n" + " mod_ssl/2.2.16 OpenSSL/0.9.8o mod_perl/2.0.4 Perl/v5.10.1\r\n" + "WWW-Authenticate: Basic realm=\"Munin\"\r\n" + "Vary: Accept-Encoding\r\n" + "Content-Encoding: gzip\r\n" + "Content-Length: 445\r\n" + "Keep-Alive: timeout=15, max=98\r\n" + "Connection: Keep-Alive\r\n" + "Content-Type: text/html; charset=iso-8859-1\r\n" + "\r\n", + .fields = valid_header_parse_result3 + },{ + .header = + "Age: 58 \r\n" + "Date: Sun, 04 Aug 2013 09:33:09 GMT\r\n" + "Expires: Sun, 04 Aug 2013 09:34:08 GMT\r\n" + "Cache-Control: max-age=60 \r\n" + "Content-Length: 17336 \r\n" + "Connection: Keep-Alive\r\n" + "Via: NS-CACHE-9.3\r\n" + "Server: Apache\r\n" + "Vary: Host\r\n" + "Last-Modified: Sun, 04 Aug 2013 09:33:07 GMT\r\n" + "Content-Type: text/html; charset=utf-8\r\n" + "Content-Encoding: gzip\r\n" + "\r\n", + .fields = valid_header_parse_result4, + .limits = { + .max_size = 340, + .max_field_size = 46, + .max_fields = 12 + } + },{ + .header = + "\r\n", + .fields = valid_header_parse_result5 + },{ + .header = + "X-Frop: This text\x80 contains obs-text\x81 characters\r\n" + "\r\n", + .fields = valid_header_parse_result6 + },{ + .header = + "X-Frop: This text\x01 contains invalid\x7f characters\r\n" + "\r\n", + .fields = valid_header_parse_result7 + } +}; + +static const unsigned int valid_header_parse_test_count = N_ELEMENTS(valid_header_parse_tests); + +static void test_http_header_parse_valid(void) +{ + unsigned int i; + + for (i = 0; i < valid_header_parse_test_count; i++) T_BEGIN { + struct istream *input; + struct http_header_parser *parser; + const struct http_header_limits *limits; + const char *header, *field_name, *error = NULL; + const unsigned char *field_data; + size_t field_size; + int ret; + unsigned int j, pos, header_len; + + header = valid_header_parse_tests[i].header; + header_len = strlen(header); + limits = &valid_header_parse_tests[i].limits; + input = test_istream_create_data(header, header_len); + parser = http_header_parser_init(input, limits, + valid_header_parse_tests[i].flags); + + test_begin(t_strdup_printf("http header valid [%d]", i)); + + j = 0; pos = 0; test_istream_set_size(input, 0); + while ((ret=http_header_parse_next_field + (parser, &field_name, &field_data, &field_size, &error)) >= 0) { + const struct http_header_parse_result *result; + const char *field_value; + + if (ret == 0) { + if (pos == header_len) + break; + test_istream_set_size(input, ++pos); + continue; + } + + if (field_name == NULL) break; + + result = &valid_header_parse_tests[i].fields[j]; + field_value = t_strndup(field_data, field_size); + + if (result->name == NULL) { + test_out_reason("valid", FALSE, t_strdup_printf + ("%s: %s", field_name, str_sanitize(field_value, 100))); + break; + } + + test_out_reason("valid", + strcmp(result->name, field_name) == 0 && + strcmp(result->value, field_value) == 0, + t_strdup_printf("%s: %s", field_name, + str_sanitize(field_value, 100))); + j++; + } + + test_out_reason("parse success", ret > 0, error); + test_end(); + i_stream_unref(&input); + http_header_parser_deinit(&parser); + } T_END; +} + +static const struct http_header_parse_test invalid_header_parse_tests[] = { + { + .header = + "Date: Sat, 06 Oct 2012 16:01:44 GMT\r\n" + "Server : Apache/2.2.16 (Debian)\r\n" + "Last-Modified: Mon, 30 Jul 2012 11:09:28 GMT\r\n" + "\r\n" + },{ + .header = + "Date: Sat, 06 Oct 2012 17:18:22 GMT\r\n" + "Server: Apache/2.2.3 (CentOS)\r\n" + "X Powered By: PHP/5.3.6\r\n" + "\r\n" + },{ + .header = + "Host: www.example.com\n\r" + "Accept: image/png,image/*;q=0.8,*/*;q=0.5\n\r" + "Accept-Language: en-us,en;q=0.5\n\r" + "Accept-Encoding: gzip, deflate\n\r" + "\n\r" + },{ + .header = + "Host: p5-lrqzb4yavu4l7nagydw-428649-i2-v6exp3-ds.metric.example.com\n" + "User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64; rv:15.0)\n" + "Accept:\t\timage/png,image/*;q=0.8,*/\1;q=0.5\n" + "\n", + .flags = HTTP_HEADER_PARSE_FLAG_STRICT + },{ + .header = + "Date: Sat, 06 Oct 2012 17:18:22 GMT\r\n" + "Server: Apache/2.2.3\177 (CentOS)\r\n" + "\r\n", + .flags = HTTP_HEADER_PARSE_FLAG_STRICT + },{ + .header = + "Date: Sat, 06 Oct 2012 17:12:37 GMT\r\n" + "Server: Apache/2.2.16 (Debian) PHP/5.3.3-7+squeeze14 with\r\n" + "Suhosin-Patch proxy_html/3.0.1 mod_python/3.3.1 Python/2.6.6\r\n" + "mod_ssl/2.2.16 OpenSSL/0.9.8o mod_perl/2.0.4 Perl/v5.10.1\r\n" + "\r\n" + },{ + .header = + "Date: Sat, 06 Oct 2012 17:12:37 GMT\r\n" + },{ + .header = + "Age: 58 \r\n" + "Date: Sun, 04 Aug 2013 09:33:09 GMT\r\n" + "Expires: Sun, 04 Aug 2013 09:34:08 GMT\r\n" + "Cache-Control: max-age=60 \r\n" + "Content-Length: 17336 \r\n" + "Connection: Keep-Alive\r\n" + "Via: NS-CACHE-9.3\r\n" + "Server: Apache\r\n" + "Vary: Host\r\n" + "Last-Modified: Sun, 04 Aug 2013 09:33:07 GMT\r\n" + "Content-Type: text/html; charset=utf-8\r\n" + "Content-Encoding: gzip\r\n" + "\r\n", + .limits = { .max_size = 339 } + },{ + .header = + "Age: 58 \r\n" + "Date: Sun, 04 Aug 2013 09:33:09 GMT\r\n" + "Expires: Sun, 04 Aug 2013 09:34:08 GMT\r\n" + "Cache-Control: max-age=60 \r\n" + "Content-Length: 17336 \r\n" + "Connection: Keep-Alive\r\n" + "Via: NS-CACHE-9.3\r\n" + "Server: Apache\r\n" + "Vary: Host\r\n" + "Last-Modified: Sun, 04 Aug 2013 09:33:07 GMT\r\n" + "Content-Type: text/html; charset=utf-8\r\n" + "Content-Encoding: gzip\r\n" + "\r\n", + .fields = valid_header_parse_result4, + .limits = { .max_field_size = 45 } + },{ + .header = + "Age: 58 \r\n" + "Date: Sun, 04 Aug 2013 09:33:09 GMT\r\n" + "Expires: Sun, 04 Aug 2013 09:34:08 GMT\r\n" + "Cache-Control: max-age=60 \r\n" + "Content-Length: 17336 \r\n" + "Connection: Keep-Alive\r\n" + "Via: NS-CACHE-9.3\r\n" + "Server: Apache\r\n" + "Vary: Host\r\n" + "Last-Modified: Sun, 04 Aug 2013 09:33:07 GMT\r\n" + "Content-Type: text/html; charset=utf-8\r\n" + "Content-Encoding: gzip\r\n" + "\r\n", + .fields = valid_header_parse_result4, + .limits = { .max_fields = 11 } + } +}; + +static const unsigned int invalid_header_parse_test_count = N_ELEMENTS(invalid_header_parse_tests); + +static void test_http_header_parse_invalid(void) +{ + unsigned int i; + + for (i = 0; i < invalid_header_parse_test_count; i++) T_BEGIN { + struct istream *input; + struct http_header_parser *parser; + const struct http_header_limits *limits; + const char *header, *field_name, *error = NULL; + const unsigned char *field_data; + size_t field_size; + int ret; + + header = invalid_header_parse_tests[i].header; + limits = &invalid_header_parse_tests[i].limits; + input = i_stream_create_from_data(header, strlen(header)); + parser = http_header_parser_init(input, limits, + invalid_header_parse_tests[i].flags); + + test_begin(t_strdup_printf("http header invalid [%d]", i)); + + while ((ret=http_header_parse_next_field + (parser, &field_name, &field_data, &field_size, &error)) > 0) { + if (field_name == NULL) break; + } + + test_out_reason("parse failure", ret < 0, error); + test_end(); + i_stream_unref(&input); + http_header_parser_deinit(&parser); + } T_END; +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_http_header_parse_valid, + test_http_header_parse_invalid, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-http/test-http-payload.c b/src/lib-http/test-http-payload.c new file mode 100644 index 0000000..4e63861 --- /dev/null +++ b/src/lib-http/test-http-payload.c @@ -0,0 +1,2445 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "llist.h" +#include "path-util.h" +#include "hostpid.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "istream-crlf.h" +#include "iostream-temp.h" +#include "iostream-ssl.h" +#include "iostream-ssl-test.h" +#ifdef HAVE_OPENSSL +#include "iostream-openssl.h" +#endif +#include "connection.h" +#include "test-common.h" +#include "test-subprocess.h" +#include "http-url.h" +#include "http-request.h" +#include "http-server.h" +#include "http-client.h" + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> +#include <dirent.h> + +#define CLIENT_PROGRESS_TIMEOUT 30 +#define SERVER_KILL_TIMEOUT_SECS 20 + +enum payload_handling { + PAYLOAD_HANDLING_LOW_LEVEL, + PAYLOAD_HANDLING_FORWARD, + PAYLOAD_HANDLING_HANDLER, +}; + +static bool debug = FALSE; +static bool small_socket_buffers = FALSE; +static const char *failure = NULL; +static struct timeout *to_continue = NULL; +static bool files_finished = FALSE; +static bool running_continue = FALSE; + +static struct test_settings { + /* client */ + bool client_blocking; + unsigned int max_pending; + unsigned int client_ioloop_nesting; + bool request_100_continue; + unsigned int parallel_clients; + bool parallel_clients_global; + size_t read_client_partial; + bool unknown_size; + + /* server */ + bool server_blocking; + bool server_ostream; + enum payload_handling server_payload_handling; + size_t read_server_partial; + bool server_cork; + + bool ssl; +} tset; + +static struct ip_addr bind_ip; +static in_port_t bind_port = 0; +static int fd_listen = -1; +static struct ioloop *ioloop_nested = NULL; +static unsigned ioloop_nested_first = 0; +static unsigned ioloop_nested_last = 0; +static unsigned ioloop_nested_depth = 0; + +static void main_deinit(void); + +/* + * Test settings + */ + +static void test_init_defaults(void) +{ + i_zero(&tset); + tset.max_pending = 200; + tset.server_payload_handling = PAYLOAD_HANDLING_FORWARD; + tset.parallel_clients = 1; +} + +/* + * Test files + */ +static const char unsafe_characters[] = "\"<>#%{}|\\^~[]` ;/?:@=&"; + +static ARRAY_TYPE(const_string) files; +static pool_t files_pool; + +static void test_files_read_dir(const char *path) +{ + DIR *dirp; + + /* open the directory */ + if ((dirp = opendir(path)) == NULL) { + if (errno == ENOENT || errno == EACCES) + return; + i_fatal("test files: " + "failed to open directory %s: %m", path); + } + + /* read entries */ + for (;;) { + const char *file; + struct dirent *dp; + struct stat st; + + errno = 0; + if ((dp = readdir(dirp)) == NULL) + break; + if (*dp->d_name == '.' || + dp->d_name[strcspn(dp->d_name, unsafe_characters)] != '\0') + continue; + + file = t_abspath_to(dp->d_name, path); + if (stat(file, &st) == 0) { + if (S_ISREG(st.st_mode)) { + file += 2; /* skip "./" */ + file = p_strdup(files_pool, file); + array_push_back(&files, &file); + } else if (S_ISDIR(st.st_mode)) { + test_files_read_dir(file); + } + } + } + + if (errno != 0) + i_fatal("test files: " + "failed to read directory %s: %m", path); + + /* Close the directory */ + if (closedir(dirp) < 0) + i_error("test files: " + "failed to close directory %s: %m", path); +} + +static void test_files_init(void) +{ + /* initialize file array */ + files_pool = pool_alloconly_create( + MEMPOOL_GROWING"http_server_request", 4096); + p_array_init(&files, files_pool, 512); + + /* obtain all filenames */ + test_files_read_dir("."); +} + +static void test_files_deinit(void) +{ + pool_unref(&files_pool); +} + +static struct istream * +test_file_open(const char *path, unsigned int *status_r, const char **reason_r) + ATTR_NULL(2, 3) +{ + int fd; + + if (status_r != NULL) + *status_r = 200; + if (reason_r != NULL) + *reason_r = "OK"; + + fd = open(path, O_RDONLY); + if (fd < 0) { + if (debug) + i_debug("test files: open(%s) failed: %m", path); + + switch (errno) { + case EFAULT: + case ENOENT: + if (status_r != NULL) + *status_r = 404; + if (reason_r != NULL) + *reason_r = "Not Found"; + break; + case EISDIR: + case EACCES: + if (status_r != NULL) + *status_r = 403; + if (reason_r != NULL) + *reason_r = "Forbidden"; + break; + default: + if (status_r != NULL) + *status_r = 500; + if (reason_r != NULL) + *reason_r = "Internal Server Error"; + } + return NULL; + } + + return i_stream_create_fd_autoclose(&fd, 40960); +} + +/* + * Test server + */ + +struct client { + pool_t pool; + struct client *prev, *next; + + struct http_server_connection *http_conn; +}; + +struct client_request { + struct client *client; + struct http_server_request *server_req; + + const char *path; + + struct istream *data; + struct istream *payload_input; + struct ostream *payload_output; + struct io *io; + + bool all_sent:1; +}; + +static const struct http_server_callbacks http_callbacks; +static struct http_server *http_server; + +static struct io *io_listen; +static struct client *clients; + +/* location: /succes */ + +static void client_handle_success_request(struct client_request *creq) +{ + struct http_server_request *req = creq->server_req; + const struct http_request *hreq = http_server_request_get(req); + struct http_server_response *resp; + + if (strcmp(hreq->method, "GET") != 0) { + http_server_request_fail(req, + 405, "Method Not Allowed"); + return; + } + + resp = http_server_response_create(req, 200, "OK"); + http_server_response_submit(resp); +} + +/* location: /download/... */ + +static void +client_handle_download_request(struct client_request *creq, + const char *path) +{ + struct http_server_request *req = creq->server_req; + const struct http_request *hreq = http_server_request_get(req); + struct http_server_response *resp; + const char *fpath, *reason; + struct istream *fstream; + struct ostream *output; + unsigned int status; + int ret; + + if (strcmp(hreq->method, "GET") != 0) { + http_server_request_fail(req, + 405, "Method Not Allowed"); + return; + } + + fpath = t_strconcat(".", path, NULL); + + if (debug) { + i_debug("test server: download: " + "sending payload for %s", fpath); + } + + fstream = test_file_open(fpath, &status, &reason); + if (fstream == NULL) { + http_server_request_fail(req, status, reason); + return; + } + + resp = http_server_response_create(req, 200, "OK"); + http_server_response_add_header(resp, "Content-Type", "text/plain"); + + if (tset.server_blocking) { + output = http_server_response_get_payload_output( + resp, IO_BLOCK_SIZE, TRUE); + + switch (o_stream_send_istream(output, fstream)) { + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT: + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT: + i_unreached(); + case OSTREAM_SEND_ISTREAM_RESULT_FINISHED: + /* finish it */ + ret = o_stream_finish(output); + i_assert(ret != 0); + if (ret > 0) + break; + /* fall through */ + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT: + i_assert(output->stream_errno != 0); + i_fatal("test server: download: " + "write(%s) failed: %s", + o_stream_get_name(output), + o_stream_get_error(output)); + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT: + i_assert(fstream->stream_errno != 0); + i_fatal("test server: download: " + "read(%s) failed: %s", + i_stream_get_name(fstream), + i_stream_get_error(fstream)); + } + + if (debug) { + i_debug("test server: download: " + "finished sending blocking payload for %s" + "(%"PRIuUOFF_T":%"PRIuUOFF_T")", + fpath, fstream->v_offset, output->offset); + } + + o_stream_destroy(&output); + } else { + http_server_response_set_payload(resp, fstream); + http_server_response_submit(resp); + } + i_stream_unref(&fstream); +} + +/* location: /echo */ + +static int client_request_echo_send_more(struct client_request *creq) +{ + struct ostream *output = creq->payload_output; + enum ostream_send_istream_result res; + uoff_t offset; + int ret; + + if ((ret = o_stream_flush(output)) <= 0) { + if (ret < 0) { + i_fatal("test server: echo: " + "write(%s) failed for %s (flush): %s", + o_stream_get_name(output), creq->path, + o_stream_get_error(output)); + } + return ret; + } + + if (creq->all_sent) { + if (debug) { + i_debug("test server: echo: " + "flushed all payload for %s", creq->path); + } + i_stream_unref(&creq->data); + o_stream_destroy(&creq->payload_output); + return 1; + } + + i_assert(output != NULL); + i_assert(creq->data != NULL); + + offset = creq->data->v_offset; + o_stream_set_max_buffer_size(output, IO_BLOCK_SIZE); + res = o_stream_send_istream(output, creq->data); + o_stream_set_max_buffer_size(output, SIZE_MAX); + + i_assert(creq->data->v_offset >= offset); + if (debug) { + i_debug("test server: echo: sent data for %s " + "(sent %"PRIuUOFF_T", buffered %zu)", + creq->path, (uoff_t)(creq->data->v_offset - offset), + o_stream_get_buffer_used_size(output)); + } + + switch (res) { + case OSTREAM_SEND_ISTREAM_RESULT_FINISHED: + /* finish it */ + creq->all_sent = TRUE; + if ((ret = o_stream_finish(output)) < 0) { + i_fatal("test server: echo: " + "write(%s) failed for %s (finish): %s", + o_stream_get_name(output), creq->path, + o_stream_get_error(output)); + } + if (debug) { + i_debug("test server: echo: " + "finished sending payload for %s", creq->path); + } + if (ret == 0) + return 0; + if (debug) { + i_debug("test server: echo: " + "flushed all payload for %s", creq->path); + } + i_stream_unref(&creq->data); + o_stream_destroy(&creq->payload_output); + return 1; + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT: + i_unreached(); + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT: + if (debug) { + i_debug("test server echo: " + "partially sent payload for %s", creq->path); + } + return 1; + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT: + i_fatal("test server: echo: " + "read(%s) failed for %s: %s", + i_stream_get_name(creq->data), creq->path, + i_stream_get_error(creq->data)); + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT: + i_fatal("test server: echo: " + "write(%s) failed for %s: %s", + o_stream_get_name(output), creq->path, + o_stream_get_error(output)); + } + i_unreached(); +} + +static void +client_request_echo_ostream_nonblocking(struct client_request *creq, + struct http_server_response *resp, + struct istream *data) +{ + creq->data = data; + i_stream_ref(data); + + creq->payload_output = http_server_response_get_payload_output( + resp, IO_BLOCK_SIZE, FALSE); + if (tset.server_cork) + o_stream_cork(creq->payload_output); + o_stream_set_flush_callback(creq->payload_output, + client_request_echo_send_more, creq); + o_stream_set_flush_pending(creq->payload_output, TRUE); +} + +static void +client_request_echo_blocking(struct client_request *creq, + struct http_server_response *resp, + struct istream *input) +{ + const unsigned char *data; + size_t size; + int ret; + + while ((ret = i_stream_read_more(input, &data, &size)) > 0) { + ret = http_server_response_send_payload(&resp, data, size); + i_assert(ret <= 0); + if (ret < 0) + break; + i_stream_skip(input, size); + } + i_assert(ret < 0); + if (input->stream_errno != 0) { + i_fatal("test server: echo: " + "read(%s) failed for %s: %s", + i_stream_get_name(input), creq->path, + i_stream_get_error(input)); + } else if (i_stream_have_bytes_left(input)) { + i_fatal("test server: echo: " + "failed to send all blocking payload for %s", + creq->path); + } + + /* finish it */ + if (http_server_response_finish_payload(&resp) < 0) { + i_fatal("test server: echo: " + "failed to finish blocking payload for %s", creq->path); + } + + if (debug) { + i_debug("test server: echo: " + "sent all payload for %s", creq->path); + } +} + +static void +client_request_echo_ostream_blocking(struct client_request *creq, + struct http_server_response *resp, + struct istream *input) +{ + struct ostream *payload_output; + int ret; + + payload_output = http_server_response_get_payload_output( + resp, IO_BLOCK_SIZE, TRUE); + + if (tset.server_cork) + o_stream_cork(payload_output); + + switch (o_stream_send_istream(payload_output, input)) { + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT: + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT: + i_unreached(); + case OSTREAM_SEND_ISTREAM_RESULT_FINISHED: + /* finish it */ + ret = o_stream_finish(payload_output); + i_assert(ret != 0); + if (ret > 0) + break; + /* fall through */ + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT: + i_assert(payload_output->stream_errno != 0); + i_fatal("test server: echo: " + "write(%s) failed for %s: %s", + o_stream_get_name(payload_output), creq->path, + o_stream_get_error(payload_output)); + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT: + i_assert(input->stream_errno != 0); + i_fatal("test server: echo: " + "read(%s) failed for %s: %s", + i_stream_get_name(input), creq->path, + i_stream_get_error(input)); + } + + if (debug) { + i_debug("test server: echo: " + "sent all payload for %s", creq->path); + } + + o_stream_destroy(&payload_output); +} + +static void client_request_finish_payload_in(struct client_request *creq) +{ + struct http_server_response *resp; + struct istream *payload_input; + + payload_input = + iostream_temp_finish(&creq->payload_output, 4096); + + if (debug) { + i_debug("test server: echo: " + "finished receiving payload for %s", creq->path); + } + + resp = http_server_response_create(creq->server_req, 200, "OK"); + http_server_response_add_header(resp, "Content-Type", "text/plain"); + + if (tset.server_ostream) { + client_request_echo_ostream_nonblocking(creq, resp, + payload_input); + } else { + http_server_response_set_payload(resp, payload_input); + http_server_response_submit(resp); + } + + i_stream_unref(&payload_input); +} + +static void client_request_read_echo(struct client_request *creq) +{ + enum ostream_send_istream_result res; + + o_stream_set_max_buffer_size(creq->payload_output, IO_BLOCK_SIZE); + res = o_stream_send_istream(creq->payload_output, creq->payload_input); + o_stream_set_max_buffer_size(creq->payload_output, SIZE_MAX); + + switch (res) { + case OSTREAM_SEND_ISTREAM_RESULT_FINISHED: + break; + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT: + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT: + return; + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT: + i_fatal("test server: echo: " + "Failed to read all echo payload [%s]", + creq->path); + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT: + i_fatal("test server: echo: " + "Failed to write all echo payload [%s]", + creq->path); + } + + client_request_finish_payload_in(creq); + i_stream_unref(&creq->payload_input); +} + +static void client_request_read_echo_more(struct client_request *creq) +{ + client_request_read_echo(creq); + + if (creq->payload_input != NULL) + return; + + io_remove(&creq->io); + + if (debug) { + i_debug("test server: echo: " + "finished receiving payload for %s", + creq->path); + } +} + +static void +client_handle_echo_request(struct client_request *creq, + const char *path) +{ + struct http_server_request *req = creq->server_req; + const struct http_request *hreq = http_server_request_get(req); + struct http_server_response *resp; + struct ostream *payload_output; + uoff_t size; + + creq->path = p_strdup(http_server_request_get_pool(req), path); + + if (strcmp(hreq->method, "PUT") != 0) { + http_server_request_fail(req, + 405, "Method Not Allowed"); + return; + } + + size = 0; + if (http_request_get_payload_size(hreq, &size) > 0 && size == 0) { + if (debug) { + i_debug("test server: echo: " + "empty payload for %s", creq->path); + } + + resp = http_server_response_create(creq->server_req, 200, "OK"); + http_server_response_add_header( + resp, "Content-Type", "text/plain"); + http_server_response_submit(resp); + return; + } + + payload_output = iostream_temp_create("/tmp/test-http-server", 0); + + if (tset.server_blocking) { + struct istream *payload_input; + + payload_input = + http_server_request_get_payload_input(req, TRUE); + + if (tset.read_server_partial > 0) { + struct istream *partial = + i_stream_create_limit(payload_input, + tset.read_server_partial); + i_stream_unref(&payload_input); + payload_input = partial; + } + + if (o_stream_send_istream(payload_output, payload_input) != + OSTREAM_SEND_ISTREAM_RESULT_FINISHED) { + i_fatal("test server: echo: " + "failed to receive blocking echo payload"); + } + i_stream_unref(&payload_input); + + payload_input = iostream_temp_finish(&payload_output, 4096); + + if (debug) { + i_debug("test server: echo: " + "finished receiving blocking payload for %s", + path); + } + + resp = http_server_response_create(req, 200, "OK"); + http_server_response_add_header(resp, + "Content-Type", "text/plain"); + + if (tset.server_ostream) { + client_request_echo_ostream_blocking(creq, resp, + payload_input); + } else { + client_request_echo_blocking(creq, resp, payload_input); + } + i_stream_unref(&payload_input); + } else { + creq->payload_output = payload_output; + + switch (tset.server_payload_handling) { + case PAYLOAD_HANDLING_LOW_LEVEL: + creq->payload_input = + http_server_request_get_payload_input(req, FALSE); + + if (tset.read_server_partial > 0) { + struct istream *partial = + i_stream_create_limit(creq->payload_input, + tset.read_server_partial); + i_stream_unref(&creq->payload_input); + creq->payload_input = partial; + } + + creq->io = io_add_istream(creq->payload_input, + client_request_read_echo_more, creq); + client_request_read_echo_more(creq); + break; + case PAYLOAD_HANDLING_FORWARD: + http_server_request_forward_payload(req, + payload_output, SIZE_MAX, + client_request_finish_payload_in, creq); + break; + case PAYLOAD_HANDLING_HANDLER: + creq->payload_input = + http_server_request_get_payload_input(req, FALSE); + http_server_request_handle_payload(req, + client_request_read_echo, creq); + break; + } + } +} + +/* request */ + +static void http_server_request_destroyed(struct client_request *creq); + +static struct client_request * +client_request_init(struct client *client, + struct http_server_request *req) +{ + struct client_request *creq; + pool_t pool = http_server_request_get_pool(req); + + http_server_request_ref(req); + + creq = p_new(pool, struct client_request, 1); + creq->client = client; + creq->server_req = req; + + http_server_request_set_destroy_callback(req, + http_server_request_destroyed, creq); + + return creq; +} + +static void client_request_deinit(struct client_request **_creq) +{ + struct client_request *creq = *_creq; + struct http_server_request *req = creq->server_req; + + *_creq = NULL; + + i_stream_unref(&creq->data); + i_stream_unref(&creq->payload_input); + io_remove(&creq->io); + + http_server_request_unref(&req); +} + +static void http_server_request_destroyed(struct client_request *creq) +{ + client_request_deinit(&creq); +} + +static void +client_handle_request(void *context, + struct http_server_request *req) +{ + const struct http_request *hreq = http_server_request_get(req); + const char *path = hreq->target.url->path, *p; + struct client *client = (struct client *)context; + struct client_request *creq; + + if (debug) { + i_debug("test server: request method=`%s' path=`%s'", + hreq->method, path); + } + + creq = client_request_init(client, req); + + if (strcmp(path, "/success") == 0) { + client_handle_success_request(creq); + return; + } + + if ((p = strchr(path+1, '/')) == NULL) { + http_server_request_fail(req, 404, "Not found"); + return; + } + if (strncmp(path, "/download", p-path) == 0) { + client_handle_download_request(creq, p); + return; + } + if (strncmp(path, "/echo", p-path) == 0) { + client_handle_echo_request(creq, p); + return; + } + + http_server_request_fail(req, 404, "Not found"); + return; +} + +/* client connection */ + +static void client_connection_destroy(void *context, const char *reason); + +static const struct http_server_callbacks http_callbacks = { + .connection_destroy = client_connection_destroy, + .handle_request = client_handle_request, +}; + +static void client_init(int fd) +{ + struct client *client; + pool_t pool; + + net_set_nonblock(fd, TRUE); + + pool = pool_alloconly_create("client", 512); + client = p_new(pool, struct client, 1); + client->pool = pool; + + client->http_conn = http_server_connection_create( + http_server, fd, fd, tset.ssl, &http_callbacks, client); + DLLIST_PREPEND(&clients, client); +} + +static void client_deinit(struct client **_client) +{ + struct client *client = *_client; + + *_client = NULL; + + DLLIST_REMOVE(&clients, client); + + if (client->http_conn != NULL) { + http_server_connection_close(&client->http_conn, + "deinit"); + } + pool_unref(&client->pool); +} + +static void +client_connection_destroy(void *context, const char *reason ATTR_UNUSED) +{ + struct client *client = context; + + client->http_conn = NULL; + client_deinit(&client); +} + +static void client_accept(void *context ATTR_UNUSED) +{ + int fd; + + for (;;) { + /* accept new client */ + if ((fd = net_accept(fd_listen, NULL, NULL)) < 0) { + if (errno == EAGAIN) + break; + if (errno == ECONNABORTED) + continue; + i_fatal("test server: accept() failed: %m"); + } + + client_init(fd); + } +} + +/* */ + +static void test_server_init(const struct http_server_settings *server_set) +{ + /* open server socket */ + io_listen = io_add(fd_listen, IO_READ, client_accept, NULL); + + http_server = http_server_init(server_set); +} + +static void test_server_deinit(void) +{ + /* close server socket */ + io_remove(&io_listen); + + /* deinitialize */ + http_server_deinit(&http_server); +} + +/* + * Test client + */ + +struct test_client_request { + int refcount; + + struct test_client_request *prev, *next; + struct http_client *client; + struct http_client_request *hreq; + + struct io *io; + struct istream *payload; + struct istream *file_in, *file_out; + unsigned int files_idx; +}; + +static struct http_client **http_clients; +static struct test_client_request *client_requests; +static unsigned int client_files_first, client_files_last; +struct timeout *to_client_progress = NULL; + +static struct test_client_request * +test_client_request_new(struct http_client *client) +{ + struct test_client_request *tcreq; + + tcreq = i_new(struct test_client_request, 1); + tcreq->refcount = 1; + tcreq->client = client; + DLLIST_PREPEND(&client_requests, tcreq); + + return tcreq; +} + +static void test_client_request_ref(struct test_client_request *tcreq) +{ + tcreq->refcount++; +} + +static void test_client_request_unref(struct test_client_request **_tcreq) +{ + struct test_client_request *tcreq = *_tcreq; + + *_tcreq = NULL; + + i_assert(tcreq->refcount > 0); + if (--tcreq->refcount > 0) + return; + + io_remove(&tcreq->io); + i_stream_unref(&tcreq->payload); + i_stream_unref(&tcreq->file_in); + i_stream_unref(&tcreq->file_out); + + DLLIST_REMOVE(&client_requests, tcreq); + i_free(tcreq); +} + +static void test_client_request_destroy(struct test_client_request *tcreq) +{ + test_client_request_unref(&tcreq); +} + +static void test_client_switch_ioloop(void) +{ + struct test_client_request *tcreq; + + if (to_continue != NULL) + to_continue = io_loop_move_timeout(&to_continue); + if (to_client_progress != NULL) + to_client_progress = io_loop_move_timeout(&to_client_progress); + + for (tcreq = client_requests; tcreq != NULL; + tcreq = tcreq->next) { + if (tcreq->io != NULL) + tcreq->io = io_loop_move_io(&tcreq->io); + if (tcreq->payload != NULL) + i_stream_switch_ioloop(tcreq->payload); + } +} + +static void test_client_progress_timeout(void *context ATTR_UNUSED) +{ + /* Terminate test due to lack of progress */ + failure = "Test is hanging"; + timeout_remove(&to_client_progress); + io_loop_stop(current_ioloop); +} + +static void +test_client_create_clients(const struct http_client_settings *client_set) +{ + struct http_client_context *http_context = NULL; + unsigned int i; + + if (!small_socket_buffers) { + to_client_progress = timeout_add( + CLIENT_PROGRESS_TIMEOUT*1000, + test_client_progress_timeout, NULL); + } + + if (!tset.parallel_clients_global) + http_context = http_client_context_create(client_set); + + if (tset.parallel_clients < 1) + tset.parallel_clients = 1; + http_clients = i_new(struct http_client *, tset.parallel_clients); + for (i = 0; i < tset.parallel_clients; i++) { + http_clients[i] = (tset.parallel_clients_global ? + http_client_init(client_set) : + http_client_init_shared(http_context, NULL)); + } + + if (!tset.parallel_clients_global) + http_client_context_unref(&http_context); +} + +/* download */ + +static void test_client_download_continue(void); + +static void test_client_download_finished(struct test_client_request *tcreq) +{ + const char **paths; + unsigned int files_idx = tcreq->files_idx; + unsigned int count; + + paths = array_get_modifiable(&files, &count); + i_assert(files_idx < count); + i_assert(client_files_first < count); + i_assert(paths[files_idx] != NULL); + + paths[files_idx] = NULL; + test_client_download_continue(); +} + +static void +test_client_download_payload_input(struct test_client_request *tcreq) +{ + struct istream *payload = tcreq->payload; + const unsigned char *pdata, *fdata; + size_t psize, fsize, pleft; + off_t ret; + + if (to_client_progress != NULL) + timeout_reset(to_client_progress); + + /* read payload */ + while ((ret = i_stream_read_more(payload, &pdata, &psize)) > 0) { + if (debug) { + i_debug("test client: download: " + "got data for [%u] (size=%d)", + tcreq->files_idx, (int)psize); + } + /* compare with file on disk */ + pleft = psize; + while ((ret = i_stream_read_more(tcreq->file_in, + &fdata, &fsize)) > 0 && + pleft > 0) { + fsize = (fsize > pleft ? pleft : fsize); + if (memcmp(pdata, fdata, fsize) != 0) { + i_fatal("test client: download: " + "received data does not match file " + "(%"PRIuUOFF_T":%"PRIuUOFF_T")", + payload->v_offset, + tcreq->file_in->v_offset); + } + i_stream_skip(tcreq->file_in, fsize); + pleft -= fsize; + pdata += fsize; + } + if (ret < 0 && tcreq->file_in->stream_errno != 0) { + i_fatal("test client: download: " + "failed to read file: %s", + i_stream_get_error(tcreq->file_in)); + } + i_stream_skip(payload, psize); + } + + if (ret == 0) { + if (debug) { + i_debug("test client: download: " + "need more data for [%u]", + tcreq->files_idx); + } + /* we will be called again for this request */ + } else { + (void)i_stream_read(tcreq->file_in); + if (payload->stream_errno != 0) { + i_fatal("test client: download: " + "failed to read request payload: %s", + i_stream_get_error(payload)); + } if (i_stream_have_bytes_left(tcreq->file_in)) { + if (i_stream_read_more(tcreq->file_in, + &fdata, &fsize) <= 0) + fsize = 0; + i_fatal("test client: download: " + "payload ended prematurely " + "(at least %zu bytes left)", fsize); + } else if (debug) { + i_debug("test client: download: " + "finished request for [%u]", + tcreq->files_idx); + } + + /* finished */ + tcreq->payload = NULL; + test_client_download_finished(tcreq); + + /* dereference payload stream; finishes the request */ + i_stream_unref(&tcreq->file_in); + io_remove(&tcreq->io); /* holds a reference too */ + i_stream_unref(&payload); + } +} + +static void +test_client_download_response(const struct http_response *resp, + struct test_client_request *tcreq) +{ + const char **paths; + const char *path; + unsigned int count, status; + struct istream *fstream; + const char *reason; + + if (debug) { + i_debug("test client: download: got response for [%u]", + tcreq->files_idx); + } + + if (to_client_progress != NULL) + timeout_reset(to_client_progress); + + paths = array_get_modifiable(&files, &count); + i_assert(tcreq->files_idx < count); + i_assert(client_files_first < count); + path = paths[tcreq->files_idx]; + i_assert(path != NULL); + + if (debug) { + i_debug("test client: download: path for [%u]: %s", + tcreq->files_idx, path); + } + + fstream = test_file_open(path, &status, &reason); + i_assert(fstream != NULL); + + if (status != resp->status) { + i_fatal("test client: download: " + "got wrong response for %s: %u %s " + "(expected: %u %s)", path, + resp->status, resp->reason, status, reason); + } + + if (resp->status / 100 != 2) { + if (debug) { + i_debug("test client: download: " + "HTTP request for %s failed: %u %s", + path, resp->status, resp->reason); + } + i_stream_unref(&fstream); + test_client_download_finished(tcreq); + return; + } + + if (resp->payload == NULL) { + if (debug) { + i_debug("test client: download: " + "no payload for %s [%u]", + path, tcreq->files_idx); + } + i_stream_unref(&fstream); + test_client_download_finished(tcreq); + return; + } + + i_assert(fstream != NULL); + if (tset.read_client_partial == 0) { + i_stream_ref(resp->payload); + tcreq->payload = resp->payload; + tcreq->file_in = fstream; + } else { + struct istream *payload = resp->payload; + tcreq->payload = i_stream_create_limit( + payload, tset.read_client_partial); + tcreq->file_in = i_stream_create_limit( + fstream, tset.read_client_partial); + i_stream_unref(&fstream); + } + + tcreq->io = io_add_istream(tcreq->payload, + test_client_download_payload_input, tcreq); + test_client_download_payload_input(tcreq); +} + +static void test_client_download_continue(void) +{ + struct test_client_request *tcreq; + struct http_client_request *hreq; + const char *const *paths; + unsigned int count; + + paths = array_get(&files, &count); + i_assert(client_files_first <= count); + i_assert(client_files_last <= count); + + i_assert(client_files_first <= client_files_last); + for (; client_files_first < client_files_last && + paths[client_files_first] == NULL; client_files_first++) + + if (debug) { + i_debug("test client: download: received until [%u]", + client_files_first-1); + } + + if (client_files_first >= count) { + io_loop_stop(current_ioloop); + return; + } + + for (; (client_files_last < count && + (client_files_last - client_files_first) < tset.max_pending); + client_files_last++) { + struct http_client *http_client = + http_clients[client_files_last % tset.parallel_clients]; + const char *path = paths[client_files_last]; + + tcreq = test_client_request_new(http_client); + tcreq->files_idx = client_files_last; + + if (debug) { + i_debug("test client: download: retrieving %s [%u]", + path, tcreq->files_idx); + } + hreq = tcreq->hreq = http_client_request( + http_client, "GET", net_ip2addr(&bind_ip), + t_strconcat("/download/", path, NULL), + test_client_download_response, tcreq); + http_client_request_set_port(hreq, bind_port); + http_client_request_set_ssl(hreq, tset.ssl); + http_client_request_set_destroy_callback( + hreq, test_client_request_destroy, tcreq); + http_client_request_submit(hreq); + } +} + +static void test_client_download(const struct http_client_settings *client_set) +{ + /* create client(s) */ + test_client_create_clients(client_set); + + /* start querying server */ + client_files_first = client_files_last = 0; + test_client_download_continue(); +} + +/* echo */ + +static void test_client_echo_continue(void *context); + +static void test_client_echo_finished(struct test_client_request *tcreq) +{ + unsigned int files_idx = tcreq->files_idx; + const char **paths; + unsigned int count; + + paths = array_get_modifiable(&files, &count); + i_assert(files_idx < count); + i_assert(client_files_first < count); + i_assert(paths[files_idx] != NULL); + + if (tcreq->file_out != NULL) + return; + if (tcreq->file_in != NULL) + return; + + if (debug) { + i_debug("test client: echo: finished [%u]: %s", + files_idx, paths[files_idx]); + } + + paths[files_idx] = NULL; + files_finished = TRUE; + if (!running_continue && to_continue == NULL) { + to_continue = timeout_add_short(0, + test_client_echo_continue, NULL); + } +} + +static void test_client_echo_payload_input(struct test_client_request *tcreq) +{ + struct istream *payload = tcreq->payload; + const unsigned char *pdata, *fdata; + size_t psize, fsize, pleft; + off_t ret; + + if (to_client_progress != NULL) + timeout_reset(to_client_progress); + + /* read payload */ + while ((ret = i_stream_read_more(payload, &pdata, &psize)) > 0) { + if (debug) { + i_debug("test client: echo: " + "got data for [%u] (size=%d)", + tcreq->files_idx, (int)psize); + } + /* compare with file on disk */ + pleft = psize; + while ((ret = i_stream_read_more(tcreq->file_in, + &fdata, &fsize)) > 0 && + pleft > 0) { + fsize = (fsize > pleft ? pleft : fsize); + if (memcmp(pdata, fdata, fsize) != 0) { + i_fatal("test client: echo: " + "received data does not match file " + "(%"PRIuUOFF_T":%"PRIuUOFF_T")", + payload->v_offset, + tcreq->file_in->v_offset); + } + i_stream_skip(tcreq->file_in, fsize); + pleft -= fsize; + pdata += fsize; + } + if (ret < 0 && tcreq->file_in->stream_errno != 0) { + i_fatal("test client: echo: " + "failed to read file: %s", + i_stream_get_error(tcreq->file_in)); + } + i_stream_skip(payload, psize); + } + + if (ret == 0) { + if (debug) { + i_debug("test client: echo: " + "need more data for [%u]", + tcreq->files_idx); + } + /* we will be called again for this request */ + } else { + (void)i_stream_read(tcreq->file_in); + if (payload->stream_errno != 0) { + i_fatal("test client: echo: " + "failed to read request payload: %s", + i_stream_get_error(payload)); + } if (i_stream_have_bytes_left(tcreq->file_in)) { + if (i_stream_read_more(tcreq->file_in, + &fdata, &fsize) <= 0) + fsize = 0; + i_fatal("test client: echo: " + "payload ended prematurely " + "(at least %zu bytes left)", fsize); + } else if (debug) { + i_debug("test client: echo: " + "finished request for [%u]", + tcreq->files_idx); + } + + /* finished */ + tcreq->payload = NULL; + i_stream_unref(&tcreq->file_in); + test_client_echo_finished(tcreq); + + /* dereference payload stream; finishes the request */ + io_remove(&tcreq->io); /* holds a reference too */ + i_stream_unref(&payload); + } +} + +static void +test_client_echo_response(const struct http_response *resp, + struct test_client_request *tcreq) +{ + const char **paths; + const char *path; + unsigned int count, status; + struct istream *fstream; + + if (debug) { + i_debug("test client: echo: got response for [%u]", + tcreq->files_idx); + } + + if (to_client_progress != NULL) + timeout_reset(to_client_progress); + + paths = array_get_modifiable(&files, &count); + i_assert(tcreq->files_idx < count); + i_assert(client_files_first < count); + path = paths[tcreq->files_idx]; + i_assert(path != NULL); + + if (debug) { + i_debug("test client: echo: path for [%u]: %s", + tcreq->files_idx, path); + } + + if (resp->status / 100 != 2) { + i_fatal("test client: echo: " + "HTTP request for %s failed: %u %s", + path, resp->status, resp->reason); + } + + fstream = test_file_open(path, &status, NULL); + if (fstream == NULL) { + i_fatal("test client: echo: failed to open %s", path); + } + + if (tset.unknown_size) { + struct istream *ustream; + + ustream = i_stream_create_crlf(fstream); + i_stream_unref(&fstream); + fstream = ustream; + } + + if (tset.read_server_partial > 0) { + struct istream *partial = + i_stream_create_limit(fstream, tset.read_server_partial); + i_stream_unref(&fstream); + fstream = partial; + } + + if (resp->payload == NULL) { + // FIXME: check file is empty + if (debug) { + i_debug("test client: echo: " + "no payload for %s [%u]", + path, tcreq->files_idx); + } + i_stream_unref(&fstream); + test_client_echo_finished(tcreq); + return; + } + + i_assert(fstream != NULL); + tcreq->file_in = fstream; + + i_stream_ref(resp->payload); + tcreq->payload = resp->payload; + tcreq->io = io_add_istream(resp->payload, + test_client_echo_payload_input, tcreq); + test_client_echo_payload_input(tcreq); +} + +static void +test_client_echo_nonblocking(struct test_client_request *tcreq ATTR_UNUSED, + struct http_client_request *hreq, + struct istream *fstream) +{ + http_client_request_set_payload(hreq, fstream, + tset.request_100_continue); + http_client_request_submit(hreq); +} + +static void +test_client_echo_blocking(struct test_client_request *tcreq, + struct http_client_request *hreq, + struct istream *fstream) +{ + const unsigned char *data; + size_t size; + int ret; + + test_client_request_ref(tcreq); + tcreq->file_out = fstream; + + while ((ret = i_stream_read_more(fstream, &data, &size)) > 0) { + ret = http_client_request_send_payload(&hreq, data, size); + i_assert(ret <= 0); + if (ret < 0) + break; + i_stream_skip(fstream, size); + } + i_assert(ret < 0); + if (fstream->stream_errno != 0) { + i_fatal("test client: echo: " + "read(%s) failed: %s [%u]", + i_stream_get_name(fstream), + i_stream_get_error(fstream), + tcreq->files_idx); + } else if (i_stream_have_bytes_left(fstream)) { + i_fatal("test client: echo: " + "failed to send all blocking payload [%u]", + tcreq->files_idx); + } + + /* finish it */ + if (http_client_request_finish_payload(&hreq) < 0) { + i_fatal("test client: echo: " + "failed to finish blocking payload [%u]", + tcreq->files_idx); + } + http_client_wait(tcreq->client); + + if (debug) { + i_debug("test client: echo: " + "sent all payload [%u]", + tcreq->files_idx); + } + + tcreq->file_out = NULL; + test_client_echo_finished(tcreq); + test_client_request_unref(&tcreq); +} + +static void test_client_echo_continue(void *context ATTR_UNUSED) +{ + struct test_client_request *tcreq; + struct http_client_request *hreq; + const char **paths; + unsigned int count, first_submitted; + bool prev_files_finished = files_finished; + + running_continue = TRUE; + files_finished = FALSE; + timeout_remove(&to_continue); + + paths = array_get_modifiable(&files, &count); + + i_assert(client_files_first <= count); + i_assert(client_files_last <= count); + + i_assert(client_files_first <= client_files_last); + for (; client_files_first < client_files_last && + paths[client_files_first] == NULL; client_files_first++); + + if (debug) { + i_debug("test client: echo: received until [%u/%u]", + client_files_first-1, count); + } + + if (debug && client_files_first < count) { + const char *path = paths[client_files_first]; + i_debug("test client: echo: next blocking: %s [%d]", + (path == NULL ? "none" : path), client_files_first); + } + + if (client_files_first >= count || failure != NULL) { + running_continue = FALSE; + files_finished = prev_files_finished; + io_loop_stop(current_ioloop); + return; + } + + first_submitted = client_files_last; + for (; (client_files_last < count && + (client_files_last - client_files_first) < tset.max_pending); + client_files_last++) { + struct http_client *http_client = + http_clients[client_files_last % tset.parallel_clients]; + struct istream *fstream; + const char *path = paths[client_files_last]; + + fstream = test_file_open(path, NULL, NULL); + if (fstream == NULL) { + paths[client_files_last] = NULL; + if (debug) { + i_debug("test client: echo: " + "skipping %s [%u]", + path, client_files_last); + } + continue; + } + + if (debug) { + i_debug("test client: echo: retrieving %s [%u]", + path, client_files_last); + } + + if (tset.unknown_size) { + struct istream *ustream; + + ustream = i_stream_create_crlf(fstream); + i_stream_unref(&fstream); + fstream = ustream; + } + + tcreq = test_client_request_new(http_client); + tcreq->files_idx = client_files_last; + + hreq = tcreq->hreq = http_client_request(http_client, + "PUT", net_ip2addr(&bind_ip), + t_strconcat("/echo/", path, NULL), + test_client_echo_response, tcreq); + http_client_request_set_port(hreq, bind_port); + http_client_request_set_ssl(hreq, tset.ssl); + http_client_request_set_destroy_callback(hreq, + test_client_request_destroy, tcreq); + + if (!tset.client_blocking) + test_client_echo_nonblocking(tcreq, hreq, fstream); + else + test_client_echo_blocking(tcreq, hreq, fstream); + i_stream_unref(&fstream); + + if (tset.client_blocking && paths[client_files_last] != NULL) { + running_continue = FALSE; + files_finished = prev_files_finished; + return; + } + } + + if (files_finished && to_continue == NULL) { + to_continue = timeout_add_short( + 0, test_client_echo_continue, NULL); + } + running_continue = FALSE; + files_finished = prev_files_finished; + + /* run nested ioloop (if requested) if new requests cross a nesting + boundary */ + if (ioloop_nested != NULL) { + unsigned int i; + + i_assert(ioloop_nested_first <= count); + i_assert(ioloop_nested_last <= count); + for (i = ioloop_nested_first; i < ioloop_nested_last; i++) { + if (paths[i] != NULL) { + if (debug) { + i_debug("test client: " + "not leaving ioloop [%u]", i); + } + break; + } + } + + if (i == ioloop_nested_last) + io_loop_stop(ioloop_nested); + } else if (tset.client_ioloop_nesting > 0 && + ((client_files_last / tset.client_ioloop_nesting) != + (first_submitted / tset.client_ioloop_nesting))) { + struct ioloop *prev_ioloop = current_ioloop; + unsigned int i; + + ioloop_nested_first = first_submitted; + ioloop_nested_last = + first_submitted + tset.client_ioloop_nesting; + if (ioloop_nested_last > client_files_last) + ioloop_nested_last = client_files_last; + + if (debug) { + i_debug("test client: " + "echo: entering ioloop for %u...%u (depth=%u)", + ioloop_nested_first, ioloop_nested_last, + ioloop_nested_depth); + } + + ioloop_nested_depth++; + + ioloop_nested = io_loop_create(); + for (i = 0; i < tset.parallel_clients; i++) + http_client_switch_ioloop(http_clients[i]); + test_client_switch_ioloop(); + + io_loop_run(ioloop_nested); + + io_loop_set_current(prev_ioloop); + for (i = 0; i < tset.parallel_clients; i++) + http_client_switch_ioloop(http_clients[i]); + test_client_switch_ioloop(); + io_loop_set_current(ioloop_nested); + io_loop_destroy(&ioloop_nested); + ioloop_nested = NULL; + + ioloop_nested_depth--; + + if (debug) { + i_debug("test client: echo: leaving ioloop for %u...%u " + "(depth=%u)", ioloop_nested_first, + ioloop_nested_last, ioloop_nested_depth); + } + ioloop_nested_first = ioloop_nested_last = 0; + + if (client_files_first >= count || failure != NULL) { + io_loop_stop(current_ioloop); + return; + } + } +} + +static void test_client_echo(const struct http_client_settings *client_set) +{ + /* create client */ + test_client_create_clients(client_set); + + /* start querying server */ + client_files_first = client_files_last = 0; + + i_assert(to_continue == NULL); + to_continue = timeout_add_short(0, test_client_echo_continue, NULL); +} + +/* cleanup */ + +static void test_client_deinit(void) +{ + unsigned int i; + + for (i = 0; i < tset.parallel_clients; i++) + http_client_deinit(&http_clients[i]); + i_free(http_clients); + + tset.parallel_clients = 1; + + timeout_remove(&to_continue); + timeout_remove(&to_client_progress); +} + +/* + * Tests + */ + +struct test_server_data { + const struct http_server_settings *set; +}; + +static void test_open_server_fd(void) +{ + i_close_fd(&fd_listen); + fd_listen = net_listen(&bind_ip, &bind_port, 128); + if (fd_listen == -1) { + i_fatal("listen(%s:%u) failed: %m", + net_ip2addr(&bind_ip), bind_port); + } + net_set_nonblock(fd_listen, TRUE); +} + +static int test_run_server(struct test_server_data *data) +{ + const struct http_server_settings *server_set = data->set; + struct ioloop *ioloop; + + i_set_failure_prefix("SERVER: "); + + if (debug) + i_debug("PID=%s", my_pid); + + ioloop_nested = NULL; + ioloop_nested_depth = 0; + ioloop = io_loop_create(); + test_server_init(server_set); + io_loop_run(ioloop); + test_server_deinit(); + io_loop_destroy(&ioloop); + + if (debug) + i_debug("Terminated"); + + i_close_fd(&fd_listen); + test_files_deinit(); + main_deinit(); + return 0; +} + +static void +test_run_client( + const struct http_client_settings *client_set, + void (*client_init)(const struct http_client_settings *client_set)) +{ + struct ioloop *ioloop; + + i_set_failure_prefix("CLIENT: "); + + if (debug) + i_debug("PID=%s", my_pid); + + ioloop_nested = NULL; + ioloop_nested_depth = 0; + ioloop = io_loop_create(); + client_init(client_set); + io_loop_run(ioloop); + test_client_deinit(); + io_loop_destroy(&ioloop); + + if (debug) + i_debug("Terminated"); +} + +static void +test_run_client_server( + const struct http_client_settings *client_set, + const struct http_server_settings *server_set, + void (*client_init)(const struct http_client_settings *client_set)) +{ + struct test_server_data data; + + failure = NULL; + + test_files_init(); + + i_zero(&data); + data.set = server_set; + + /* Fork server */ + test_open_server_fd(); + test_subprocess_fork(test_run_server, &data, FALSE); + i_close_fd(&fd_listen); + + /* Run client */ + test_run_client(client_set, client_init); + + i_unset_failure_prefix(); + test_subprocess_kill_all(SERVER_KILL_TIMEOUT_SECS); + test_files_deinit(); +} + +static void +test_init_server_settings(struct http_server_settings *server_set_r) +{ + i_zero(server_set_r); + server_set_r->request_limits.max_payload_size = UOFF_T_MAX; + server_set_r->debug = debug; + + if (small_socket_buffers) { + server_set_r->socket_send_buffer_size = 40960; + server_set_r->socket_recv_buffer_size = 40960; + } +} + +static void +test_init_client_settings(struct http_client_settings *client_set_r) +{ + i_zero(client_set_r); + client_set_r->max_redirects = 0; + client_set_r->max_attempts = 1; + client_set_r->max_idle_time_msecs = 5* 1000; + client_set_r->debug = debug; + + if (small_socket_buffers) { + client_set_r->socket_send_buffer_size = 40960; + client_set_r->socket_recv_buffer_size = 40960; + client_set_r->request_timeout_msecs = 20 * 60 * 1000; + client_set_r->connect_timeout_msecs = 20 * 60 * 1000; + } +} + +static void +test_run_sequential( + void (*client_init)(const struct http_client_settings *client_set)) +{ + struct http_server_settings http_server_set; + struct http_client_settings http_client_set; + struct ssl_iostream_settings ssl_server_set, ssl_client_set; + + /* download files from blocking server */ + + /* ssl settings */ + ssl_iostream_test_settings_server(&ssl_server_set); + ssl_server_set.verbose = debug; + ssl_iostream_test_settings_client(&ssl_client_set); + ssl_client_set.verbose = debug; + + /* server settings */ + test_init_server_settings(&http_server_set); + http_server_set.ssl = &ssl_server_set; + http_server_set.max_pipelined_requests = 0; + + /* client settings */ + test_init_client_settings(&http_client_set); + http_client_set.ssl = &ssl_client_set; + http_client_set.max_parallel_connections = 1; + http_client_set.max_pipelined_requests = 1; + + test_run_client_server(&http_client_set, &http_server_set, client_init); + + test_out_reason("sequential", (failure == NULL), failure); +} + +static void +test_run_pipeline( + void (*client_init)(const struct http_client_settings *client_set)) +{ + struct http_server_settings http_server_set; + struct http_client_settings http_client_set; + struct ssl_iostream_settings ssl_server_set, ssl_client_set; + + /* download files from blocking server */ + + /* ssl settings */ + ssl_iostream_test_settings_server(&ssl_server_set); + ssl_server_set.verbose = debug; + ssl_iostream_test_settings_client(&ssl_client_set); + ssl_client_set.verbose = debug; + + /* server settings */ + test_init_server_settings(&http_server_set); + http_server_set.ssl = &ssl_server_set; + http_server_set.max_pipelined_requests = 4; + + /* client settings */ + test_init_client_settings(&http_client_set); + http_client_set.ssl = &ssl_client_set; + http_client_set.max_parallel_connections = 1; + http_client_set.max_pipelined_requests = 8; + + test_run_client_server(&http_client_set, &http_server_set, client_init); + + test_out_reason("pipeline", (failure == NULL), failure); +} + +static void +test_run_parallel( + void (*client_init)(const struct http_client_settings *client_set)) +{ + struct http_server_settings http_server_set; + struct http_client_settings http_client_set; + struct ssl_iostream_settings ssl_server_set, ssl_client_set; + + /* download files from blocking server */ + + /* ssl settings */ + ssl_iostream_test_settings_server(&ssl_server_set); + ssl_server_set.verbose = debug; + ssl_iostream_test_settings_client(&ssl_client_set); + ssl_client_set.verbose = debug; + + /* server settings */ + test_init_server_settings(&http_server_set); + http_server_set.ssl = &ssl_server_set; + http_server_set.max_pipelined_requests = 4; + + /* client settings */ + test_init_client_settings(&http_client_set); + http_client_set.ssl = &ssl_client_set; + http_client_set.max_parallel_connections = 40; + http_client_set.max_pipelined_requests = 8; + + test_run_client_server(&http_client_set, &http_server_set, client_init); + + test_out_reason("parallel", (failure == NULL), failure); +} + +static void test_download_server_nonblocking(void) +{ + test_begin("http payload download (server non-blocking)"); + test_init_defaults(); + test_run_sequential(test_client_download); + test_run_pipeline(test_client_download); + test_run_parallel(test_client_download); + test_end(); +} + +static void test_download_server_blocking(void) +{ + test_begin("http payload download (server blocking)"); + test_init_defaults(); + tset.server_blocking = TRUE; + test_run_sequential(test_client_download); + test_run_pipeline(test_client_download); + test_run_parallel(test_client_download); + test_end(); +} + +static void test_echo_server_nonblocking(void) +{ + test_begin("http payload echo " + "(server non-blocking)"); + test_init_defaults(); + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo " + "(server non-blocking; low-level)"); + test_init_defaults(); + tset.server_payload_handling = PAYLOAD_HANDLING_LOW_LEVEL; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo " + "(server non-blocking; handler)"); + test_init_defaults(); + tset.server_payload_handling = PAYLOAD_HANDLING_HANDLER; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo " + "(server non-blocking; size unknown)"); + test_init_defaults(); + tset.unknown_size = TRUE; + tset.server_payload_handling = PAYLOAD_HANDLING_FORWARD; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo " + "(server non-blocking; ostream)"); + test_init_defaults(); + tset.server_ostream = TRUE; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo " + "(server non-blocking; ostream; cork)"); + test_init_defaults(); + tset.server_ostream = TRUE; + tset.server_cork = TRUE; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); +} + +static void test_echo_server_blocking(void) +{ + test_begin("http payload echo (server blocking)"); + test_init_defaults(); + tset.server_blocking = TRUE; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo (server blocking; ostream)"); + test_init_defaults(); + tset.server_blocking = TRUE; + tset.server_ostream = TRUE; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo (server blocking; ostream; cork)"); + test_init_defaults(); + tset.server_ostream = TRUE; + tset.server_blocking = TRUE; + tset.server_cork = TRUE; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); +} + +static void test_echo_server_nonblocking_sync(void) +{ + test_begin("http payload echo " + "(server non-blocking; 100-continue)"); + test_init_defaults(); + tset.request_100_continue = TRUE; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo " + "(server non-blocking; 100-continue; low-level)"); + test_init_defaults(); + tset.request_100_continue = TRUE; + tset.server_payload_handling = PAYLOAD_HANDLING_LOW_LEVEL; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo " + "(server non-blocking; 100-continue; handler)"); + test_init_defaults(); + tset.request_100_continue = TRUE; + tset.server_payload_handling = PAYLOAD_HANDLING_HANDLER; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); +} + +static void test_echo_server_blocking_sync(void) +{ + test_begin("http payload echo (server blocking; 100-continue)"); + test_init_defaults(); + tset.server_blocking = TRUE; + tset.request_100_continue = TRUE; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo " + "(server blocking; ostream; 100-continue)"); + test_init_defaults(); + tset.server_blocking = TRUE; + tset.server_ostream = TRUE; + tset.request_100_continue = TRUE; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); +} + +static void test_echo_server_nonblocking_partial(void) +{ + test_begin("http payload echo " + "(server non-blocking; partial short)"); + test_init_defaults(); + tset.read_server_partial = 1024; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + test_begin("http payload echo " + "(server non-blocking; partial long)"); + test_init_defaults(); + tset.read_server_partial = IO_BLOCK_SIZE + 1024; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo (server non-blocking; " + "partial short; low-level)"); + test_init_defaults(); + tset.server_payload_handling = PAYLOAD_HANDLING_LOW_LEVEL; + tset.read_server_partial = 1024; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + test_begin("http payload echo " + "(server non-blocking; partial long; low-level)"); + test_init_defaults(); + tset.server_payload_handling = PAYLOAD_HANDLING_LOW_LEVEL; + tset.read_server_partial = IO_BLOCK_SIZE + 1024; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo " + "(server non-blocking; partial short; handler)"); + test_init_defaults(); + tset.server_payload_handling = PAYLOAD_HANDLING_HANDLER; + tset.read_server_partial = 1024; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + test_begin("http payload echo " + "(server non-blocking; partial long; handler)"); + test_init_defaults(); + tset.server_payload_handling = PAYLOAD_HANDLING_HANDLER; + tset.read_server_partial = IO_BLOCK_SIZE + 1024; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo " + "(server non-blocking; partial short; ostream)"); + test_init_defaults(); + tset.server_ostream = TRUE; + tset.read_server_partial = 1024; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + test_begin("http payload echo " + "(server non-blocking; partial long; ostream)"); + test_init_defaults(); + tset.server_ostream = TRUE; + tset.read_server_partial = IO_BLOCK_SIZE + 1024; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo " + "(server non-blocking; partial short; ostream; corked)"); + test_init_defaults(); + tset.server_ostream = TRUE; + tset.server_cork = TRUE; + tset.read_server_partial = 1024; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + test_begin("http payload echo " + "(server non-blocking; partial long; ostream; corked)"); + test_init_defaults(); + tset.server_ostream = TRUE; + tset.server_cork = TRUE; + tset.read_server_partial = IO_BLOCK_SIZE + 1024; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); +} + +static void test_echo_server_blocking_partial(void) +{ + test_begin("http payload echo (server blocking; partial short)"); + test_init_defaults(); + tset.server_blocking = TRUE; + tset.read_server_partial = 1024; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + test_begin("http payload echo (server blocking; partial long)"); + test_init_defaults(); + tset.server_blocking = TRUE; + tset.read_server_partial = IO_BLOCK_SIZE + 1024; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo " + "(server blocking; partial short; ostream; cork)"); + test_init_defaults(); + tset.server_blocking = TRUE; + tset.server_ostream = TRUE; + tset.server_cork = TRUE; + tset.read_server_partial = 1024; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + test_begin("http payload echo " + "(server blocking; partial long; ostream; cork)"); + test_init_defaults(); + tset.server_blocking = TRUE; + tset.server_ostream = TRUE; + tset.server_cork = TRUE; + tset.read_server_partial = IO_BLOCK_SIZE + 1024; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); +} + +static void test_download_client_partial(void) +{ + test_begin("http payload download (client partial)"); + test_init_defaults(); + tset.read_client_partial = 1024; + test_run_sequential(test_client_download); + test_run_pipeline(test_client_download); + test_run_parallel(test_client_download); + test_end(); + test_begin("http payload download (client partial long)"); + test_init_defaults(); + tset.read_client_partial = IO_BLOCK_SIZE + 1024; + test_run_sequential(test_client_download); + test_run_pipeline(test_client_download); + test_run_parallel(test_client_download); + test_end(); +} + +static void test_download_client_nested_ioloop(void) +{ + test_begin("http payload echo (client nested ioloop)"); + test_init_defaults(); + tset.client_ioloop_nesting = 10; + test_run_parallel(test_client_echo); + test_end(); +} + +static void test_echo_client_shared(void) +{ + test_begin("http payload download " + "(server non-blocking; client shared)"); + test_init_defaults(); + tset.parallel_clients = 4; + test_run_sequential(test_client_download); + tset.parallel_clients = 4; + test_run_pipeline(test_client_download); + tset.parallel_clients = 4; + test_run_parallel(test_client_download); + test_end(); + + test_begin("http payload download " + "(server blocking; client shared)"); + test_init_defaults(); + tset.server_blocking = TRUE; + tset.parallel_clients = 4; + test_run_sequential(test_client_download); + tset.parallel_clients = 4; + test_run_pipeline(test_client_download); + tset.parallel_clients = 4; + test_run_parallel(test_client_download); + test_end(); + + test_begin("http payload echo " + "(server non-blocking; client shared)"); + test_init_defaults(); + tset.parallel_clients = 4; + test_run_sequential(test_client_echo); + tset.parallel_clients = 4; + test_run_pipeline(test_client_echo); + tset.parallel_clients = 4; + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo " + "(server blocking; client shared)"); + test_init_defaults(); + tset.server_blocking = TRUE; + tset.server_ostream = TRUE; + tset.parallel_clients = 4; + test_run_sequential(test_client_echo); + tset.parallel_clients = 4; + test_run_pipeline(test_client_echo); + tset.parallel_clients = 4; + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo " + "(server non-blocking; client global)"); + test_init_defaults(); + tset.parallel_clients = 4; + tset.parallel_clients_global = TRUE; + test_run_sequential(test_client_echo); + tset.parallel_clients = 4; + tset.parallel_clients_global = TRUE; + test_run_pipeline(test_client_echo); + tset.parallel_clients = 4; + tset.parallel_clients_global = TRUE; + test_run_parallel(test_client_echo); + test_end(); +} + +#ifdef HAVE_OPENSSL +static void test_echo_ssl(void) +{ + test_begin("http payload echo (ssl)"); + test_init_defaults(); + tset.ssl = TRUE; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo (ssl; unknown size)"); + test_init_defaults(); + tset.unknown_size = TRUE; + tset.ssl = TRUE; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo (ssl; server ostream, cork)"); + test_init_defaults(); + tset.ssl = TRUE; + tset.server_ostream = TRUE; + tset.server_cork = TRUE; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); +} +#endif + +static void test_echo_client_blocking(void) +{ + test_begin("http payload echo (client blocking)"); + test_init_defaults(); + tset.client_blocking = TRUE; + test_run_sequential(test_client_echo); + test_run_pipeline(test_client_echo); + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo (client blocking; client shared)"); + test_init_defaults(); + tset.client_blocking = TRUE; + tset.parallel_clients = 4; + test_run_sequential(test_client_echo); + tset.parallel_clients = 4; + test_run_pipeline(test_client_echo); + tset.parallel_clients = 4; + test_run_parallel(test_client_echo); + test_end(); + + test_begin("http payload echo (client blocking; client global)"); + test_init_defaults(); + tset.client_blocking = TRUE; + tset.parallel_clients = 4; + tset.parallel_clients_global = TRUE; + test_run_sequential(test_client_echo); + tset.parallel_clients = 4; + tset.parallel_clients_global = TRUE; + test_run_pipeline(test_client_echo); + tset.parallel_clients = 4; + tset.parallel_clients_global = TRUE; + test_run_parallel(test_client_echo); + test_end(); +} + +static void (*const test_functions[])(void) = { + test_download_server_nonblocking, + test_download_server_blocking, + test_echo_server_nonblocking, + test_echo_server_blocking, + test_echo_server_nonblocking_sync, + test_echo_server_blocking_sync, + test_echo_server_nonblocking_partial, + test_echo_server_blocking_partial, + test_download_client_partial, + test_download_client_nested_ioloop, + test_echo_client_shared, +#ifdef HAVE_OPENSSL + test_echo_ssl, +#endif + test_echo_client_blocking, + NULL +}; + +/* + * Main + */ + +static void main_init(void) +{ +#ifdef HAVE_OPENSSL + ssl_iostream_openssl_init(); +#endif +} + +static void main_deinit(void) +{ + ssl_iostream_context_cache_free(); +#ifdef HAVE_OPENSSL + ssl_iostream_openssl_deinit(); +#endif +} + +int main(int argc, char *argv[]) +{ + int c; + int ret; + + lib_init(); + main_init(); + + while ((c = getopt(argc, argv, "DS")) > 0) { + switch (c) { + case 'D': + debug = TRUE; + break; + case 'S': + small_socket_buffers = TRUE; + break; + default: + i_fatal("Usage: %s [-D]", argv[0]); + } + } + + test_subprocesses_init(debug); + + /* listen on localhost */ + i_zero(&bind_ip); + bind_ip.family = AF_INET; + bind_ip.u.ip4.s_addr = htonl(INADDR_LOOPBACK); + + ret = test_run(test_functions); + + test_subprocesses_deinit(); + main_deinit(); + lib_deinit(); + return ret; +} diff --git a/src/lib-http/test-http-request-parser.c b/src/lib-http/test-http-request-parser.c new file mode 100644 index 0000000..414e66b --- /dev/null +++ b/src/lib-http/test-http-request-parser.c @@ -0,0 +1,701 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "buffer.h" +#include "str.h" +#include "str-sanitize.h" +#include "istream.h" +#include "ostream.h" +#include "test-common.h" +#include "http-url.h" +#include "http-request-parser.h" + +#include <time.h> + +/* + * Test: valid requests + */ + +struct http_request_valid_parse_test { + const char *request; + const char *method; + const char *target_raw; + struct { + enum http_request_target_format format; + struct http_url url; + } target; + unsigned char version_major; + unsigned char version_minor; + uoff_t content_length; + const char *payload; + bool connection_close; + bool expect_100_continue; + + enum http_request_parse_flags flags; +}; + +static const struct http_url default_base_url = { + .host = { .name = "example.org" }, +}; + +static const struct http_request_valid_parse_test +valid_request_parse_tests[] = { + { + .request = + "GET / HTTP/1.1\r\n" + "Host: example.com\r\n" + "\r\n", + .method = "GET", + .target_raw = "/", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ORIGIN, + .url = { + .host = { .name = "example.com" }, + .path = "/", + }, + }, + .version_major = 1, .version_minor = 1, + }, + { + .request = + "GET / HTTP/1.1\r\n" + "Host: \r\n" + "\r\n", + .method = "GET", + .target_raw = "/", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ORIGIN, + .url = { + .host = { .name = "example.org" }, + .path = "/", + }, + }, + .version_major = 1, .version_minor = 1, + }, + { + .request = + "GET / HTTP/1.0\r\n" + "\r\n", + .method = "GET", + .target_raw = "/", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ORIGIN, + .url = { + .host = { .name = "example.org" }, + .path = "/", + }, + }, + .version_major = 1, .version_minor = 0, + .connection_close = TRUE, + }, + { + .request = + "OPTIONS * HTTP/1.1\r\n" + "Host: example.com\r\n" + "Connection: Keep-Alive\r\n" + "\r\n", + .method = "OPTIONS", + .target_raw = "*", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ASTERISK, + .url = { + .host = { .name = "example.com" }, + }, + }, + .version_major = 1, .version_minor = 1, + }, + { + .request = + "OPTIONS * HTTP/1.0\r\n" + "Connection: Keep-Alive\r\n" + "\r\n", + .method = "OPTIONS", + .target_raw = "*", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ASTERISK, + .url = { + .host = { .name = "example.org" }, + }, + }, + .version_major = 1, .version_minor = 0, + }, + { + .request = + "CONNECT example.com:443 HTTP/1.2\r\n" + "Host: example.com:443\r\n" + "\r\n", + .method = "CONNECT", + .target_raw = "example.com:443", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_AUTHORITY, + .url = { + .host = { .name = "example.com" }, + .port = 443, + } + }, + .version_major = 1, .version_minor = 2, + }, + { + .request = + "GET https://www.example.com:443 HTTP/1.1\r\n" + "Host: www.example.com:80\r\n" + "\r\n", + .method = "GET", + .target_raw = "https://www.example.com:443", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE, + .url = { + .host = { .name = "www.example.com" }, + .port = 443, + .have_ssl = TRUE, + }, + }, + .version_major = 1, .version_minor = 1, + }, + { + .request = + "POST http://api.example.com:8080/commit?user=dirk HTTP/1.1\r\n" + "Host: api.example.com:8080\r\n" + "Content-Length: 10\r\n" + "\r\n" + "Content!\r\n", + .method = "POST", + .target_raw = "http://api.example.com:8080/commit?user=dirk", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE, + .url = { + .host = { .name = "api.example.com" }, + .port = 8080, + .path = "/commit", + }, + }, + .version_major = 1, .version_minor = 1, + .payload = "Content!\r\n", + }, + { + .request = + "GET http://www.example.com/index.php?seq=1 HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Connection: close\r\n" + "\r\n", + .method = "GET", + .target_raw = "http://www.example.com/index.php?seq=1", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE, + .url = { + .host = { .name = "www.example.com" }, + .path = "/index.php", + }, + }, + .version_major = 1, .version_minor = 1, + .connection_close = TRUE, + }, + { + .request = + "GET http://www.example.com/index.html HTTP/1.0\r\n" + "Host: www.example.com\r\n" + "\r\n", + .method = "GET", + .target_raw = "http://www.example.com/index.html", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE, + .url = { + .host = { .name = "www.example.com" }, + .path = "/index.html", + }, + }, + .version_major = 1, .version_minor = 0, + .connection_close = TRUE, + }, + { + .request = + "GET http://www.example.com/index.html HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Expect: 100-continue\r\n" + "\r\n", + .method = "GET", + .target_raw = "http://www.example.com/index.html", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE, + .url = { + .host = { .name = "www.example.com" }, + .path = "/index.html", + }, + }, + .version_major = 1, .version_minor = 1, + .expect_100_continue = TRUE + }, + { + .request = + "GET / HTTP/1.1\r\n" + "Date: Mon, 09 Kul 2018 02:24:29 GMT\r\n" + "Host: example.com\r\n" + "\r\n", + .method = "GET", + .target_raw = "/", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ORIGIN, + .url = { + .host = { .name = "example.com" }, + .path = "/", + }, + }, + .version_major = 1, .version_minor = 1, + }, + { + .request = + "GET / HTTP/1.1\r\n" + "Date: Sun, 07 Oct 2012 19:52:03 GMT\r\n" + "Host: example.com\r\n" + "Date: Sun, 13 Oct 2013 13:13:13 GMT\r\n" + "\r\n", + .method = "GET", + .target_raw = "/", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ORIGIN, + .url = { + .host = { .name = "example.com" }, + .path = "/", + }, + }, + .version_major = 1, .version_minor = 1, + }, + { + .request = + "GET //index.php HTTP/1.1\r\n" + "Date: Sun, 07 Oct 2012 19:52:03 GMT\r\n" + "Host: example.com\r\n" + "Date: Sun, 13 Oct 2013 13:13:13 GMT\r\n" + "\r\n", + .method = "GET", + .target_raw = "//index.php", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ORIGIN, + .url = { + .host = { .name = "example.com" }, + .path = "//index.php", + }, + }, + .version_major = 1, .version_minor = 1, + }, +}; + +static const unsigned int valid_request_parse_test_count = + N_ELEMENTS(valid_request_parse_tests); + +static const char * +_request_target_format(enum http_request_target_format target_format) +{ + switch (target_format) { + case HTTP_REQUEST_TARGET_FORMAT_ORIGIN: + return "origin"; + case HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE: + return "absolute"; + case HTTP_REQUEST_TARGET_FORMAT_AUTHORITY: + return "authority"; + case HTTP_REQUEST_TARGET_FORMAT_ASTERISK: + return "asterisk"; + } + return t_strdup_printf("<<UNKNOWN: %u>>", target_format); +} + +static void +test_http_request_url_equal(const struct http_url *urlt, + const struct http_url *urlp) +{ + if (urlp == NULL) { + test_out("request->target.url = (null)", + (urlt->host.name == NULL && urlt->port == 0)); + return; + } + if (urlp->host.name == NULL || urlt->host.name == NULL) { + test_out(t_strdup_printf("request->target.url->host.name = %s", + urlp->host.name), + (urlp->host.name == urlt->host.name)); + } else { + test_out(t_strdup_printf("request->target.url->host.name = %s", + urlp->host.name), + strcmp(urlp->host.name, urlt->host.name) == 0); + } + if (urlp->port == 0) { + test_out("request->target.url->port = (unspecified)", + urlp->port == urlt->port); + } else { + test_out(t_strdup_printf("request->target.url->port = %u", + urlp->port), + urlp->port == urlt->port); + } + test_out(t_strdup_printf("request->target.url->have_ssl = %s", + (urlp->have_ssl ? "yes" : "no")), + urlp->have_ssl == urlt->have_ssl); + if (urlp->path == NULL || urlt->path == NULL) { + test_out(t_strdup_printf("request->target.url->path = %s", + urlp->path), + urlp->path == urlt->path); + } else { + test_out(t_strdup_printf("request->target.url->path = %s", + urlp->path), + strcmp(urlp->path, urlt->path) == 0); + } +} + +static void +test_http_request_equal(const struct http_request_valid_parse_test *test, + struct http_request request, const char *payload) +{ + if (request.method == NULL || test->method == NULL) { + test_out(t_strdup_printf("request->method = %s", + request.method), + request.method == test->method); + } else { + test_out(t_strdup_printf("request->method = %s", + request.method), + strcmp(request.method, test->method) == 0); + } + + if (request.target_raw == NULL || test->target_raw == NULL) { + test_out(t_strdup_printf("request->target_raw = %s", + request.target_raw), + request.target_raw == test->target_raw); + } else { + test_out(t_strdup_printf("request->target_raw = %s", + request.target_raw), + strcmp(request.target_raw, test->target_raw) == 0); + } + + test_http_request_url_equal(&test->target.url, request.target.url); + + test_out(t_strdup_printf("request->target_format = %s", + _request_target_format(request.target.format)), + request.target.format == test->target.format); + + test_out(t_strdup_printf("request->version = %u.%u", + request.version_major, request.version_minor), + (request.version_major == test->version_major && + request.version_minor == test->version_minor)); + test_out(t_strdup_printf("request->connection_close = %s", + (request.connection_close ? "yes" : "no")), + request.connection_close == test->connection_close); + test_out(t_strdup_printf("request->expect_100_continue = %s", + (request.expect_100_continue ? "yes" : "no")), + request.expect_100_continue == test->expect_100_continue); + + if (payload == NULL || test->payload == NULL) { + test_out(t_strdup_printf("request->payload = %s", + str_sanitize(payload, 80)), + payload == test->payload); + } else { + test_out(t_strdup_printf("request->payload = %s", + str_sanitize(payload, 80)), + strcmp(payload, test->payload) == 0); + } +} + +static void test_http_request_parse_valid(void) +{ + unsigned int i; + buffer_t *payload_buffer = buffer_create_dynamic(default_pool, 1024); + + for (i = 0; i < valid_request_parse_test_count; i++) T_BEGIN { + struct istream *input; + struct ostream *output; + const struct http_request_valid_parse_test *test; + struct http_request_parser *parser; + struct http_request request, request_parsed; + enum http_request_parse_error error_code; + const char *request_text, *payload, *error; + unsigned int pos, request_text_len; + int ret = 0; + + test = &valid_request_parse_tests[i]; + request_text = test->request; + request_text_len = strlen(request_text); + input = test_istream_create_data(request_text, + request_text_len); + parser = http_request_parser_init(input, &default_base_url, + NULL, test->flags); + + test_begin(t_strdup_printf("http request valid [%d]", i)); + + payload = NULL; + for (pos = 0; pos <= request_text_len && ret == 0; pos++) { + test_istream_set_size(input, pos); + ret = http_request_parse_next(parser, NULL, + &request_parsed, + &error_code, &error); + } + test_istream_set_size(input, request_text_len); + i_stream_unref(&input); + request = request_parsed; + + while (ret > 0) { + if (request.payload != NULL) { + buffer_set_used_size(payload_buffer, 0); + output = o_stream_create_buffer(payload_buffer); + test_out("payload receive", + o_stream_send_istream( + output, request.payload) == + OSTREAM_SEND_ISTREAM_RESULT_FINISHED); + o_stream_destroy(&output); + payload = str_c(payload_buffer); + } else { + payload = NULL; + } + ret = http_request_parse_next( + parser, NULL, &request_parsed, + &error_code, &error); + } + + test_out_reason("parse success", ret == 0, error); + + if (ret == 0) { + /* verify last request only */ + test_http_request_equal(test, request, payload); + } + test_end(); + http_request_parser_deinit(&parser); + } T_END; + + buffer_free(&payload_buffer); +} + +/* + * Test: invalid requests + */ + +struct http_request_invalid_parse_test { + const char *request; + enum http_request_parse_flags flags; + + enum http_request_parse_error error_code; +}; + +static const struct http_request_invalid_parse_test +invalid_request_parse_tests[] = { + { + .request = + "GET: / HTTP/1.1\r\n" + "Host: example.com\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST + }, + { + .request = + "GET % HTTP/1.1\r\n" + "Host: example.com\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST + }, + { + .request = + "GET /frop\" HTTP/1.1\r\n" + "Host: example.com\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST + }, + { + .request = + "GET / HTCPCP/1.0\r\n" + "Host: example.com\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST + }, + { + .request = + "GET / HTTP/1.0.1\r\n" + "Host: example.com\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST + }, + { + .request = + "GET / HTTP/1.1\r\n" + "Host: \"example.com\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST + }, + { + .request = + "GET / HTTP/1.1\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST + }, + { + .request = + "GET / HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Transfer-Encoding: gzip\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST + }, + { + .request = + "GET / HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Expect: payment\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_EXPECTATION_FAILED + }, + { + .request = + "GET / HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Transfer-Encoding: cuneiform, chunked\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_NOT_IMPLEMENTED + }, + { + .request = + "GET / HTTP/1.1\r\n" + "Date: Mon, 09 Kul 2018 02:24:29 GMT\r\n" + "Host: example.com\r\n" + "\r\n", + .flags = HTTP_REQUEST_PARSE_FLAG_STRICT, + .error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST + }, + { + .request = + "GET / HTTP/1.1\r\n" + "Date: Sun, 07 Oct 2012 19:52:03 GMT\r\n" + "Host: example.com\r\n" + "Date: Sun, 13 Oct 2013 13:13:13 GMT\r\n" + "\r\n", + .flags = HTTP_REQUEST_PARSE_FLAG_STRICT, + .error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST + } + // FIXME: test request limits +}; + +static unsigned int invalid_request_parse_test_count = + N_ELEMENTS(invalid_request_parse_tests); + +static const char * +_request_parse_error(enum http_request_parse_error error) +{ + switch (error) { + case HTTP_REQUEST_PARSE_ERROR_NONE: + return "none?!"; + case HTTP_REQUEST_PARSE_ERROR_BROKEN_STREAM: + return "broken stream"; + case HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST: + return "broken request"; + case HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST: + return "bad request"; + case HTTP_REQUEST_PARSE_ERROR_NOT_IMPLEMENTED: + return "not implemented"; + case HTTP_REQUEST_PARSE_ERROR_EXPECTATION_FAILED: + return "expectation failed"; + case HTTP_REQUEST_PARSE_ERROR_METHOD_TOO_LONG: + return "method too long"; + case HTTP_REQUEST_PARSE_ERROR_TARGET_TOO_LONG: + return "target too long"; + case HTTP_REQUEST_PARSE_ERROR_PAYLOAD_TOO_LARGE: + return "payload too large"; + } + return t_strdup_printf("<<UNKNOWN: %u>>", error); +} + +static void test_http_request_parse_invalid(void) +{ + const struct http_request_invalid_parse_test *test; + struct http_request_parser *parser; + struct http_request request; + enum http_request_parse_error error_code; + const char *request_text, *error; + struct istream *input; + int ret; + unsigned int i; + + for (i = 0; i < invalid_request_parse_test_count; i++) T_BEGIN { + test = &invalid_request_parse_tests[i]; + request_text = test->request; + input = i_stream_create_from_data(request_text, + strlen(request_text)); + parser = http_request_parser_init(input, &default_base_url, + NULL, test->flags); + i_stream_unref(&input); + + test_begin(t_strdup_printf("http request invalid [%d]", i)); + + while ((ret=http_request_parse_next + (parser, NULL, &request, &error_code, &error)) > 0); + + test_out_reason("parse failure", ret < 0, error); + if (ret < 0) { + const char *error = _request_parse_error(error_code); + test_out(t_strdup_printf("parse error code = %s", + error), + error_code == test->error_code); + } + test_end(); + http_request_parser_deinit(&parser); + } T_END; +} + +/* + * Bad request tests + */ + +static const unsigned char bad_request_with_nuls[] = + "GET / HTTP/1.1\r\n" + "Host: example.com\r\n" + "User-Agent: text\0client\r\n" + "\r\n"; + +static void test_http_request_parse_bad(void) +{ + struct http_request_parser *parser; + struct http_request request; + enum http_request_parse_error error_code; + const char *header, *error; + struct istream *input; + int ret; + + /* parse failure guarantees http_request_header.size equals + strlen(http_request_header.value) */ + test_begin("http request with NULs (strict)"); + input = i_stream_create_from_data(bad_request_with_nuls, + sizeof(bad_request_with_nuls)-1); + parser = http_request_parser_init(input, &default_base_url, NULL, + HTTP_REQUEST_PARSE_FLAG_STRICT); + i_stream_unref(&input); + + while ((ret=http_request_parse_next + (parser, NULL, &request, &error_code, &error)) > 0); + test_assert(ret < 0); + http_request_parser_deinit(&parser); + test_end(); + + /* even when lenient, bad characters like NUL must not be returned */ + test_begin("http request with NULs (lenient)"); + input = i_stream_create_from_data(bad_request_with_nuls, + sizeof(bad_request_with_nuls)-1); + parser = http_request_parser_init(input, &default_base_url, NULL, 0); + i_stream_unref(&input); + + ret = http_request_parse_next + (parser, NULL, &request, &error_code, &error); + test_out("parse success", ret > 0); + header = http_request_header_get(&request, "user-agent"); + test_out("header present", header != NULL); + if (header != NULL) { + test_out(t_strdup_printf("header User-Agent: %s", header), + strcmp(header, "textclient") == 0); + } + ret = http_request_parse_next + (parser, NULL, &request, &error_code, &error); + test_out("parse end", ret == 0); + http_request_parser_deinit(&parser); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_http_request_parse_valid, + test_http_request_parse_invalid, + test_http_request_parse_bad, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-http/test-http-response-parser.c b/src/lib-http/test-http-response-parser.c new file mode 100644 index 0000000..3964a32 --- /dev/null +++ b/src/lib-http/test-http-response-parser.c @@ -0,0 +1,406 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "buffer.h" +#include "str.h" +#include "str-sanitize.h" +#include "istream.h" +#include "ostream.h" +#include "test-common.h" +#include "http-response-parser.h" + +#include <time.h> + +struct valid_parse_test_response { + unsigned char version_major; + unsigned char version_minor; + unsigned int status; + uoff_t content_length; + const char *payload; +}; + +struct valid_parse_test { + const char *input; + enum http_response_parse_flags flags; + + const struct valid_parse_test_response *responses; + unsigned int responses_count; +}; + +/* Valid response tests */ + +static const struct valid_parse_test_response valid_responses1[] = { + { + .status = 200, + .payload = "This is a piece of stupid text.\r\n" + } +}; + +static const struct valid_parse_test_response valid_responses2[] = { + { + .status = 200, + .payload = "This is a piece of stupid text.\r\n" + },{ + .status = 200, + .payload = "This is a piece of even more stupid text.\r\n" + } +}; + +static const struct valid_parse_test_response valid_responses3[] = { + { + .status = 401, + .payload = "Frop!" + } +}; + +static const struct valid_parse_test_response valid_responses4[] = { + { + .status = 200, + .payload = "Invalid date header" + } +}; + +static const struct valid_parse_test_response valid_responses5[] = { + { + .status = 200, + .payload = "Duplicate headers" + } +}; + +static const struct valid_parse_test +valid_response_parse_tests[] = { + { .input = + "HTTP/1.1 200 OK\r\n" + "Date: Sun, 07 Oct 2012 13:02:27 GMT\r\n" + "Server: Apache/2.2.16 (Debian)\r\n" + "Last-Modified: Tue, 18 Sep 2012 19:31:41 GMT\r\n" + "Etag: \"2a8400c-10751f-4c9fef0858140\"\r\n" + "Accept-Ranges: bytes\r\n" + "Content-Length: 33\r\n" + "Keep-Alive: timeout=15, max=100\r\n" + "Connection: Keep-Alive\r\n" + "Content-Type: text/plain\r\n" + "\r\n" + "This is a piece of stupid text.\r\n", + .responses = valid_responses1, + .responses_count = N_ELEMENTS(valid_responses1) + },{ + .input = + "HTTP/1.1 200 OK\r\n" + "Date: Sun, 07 Oct 2012 13:02:27 GMT\r\n" + "Server: Apache/2.2.16 (Debian)\r\n" + "Last-Modified: Tue, 18 Sep 2012 19:31:41 GMT\r\n" + "Etag: \"2a8400c-10751f-4c9fef0858140\"\r\n" + "Accept-Ranges: bytes\r\n" + "Content-Length: 33\r\n" + "Keep-Alive: timeout=15, max=100\r\n" + "Connection: Keep-Alive\r\n" + "Content-Type: text/plain\r\n" + "\r\n" + "This is a piece of stupid text.\r\n" + "HTTP/1.1 200 OK\r\n" + "Date: Sun, 07 Oct 2012 13:02:27 GMT\r\n" + "Server: Apache/2.2.16 (Debian)\r\n" + "Last-Modified: Tue, 18 Sep 2012 19:31:41 GMT\r\n" + "Etag: \"2a8400c-10751f-4c9fef0858140\"\r\n" + "Accept-Ranges: bytes\r\n" + "Content-Length: 43\r\n" + "Keep-Alive: timeout=15, max=100\r\n" + "Connection: Keep-Alive\r\n" + "Content-Type: text/plain\r\n" + "\r\n" + "This is a piece of even more stupid text.\r\n", + .responses = valid_responses2, + .responses_count = N_ELEMENTS(valid_responses2) + },{ + .input = + "HTTP/1.1 401 Authorization Required\r\n" + "Date: Sun, 07 Oct 2012 19:52:03 GMT\r\n" + "Server: Apache/2.2.16 (Debian) PHP/5.3.3-7+squeeze14\r\n" + "WWW-Authenticate: Basic realm=\"Munin\"\r\n" + "Vary: Accept-Encoding\r\n" + "Content-Encoding: gzip\r\n" + "Content-Length: 5\r\n" + "Keep-Alive: timeout=15, max=99\r\n" + "Connection: Keep-Alive\r\n" + "Content-Type: text/html; charset=iso-8859-1\r\n" + "\r\n" + "Frop!", + .responses = valid_responses3, + .responses_count = N_ELEMENTS(valid_responses3) + },{ + .input = + "HTTP/1.1 200 OK\r\n" + "Date: Sun, 07 Ocu 2012 19:52:03 GMT\r\n" + "Content-Length: 19\r\n" + "Keep-Alive: timeout=15, max=99\r\n" + "Connection: Keep-Alive\r\n" + "Date: Sun, 13 Oct 2013 13:13:13 GMT\r\n" + "\r\n" + "Invalid date header", + .responses = valid_responses4, + .responses_count = N_ELEMENTS(valid_responses4) + },{ + .input = + "HTTP/1.1 200 OK\r\n" + "Date: Sun, 07 Oct 2012 19:52:03 GMT\r\n" + "Server: Apache/2.2.16 (Debian) PHP/5.3.3-7+squeeze14\r\n" + "Content-Length: 17\r\n" + "Keep-Alive: timeout=15, max=99\r\n" + "Connection: Keep-Alive\r\n" + "Content-Type: text/html; charset=iso-8859-1\r\n" + "Date: Sun, 13 Oct 2013 13:13:13 GMT\r\n" + "\r\n" + "Duplicate headers", + .responses = valid_responses5, + .responses_count = N_ELEMENTS(valid_responses5) + } +}; + +static const unsigned int valid_response_parse_test_count = + N_ELEMENTS(valid_response_parse_tests); + +static void test_http_response_parse_valid(void) +{ + unsigned int i; + buffer_t *payload_buffer = buffer_create_dynamic(default_pool, 1024); + + for (i = 0; i < valid_response_parse_test_count; i++) T_BEGIN { + struct istream *input; + struct ostream *output; + const struct valid_parse_test *test; + const struct valid_parse_test_response *tresponse; + struct http_response_parser *parser; + struct http_response presponse; + const char *input_text, *payload, *error; + unsigned int j, pos, input_text_len; + int ret = 0; + + i_zero(&presponse); + test = &valid_response_parse_tests[i]; + input_text = test->input; + input_text_len = strlen(input_text); + input = test_istream_create_data(input_text, input_text_len); + parser = http_response_parser_init(input, NULL, + valid_response_parse_tests[i].flags); + + test_begin(t_strdup_printf("http response valid [%d]", i)); + + payload = NULL; + for (pos = 0; pos < input_text_len && ret == 0; pos++) { + test_istream_set_size(input, pos); + ret = http_response_parse_next(parser, + HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED, &presponse, &error); + } + test_istream_set_size(input, input_text_len); + i_stream_unref(&input); + + j = 0; + test_out("parse success", ret > 0); + while (ret > 0) { + if (presponse.payload != NULL) { + buffer_set_used_size(payload_buffer, 0); + output = o_stream_create_buffer(payload_buffer); + test_out("payload receive", + o_stream_send_istream(output, presponse.payload) + == OSTREAM_SEND_ISTREAM_RESULT_FINISHED); + o_stream_destroy(&output); + payload = str_c(payload_buffer); + } else { + payload = NULL; + } + + test_assert(j < test->responses_count); + if (j >= test->responses_count) + break; + tresponse = &test->responses[j]; + + /* verify last response only */ + test_out(t_strdup_printf("response->status = %d", + tresponse->status), + presponse.status == tresponse->status); + if (payload == NULL || tresponse->payload == NULL) { + test_out(t_strdup_printf("response->payload = %s", + str_sanitize(payload, 80)), + payload == tresponse->payload); + } else { + test_out(t_strdup_printf("response->payload = %s", + str_sanitize(payload, 80)), + strcmp(payload, tresponse->payload) == 0); + } + + ret = http_response_parse_next(parser, + HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED, &presponse, &error); + if (++j == test->responses_count) + test_out("parse end", ret == 0); + else + test_out("parse success", ret > 0); + } + + test_assert(ret == 0); + test_end(); + http_response_parser_deinit(&parser); + } T_END; + + buffer_free(&payload_buffer); +} + +/* + * Invalid response tests + */ + +struct invalid_parse_test { + const char *input; + enum http_response_parse_flags flags; +}; + +static struct invalid_parse_test invalid_response_parse_tests[] = { + { + .input = + "XMPP/1.0 302 Found\r\n" + "Location: http://www.example.nl/\r\n" + "Cache-Control: private\r\n" + },{ + .input = + "HTTP/1.1 302 Found\r\n" + "Location: http://www.example.nl/\r\n" + "Cache-Control: private\r\n" + },{ + .input = + "HTTP/1.1 ABC Found\r\n" + "Location: http://www.example.nl/\r\n" + "Cache-Control: private\r\n" + },{ + .input = + "HTTP/1.1 302 \177\r\n" + "Location: http://www.example.nl/\r\n" + "Cache-Control: private\r\n" + },{ + .input = + "HTTP/1.1 302 Found\n\r" + "Location: http://www.example.nl/\n\r" + "Cache-Control: private\n\r" + },{ + .input = + "HTTP/1.1 200 OK\r\n" + "Date: Sun, 07 Ocu 2012 19:52:03 GMT\r\n" + "Content-Length: 19\r\n" + "Keep-Alive: timeout=15, max=99\r\n" + "Connection: Keep-Alive\r\n" + "\r\n" + "Invalid date header", + .flags = HTTP_RESPONSE_PARSE_FLAG_STRICT + },{ + .input = + "HTTP/1.1 200 OK\r\n" + "Date: Sun, 07 Oct 2012 19:52:03 GMT\r\n" + "Server: Apache/2.2.16 (Debian) PHP/5.3.3-7+squeeze14\r\n" + "Content-Length: 17\r\n" + "Keep-Alive: timeout=15, max=99\r\n" + "Connection: Keep-Alive\r\n" + "Content-Type: text/html; charset=iso-8859-1\r\n" + "Date: Sun, 13 Oct 2013 13:13:13 GMT\r\n" + "\r\n" + "Duplicate headers", + .flags = HTTP_RESPONSE_PARSE_FLAG_STRICT + } +}; + +static const unsigned int invalid_response_parse_test_count = + N_ELEMENTS(invalid_response_parse_tests); + +static void test_http_response_parse_invalid(void) +{ + struct http_response_parser *parser; + struct http_response response; + const char *response_text, *error; + struct istream *input; + int ret; + unsigned int i; + + for (i = 0; i < invalid_response_parse_test_count; i++) T_BEGIN { + const char *test; + + test = invalid_response_parse_tests[i].input; + response_text = test; + input = i_stream_create_from_data(response_text, strlen(response_text)); + parser = http_response_parser_init(input, NULL, + invalid_response_parse_tests[i].flags); + i_stream_unref(&input); + + test_begin(t_strdup_printf("http response invalid [%d]", i)); + + while ((ret=http_response_parse_next(parser, HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED, &response, &error)) > 0); + + test_out_reason("parse failure", ret < 0, error); + test_end(); + http_response_parser_deinit(&parser); + } T_END; +} + +/* + * Bad response tests + */ + +static const unsigned char bad_response_with_nuls[] = + "HTTP/1.1 200 OK\r\n" + "Server: text\0server\r\n" + "\r\n"; + +static void test_http_response_parse_bad(void) +{ + struct http_response_parser *parser; + struct http_response response; + const char *header, *error; + struct istream *input; + int ret; + + /* parse failure guarantees http_response_header.size equals + strlen(http_response_header.value) */ + + test_begin("http response with NULs (strict)"); + input = i_stream_create_from_data(bad_response_with_nuls, + sizeof(bad_response_with_nuls)-1); + parser = http_response_parser_init(input, NULL, + HTTP_RESPONSE_PARSE_FLAG_STRICT); + i_stream_unref(&input); + + while ((ret=http_response_parse_next(parser, + HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED, &response, &error)) > 0); + test_assert(ret < 0); + http_response_parser_deinit(&parser); + test_end(); + + /* even when lenient, bad characters like NUL must not be returned */ + test_begin("http response with NULs (lenient)"); + input = i_stream_create_from_data(bad_response_with_nuls, + sizeof(bad_response_with_nuls)-1); + parser = http_response_parser_init(input, NULL, 0); + i_stream_unref(&input); + + ret = http_response_parse_next(parser, + HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED, &response, &error); + test_out("parse success", ret > 0); + header = http_response_header_get(&response, "server"); + test_out("header present", header != NULL); + if (header != NULL) { + test_out(t_strdup_printf("header Server: %s", header), + strcmp(header, "textserver") == 0); + } + ret = http_response_parse_next(parser, + HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED, &response, &error); + test_out("parse end", ret == 0); + http_response_parser_deinit(&parser); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_http_response_parse_valid, + test_http_response_parse_invalid, + test_http_response_parse_bad, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-http/test-http-server-errors.c b/src/lib-http/test-http-server-errors.c new file mode 100644 index 0000000..4a95bde --- /dev/null +++ b/src/lib-http/test-http-server-errors.c @@ -0,0 +1,1042 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "hostpid.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "time-util.h" +#include "sleep.h" +#include "connection.h" +#include "test-common.h" +#include "test-subprocess.h" +#include "http-url.h" +#include "http-request.h" +#include "http-server.h" + +#include <unistd.h> + +#define SERVER_MAX_TIMEOUT_MSECS 10*1000 +#define CLIENT_KILL_TIMEOUT_SECS 20 + +static void main_deinit(void); + +/* + * Types + */ + +struct client_connection { + struct connection conn; + + pool_t pool; +}; + +typedef void +(*test_server_init_t)(const struct http_server_settings *server_set); +typedef void (*test_client_init_t)(unsigned int index); + +/* + * State + */ + +/* common */ +static struct ip_addr bind_ip; +static in_port_t bind_port = 0; +static struct ioloop *ioloop; +static bool debug = FALSE; + +/* server */ +static struct http_server *http_server = NULL; +static struct io *io_listen; +static int fd_listen = -1; +static void (*test_server_request)(struct http_server_request *req); + +/* client */ +static struct connection_list *client_conn_list; +static unsigned int client_index; +static void (*test_client_connected)(struct client_connection *conn); +static void (*test_client_input)(struct client_connection *conn); + +/* + * Forward declarations + */ + +/* server */ +static void test_server_defaults(struct http_server_settings *http_set); +static void test_server_run(const struct http_server_settings *http_set); + +/* client */ +static void client_connection_deinit(struct client_connection **_conn); +static void test_client_run(unsigned int index); + +/* test*/ +static void +test_run_client_server(const struct http_server_settings *server_set, + test_server_init_t server_test, + test_client_init_t client_test, + unsigned int client_tests_count) ATTR_NULL(3); + +/* + * Slow request + */ + +/* client */ + +static void +test_slow_request_input(struct client_connection *conn ATTR_UNUSED) +{ + /* do nothing */ +} + +static void test_slow_request_connected(struct client_connection *conn) +{ + o_stream_nsend_str(conn->conn.output, + "GET / HTTP/1.1\r\n" + "Host: example.com\r\n" + "\r\n"); +} + +static void test_client_slow_request(unsigned int index) +{ + test_client_input = test_slow_request_input; + test_client_connected = test_slow_request_connected; + test_client_run(index); +} + +/* server */ + +struct _slow_request { + struct http_server_request *req; + struct timeout *to_delay; + bool serviced:1; +}; + +static void test_server_slow_request_destroyed(struct _slow_request *ctx) +{ + test_assert(ctx->serviced); + timeout_remove(&ctx->to_delay); + i_free(ctx); + io_loop_stop(ioloop); +} + +static void test_server_slow_request_delayed(struct _slow_request *ctx) +{ + struct http_server_response *resp; + struct http_server_request *req = ctx->req; + + resp = http_server_response_create(req, 200, "OK"); + http_server_response_submit(resp); + ctx->serviced = TRUE; + + http_server_request_unref(&req); +} + +static void test_server_slow_request_request(struct http_server_request *req) +{ + const struct http_request *hreq = http_server_request_get(req); + struct _slow_request *ctx; + + if (debug) { + i_debug("REQUEST: %s %s HTTP/%u.%u", + hreq->method, hreq->target_raw, + hreq->version_major, hreq->version_minor); + } + + ctx = i_new(struct _slow_request, 1); + ctx->req = req; + + http_server_request_set_destroy_callback( + req, test_server_slow_request_destroyed, ctx); + + http_server_request_ref(req); + ctx->to_delay = + timeout_add(4000, test_server_slow_request_delayed, ctx); +} + +static void +test_server_slow_request(const struct http_server_settings *server_set) +{ + test_server_request = test_server_slow_request_request; + test_server_run(server_set); +} + +/* test */ + +static void test_slow_request(void) +{ + struct http_server_settings http_server_set; + + test_server_defaults(&http_server_set); + http_server_set.max_client_idle_time_msecs = 1000; + + test_begin("slow request"); + test_run_client_server(&http_server_set, test_server_slow_request, + test_client_slow_request, 1); + test_end(); +} + +/* + * Hanging request payload + */ + +/* client */ + +static void +test_hanging_request_payload_connected(struct client_connection *conn) +{ + o_stream_nsend_str(conn->conn.output, + "GET / HTTP/1.1\r\n" + "Host: example.com\r\n" + "Content-Length: 1000\r\n" + "\r\n" + "To be continued... or not"); +} + +static void test_client_hanging_request_payload(unsigned int index) +{ + test_client_connected = test_hanging_request_payload_connected; + test_client_run(index); +} + +/* server */ + +struct _hanging_request_payload { + struct http_server_request *req; + struct istream *payload_input; + struct io *io; + bool serviced:1; +}; + +static void +test_server_hanging_request_payload_destroyed( + struct _hanging_request_payload *ctx) +{ + test_assert(!ctx->serviced); + io_remove(&ctx->io); + i_free(ctx); + io_loop_stop(ioloop); +} + +static void +test_server_hanging_request_payload_input(struct _hanging_request_payload *ctx) +{ + struct http_server_response *resp; + struct http_server_request *req = ctx->req; + const unsigned char *data; + size_t size; + int ret; + + if (debug) + i_debug("test server: got more payload"); + + while ((ret = i_stream_read_data(ctx->payload_input, + &data, &size, 0)) > 0) + i_stream_skip(ctx->payload_input, size); + + if (ret == 0) + return; + if (ctx->payload_input->stream_errno != 0) { + if (debug) { + i_debug("test server: failed to read payload: %s", + i_stream_get_error(ctx->payload_input)); + } + i_stream_unref(&ctx->payload_input); + io_remove(&ctx->io); + http_server_request_fail_close(req, 400, "Bad request"); + http_server_request_unref(&req); + return; + } + + i_assert(ctx->payload_input->eof); + + resp = http_server_response_create(req, 200, "OK"); + http_server_response_submit(resp); + ctx->serviced = TRUE; + + i_stream_unref(&ctx->payload_input); + http_server_request_unref(&req); +} + +static void +test_server_hanging_request_payload_request(struct http_server_request *req) +{ + const struct http_request *hreq = http_server_request_get(req); + struct _hanging_request_payload *ctx; + + if (debug) { + i_debug("REQUEST: %s %s HTTP/%u.%u", + hreq->method, hreq->target_raw, + hreq->version_major, hreq->version_minor); + } + + ctx = i_new(struct _hanging_request_payload, 1); + ctx->req = req; + + http_server_request_set_destroy_callback( + req, test_server_hanging_request_payload_destroyed, ctx); + + ctx->payload_input = http_server_request_get_payload_input(req, FALSE); + + http_server_request_ref(req); + ctx->io = io_add_istream(ctx->payload_input, + test_server_hanging_request_payload_input, + ctx); + test_server_hanging_request_payload_input(ctx); +} + +static void +test_server_hanging_request_payload( + const struct http_server_settings *server_set) +{ + test_server_request = test_server_hanging_request_payload_request; + test_server_run(server_set); +} + +/* test */ + +static void test_hanging_request_payload(void) +{ + struct http_server_settings http_server_set; + + test_server_defaults(&http_server_set); + http_server_set.max_client_idle_time_msecs = 1000; + + test_begin("hanging request payload"); + test_run_client_server(&http_server_set, + test_server_hanging_request_payload, + test_client_hanging_request_payload, 1); + test_end(); +} + +/* + * Hanging response payload + */ + +/* client */ + +static void +test_hanging_response_payload_connected(struct client_connection *conn) +{ + o_stream_nsend_str(conn->conn.output, + "GET / HTTP/1.1\r\n" + "Host: example.com\r\n" + "Content-Length: 18\r\n" + "\r\n" + "Complete payload\r\n"); +} + +static void test_client_hanging_response_payload(unsigned int index) +{ + test_client_connected = test_hanging_response_payload_connected; + test_client_run(index); +} + +/* server */ + +struct _hanging_response_payload { + struct http_server_request *req; + struct istream *payload_input; + struct io *io; + bool serviced:1; +}; + +static void +test_server_hanging_response_payload_destroyed( + struct _hanging_response_payload *ctx) +{ + test_assert(!ctx->serviced); + io_remove(&ctx->io); + i_free(ctx); + io_loop_stop(ioloop); +} + +static void +test_server_hanging_response_payload_request(struct http_server_request *req) +{ + const struct http_request *hreq = + http_server_request_get(req); + struct http_server_response *resp; + struct _hanging_response_payload *ctx; + string_t *payload; + unsigned int i; + + if (debug) { + i_debug("REQUEST: %s %s HTTP/%u.%u", + hreq->method, hreq->target_raw, + hreq->version_major, hreq->version_minor); + } + + ctx = i_new(struct _hanging_response_payload, 1); + ctx->req = req; + + http_server_request_set_destroy_callback( + req, test_server_hanging_response_payload_destroyed, ctx); + + resp = http_server_response_create(req, 200, "OK"); + T_BEGIN { + payload = t_str_new(204800); + for (i = 0; i < 3200; i++) { + str_append(payload, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n"); + } + + http_server_response_set_payload_data(resp, str_data(payload), + str_len(payload)); + } T_END; + http_server_response_submit(resp); +} + +static void +test_server_hanging_response_payload( + const struct http_server_settings *server_set) +{ + test_server_request = test_server_hanging_response_payload_request; + test_server_run(server_set); +} + +/* test */ + +static void test_hanging_response_payload(void) +{ + struct http_server_settings http_server_set; + + test_server_defaults(&http_server_set); + http_server_set.socket_send_buffer_size = 4096; + http_server_set.max_client_idle_time_msecs = 1000; + + test_begin("hanging response payload"); + test_run_client_server(&http_server_set, + test_server_hanging_response_payload, + test_client_hanging_response_payload, 1); + test_end(); +} + +/* + * Excessive payload length + */ + +/* client */ + +static void +test_excessive_payload_length_connected1(struct client_connection *conn) +{ + o_stream_nsend_str( + conn->conn.output, + "GET / HTTP/1.1\r\n" + "Host: example.com\r\n" + "Content-Length: 150\r\n" + "\r\n" + "Too long\r\nToo long\r\nToo long\r\nToo long\r\nToo long\r\n" + "Too long\r\nToo long\r\nToo long\r\nToo long\r\nToo long\r\n" + "Too long\r\nToo long\r\nToo long\r\nToo long\r\nToo long\r\n"); +} + +static void test_client_excessive_payload_length1(unsigned int index) +{ + test_client_connected = test_excessive_payload_length_connected1; + test_client_run(index); +} + +static void +test_excessive_payload_length_connected2(struct client_connection *conn) +{ + o_stream_nsend_str( + conn->conn.output, + "GET / HTTP/1.1\r\n" + "Host: example.com\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "32\r\n" + "Too long\r\nToo long\r\nToo long\r\nToo long\r\nToo long\r\n" + "\r\n" + "32\r\n" + "Too long\r\nToo long\r\nToo long\r\nToo long\r\nToo long\r\n" + "\r\n" + "32\r\n" + "Too long\r\nToo long\r\nToo long\r\nToo long\r\nToo long\r\n" + "\r\n" + "0\r\n" + "\r\n"); +} + +static void test_client_excessive_payload_length2(unsigned int index) +{ + test_client_connected = test_excessive_payload_length_connected2; + test_client_run(index); +} + +/* server */ + +struct _excessive_payload_length { + struct http_server_request *req; + buffer_t *buffer; + bool serviced:1; +}; + +static void +test_server_excessive_payload_length_destroyed( + struct _excessive_payload_length *ctx) +{ + struct http_server_response *resp; + const char *reason; + int status; + + resp = http_server_request_get_response(ctx->req); + test_assert(resp != NULL); + if (resp != NULL) { + http_server_response_get_status(resp, &status, &reason); + test_assert(status == 413); + } + + test_assert(!ctx->serviced); + buffer_free(&ctx->buffer); + i_free(ctx); + io_loop_stop(ioloop); +} + +static void +test_server_excessive_payload_length_finished( + struct _excessive_payload_length *ctx) +{ + struct http_server_response *resp; + + resp = http_server_response_create(ctx->req, 200, "OK"); + http_server_response_submit(resp); + ctx->serviced = TRUE; +} + +static void +test_server_excessive_payload_length_request(struct http_server_request *req) +{ + const struct http_request *hreq = http_server_request_get(req); + struct _excessive_payload_length *ctx; + + if (debug) { + i_debug("REQUEST: %s %s HTTP/%u.%u", + hreq->method, hreq->target_raw, + hreq->version_major, hreq->version_minor); + } + + ctx = i_new(struct _excessive_payload_length, 1); + ctx->req = req; + ctx->buffer = buffer_create_dynamic(default_pool, 128); + + http_server_request_set_destroy_callback( + req, test_server_excessive_payload_length_destroyed, ctx); + http_server_request_buffer_payload( + req, ctx->buffer, 128, + test_server_excessive_payload_length_finished, ctx); +} + +static void +test_server_excessive_payload_length( + const struct http_server_settings *server_set) +{ + test_server_request = test_server_excessive_payload_length_request; + test_server_run(server_set); +} + +/* test */ + +static void test_excessive_payload_length(void) +{ + struct http_server_settings http_server_set; + + test_server_defaults(&http_server_set); + http_server_set.max_client_idle_time_msecs = 1000; + + test_begin("excessive payload length (length)"); + test_run_client_server(&http_server_set, + test_server_excessive_payload_length, + test_client_excessive_payload_length1, 1); + test_end(); + + test_begin("excessive payload length (chunked)"); + test_run_client_server(&http_server_set, + test_server_excessive_payload_length, + test_client_excessive_payload_length2, 1); + test_end(); +} + +/* + * Response ostream disconnect + */ + +/* client */ + +static void +test_response_ostream_disconnect_connected(struct client_connection *conn) +{ + o_stream_nsend_str(conn->conn.output, + "GET / HTTP/1.1\r\n" + "Host: example.com\r\n" + "Content-Length: 18\r\n" + "\r\n" + "Complete payload\r\n"); + i_sleep_intr_msecs(10); + client_connection_deinit(&conn); + io_loop_stop(ioloop); +} + +static void test_client_response_ostream_disconnect(unsigned int index) +{ + test_client_connected = test_response_ostream_disconnect_connected; + test_client_run(index); +} + +/* server */ + +struct _response_ostream_disconnect { + struct http_server_request *req; + struct istream *payload_input; + struct ostream *payload_output; + struct io *io; + bool finished:1; + bool seen_stream_error:1; +}; + +static void +test_server_response_ostream_disconnect_destroyed( + struct _response_ostream_disconnect *ctx) +{ + test_assert(ctx->seen_stream_error); + io_remove(&ctx->io); + i_stream_unref(&ctx->payload_input); + i_free(ctx); + io_loop_stop(ioloop); +} + +static int +test_server_response_ostream_disconnect_output( + struct _response_ostream_disconnect *ctx) +{ + struct ostream *output = ctx->payload_output; + enum ostream_send_istream_result res; + int ret; + + if (ctx->finished) { + ret = o_stream_finish(output); + if (ret == 0) + return ret; + if (ret < 0) { + if (debug) { + i_debug("OUTPUT ERROR: %s", + o_stream_get_error(output)); + } + test_assert(output->stream_errno == ECONNRESET || + output->stream_errno == EPIPE); + + ctx->seen_stream_error = TRUE; + o_stream_destroy(&ctx->payload_output); + return -1; + } + return 1; + } + + o_stream_set_max_buffer_size(output, IO_BLOCK_SIZE); + res = o_stream_send_istream(output, ctx->payload_input); + o_stream_set_max_buffer_size(output, SIZE_MAX); + + switch (res) { + case OSTREAM_SEND_ISTREAM_RESULT_FINISHED: + ctx->finished = TRUE; + return test_server_response_ostream_disconnect_output(ctx); + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT: + i_unreached(); + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT: + if (debug) + i_debug("WAIT OUTPUT"); + return 1; + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT: + i_unreached(); + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT: + if (debug) { + i_debug("OUTPUT ERROR: %s", + o_stream_get_error(output)); + } + test_assert(output->stream_errno == ECONNRESET || + output->stream_errno == EPIPE); + + ctx->seen_stream_error = TRUE; + o_stream_destroy(&ctx->payload_output); + return -1; + } + i_unreached(); +} + +static void +test_server_response_ostream_disconnect_request(struct http_server_request *req) +{ + const struct http_request *hreq = http_server_request_get(req); + struct http_server_response *resp; + struct _response_ostream_disconnect *ctx; + string_t *data; + unsigned int i; + + if (debug) { + i_debug("REQUEST: %s %s HTTP/%u.%u", + hreq->method, hreq->target_raw, + hreq->version_major, hreq->version_minor); + } + + ctx = i_new(struct _response_ostream_disconnect, 1); + ctx->req = req; + + data = str_new(default_pool, 2048000); + for (i = 0; i < 32000; i++) { + str_append(data, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n"); + } + ctx->payload_input = i_stream_create_copy_from_data( + str_data(data), str_len(data)); + str_free(&data); + + resp = http_server_response_create(req, 200, "OK"); + ctx->payload_output = http_server_response_get_payload_output( + resp, IO_BLOCK_SIZE, FALSE); + + o_stream_add_destroy_callback( + ctx->payload_output, + test_server_response_ostream_disconnect_destroyed, ctx); + + o_stream_set_flush_callback( + ctx->payload_output, + test_server_response_ostream_disconnect_output, ctx); + o_stream_set_flush_pending(ctx->payload_output, TRUE); +} + +static void +test_server_response_ostream_disconnect( + const struct http_server_settings *server_set) +{ + test_server_request = test_server_response_ostream_disconnect_request; + test_server_run(server_set); +} + +/* test */ + +static void test_response_ostream_disconnect(void) +{ + struct http_server_settings http_server_set; + + test_server_defaults(&http_server_set); + http_server_set.socket_send_buffer_size = 4096; + http_server_set.max_client_idle_time_msecs = 10000; + + test_begin("response ostream disconnect"); + test_run_client_server(&http_server_set, + test_server_response_ostream_disconnect, + test_client_response_ostream_disconnect, 1); + test_end(); +} + +/* + * All tests + */ + +static void (*const test_functions[])(void) = { + test_slow_request, + test_hanging_request_payload, + test_hanging_response_payload, + test_excessive_payload_length, + test_response_ostream_disconnect, + NULL +}; + +/* + * Test client + */ + +/* client connection */ + +static void client_connection_input(struct connection *_conn) +{ + struct client_connection *conn = (struct client_connection *)_conn; + + if (test_client_input != NULL) + test_client_input(conn); +} + +static void client_connection_connected(struct connection *_conn, bool success) +{ + struct client_connection *conn = (struct client_connection *)_conn; + + if (success && test_client_connected != NULL) + test_client_connected(conn); +} + +static void client_connection_init(const struct ip_addr *ip, in_port_t port) +{ + struct client_connection *conn; + pool_t pool; + + pool = pool_alloconly_create("client connection", 512); + conn = p_new(pool, struct client_connection, 1); + conn->pool = pool; + + connection_init_client_ip(client_conn_list, &conn->conn, NULL, + ip, port); + (void)connection_client_connect(&conn->conn); +} + +static void client_connection_deinit(struct client_connection **_conn) +{ + struct client_connection *conn = *_conn; + + *_conn = NULL; + + connection_deinit(&conn->conn); + pool_unref(&conn->pool); +} + +static void client_connection_destroy(struct connection *_conn) +{ + struct client_connection *conn = (struct client_connection *)_conn; + + client_connection_deinit(&conn); +} + +/* */ + +static struct connection_settings client_connection_set = { + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .client = TRUE +}; + +static const struct connection_vfuncs client_connection_vfuncs = { + .destroy = client_connection_destroy, + .client_connected = client_connection_connected, + .input = client_connection_input +}; + +static void test_client_run(unsigned int index) +{ + client_index = index; + + if (debug) + i_debug("client connecting to %u", bind_port); + + client_conn_list = connection_list_init(&client_connection_set, + &client_connection_vfuncs); + + client_connection_init(&bind_ip, bind_port); + + io_loop_run(ioloop); + + /* close server socket */ + io_remove(&io_listen); + + connection_list_deinit(&client_conn_list); +} + +/* + * Test server + */ + +static void test_server_defaults(struct http_server_settings *http_set) +{ + /* server settings */ + i_zero(http_set); + http_set->max_client_idle_time_msecs = 5*1000; + http_set->max_pipelined_requests = 1; + http_set->debug = debug; +} + +/* client connection */ + +static void +server_handle_request(void *context ATTR_UNUSED, + struct http_server_request *req) +{ + test_server_request(req); +} + +struct http_server_callbacks http_server_callbacks = { + .handle_request = server_handle_request +}; + +static void server_connection_accept(void *context ATTR_UNUSED) +{ + int fd; + + /* accept new client */ + fd = net_accept(fd_listen, NULL, NULL); + if (fd == -1) + return; + if (fd == -2) { + i_fatal("test server: accept() failed: %m"); + } + + (void)http_server_connection_create(http_server, fd, fd, FALSE, + &http_server_callbacks, NULL); +} + +/* */ + +static void test_server_timeout(void *context ATTR_UNUSED) +{ + i_fatal("Server timed out"); +} + +static void test_server_run(const struct http_server_settings *http_set) +{ + struct timeout *to; + + to = timeout_add(SERVER_MAX_TIMEOUT_MSECS, test_server_timeout, NULL); + + /* open server socket */ + io_listen = io_add(fd_listen, IO_READ, server_connection_accept, NULL); + + http_server = http_server_init(http_set); + + io_loop_run(ioloop); + + /* close server socket */ + io_remove(&io_listen); + timeout_remove(&to); + + http_server_deinit(&http_server); +} + +/* + * Tests + */ + +struct test_client_data { + unsigned int index; + test_client_init_t client_test; +}; + +static int test_open_server_fd(void) +{ + int fd = net_listen(&bind_ip, &bind_port, 128); + if (debug) + i_debug("server listening on %u", bind_port); + if (fd == -1) { + i_fatal("listen(%s:%u) failed: %m", + net_ip2addr(&bind_ip), bind_port); + } + return fd; +} + +static int test_run_client(struct test_client_data *data) +{ + i_close_fd(&fd_listen); + + i_set_failure_prefix("CLIENT[%u]: ", data->index + 1); + + if (debug) + i_debug("PID=%s", my_pid); + + /* Wait a little for server setup */ + i_sleep_msecs(100); + + ioloop = io_loop_create(); + data->client_test(data->index); + io_loop_destroy(&ioloop); + + if (debug) + i_debug("Terminated"); + + main_deinit(); + return 0; +} + +static void +test_run_server(const struct http_server_settings *server_set, + test_server_init_t server_test) +{ + i_set_failure_prefix("SERVER: "); + + if (debug) + i_debug("PID=%s", my_pid); + + ioloop = io_loop_create(); + server_test(server_set); + io_loop_destroy(&ioloop); + + if (debug) + i_debug("Terminated"); +} + +static void +test_run_client_server(const struct http_server_settings *server_set, + test_server_init_t server_test, + test_client_init_t client_test, + unsigned int client_tests_count) +{ + unsigned int i; + + fd_listen = test_open_server_fd(); + + if (client_tests_count > 0) { + for (i = 0; i < client_tests_count; i++) { + struct test_client_data data; + + i_zero(&data); + data.index = i; + data.client_test = client_test; + + /* Fork client */ + test_subprocess_fork(test_run_client, &data, FALSE); + } + } + + /* Run server */ + test_run_server(server_set, server_test); + + i_unset_failure_prefix(); + i_close_fd(&fd_listen); + test_subprocess_kill_all(CLIENT_KILL_TIMEOUT_SECS); +} + +/* + * Main + */ + +static void main_init(void) +{ + /* nothing yet */ +} + +static void main_deinit(void) +{ + /* nothing yet; also called from sub-processes */ +} + +int main(int argc, char *argv[]) +{ + int c; + int ret; + + lib_init(); + main_init(); + + while ((c = getopt(argc, argv, "D")) > 0) { + switch (c) { + case 'D': + debug = TRUE; + break; + default: + i_fatal("Usage: %s [-D]", argv[0]); + } + } + + test_subprocesses_init(debug); + + /* listen on localhost */ + i_zero(&bind_ip); + bind_ip.family = AF_INET; + bind_ip.u.ip4.s_addr = htonl(INADDR_LOOPBACK); + + ret = test_run(test_functions); + + test_subprocesses_deinit(); + main_deinit(); + lib_deinit(); + + return ret; +} diff --git a/src/lib-http/test-http-server.c b/src/lib-http/test-http-server.c new file mode 100644 index 0000000..ceec5ce --- /dev/null +++ b/src/lib-http/test-http-server.c @@ -0,0 +1,245 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "lib-signals.h" +#include "llist.h" +#include "str.h" +#include "ioloop.h" +#include "ostream.h" +#include "connection.h" +#include "http-url.h" +#include "http-server.h" + +#include <unistd.h> + +static int fd_listen; +static struct ioloop *ioloop; +static struct io *io_listen; +static struct http_server *http_server; +static bool shut_down = FALSE; +static struct client *clients_head = NULL, *clients_tail = NULL; + +struct client { + struct client *prev, *next; + + struct ip_addr server_ip, ip; + in_port_t server_port, port; + struct http_server_connection *http_conn; +}; + +static void +client_destroy(struct client **_client, const char *reason) +{ + struct client *client = *_client; + + if (client->http_conn != NULL) { + /* We're not in the lib-http/server's connection destroy callback. + If at all possible, avoid destroying client objects directly. + */ + http_server_connection_close(&client->http_conn, reason); + } + DLLIST2_REMOVE(&clients_head, &clients_tail, client); + i_free(client); + + if (clients_head == NULL) + io_loop_stop(ioloop); +} + +/* This function just serves as an illustration of what to do when client + objects are destroyed by some actor other than lib-http/server. The best way + to close all clients is to drop the whole http-server, which will close all + connections, which in turn calls the connection_destroy() callbacks. Using a + function like this just complicates matters. */ +static void +clients_destroy_all(void) +{ + while (clients_head != NULL) { + struct client *client = clients_head; + client_destroy(&client, "Shutting down server"); + } +} + +static void +client_http_handle_request(void *context, + struct http_server_request *req) +{ + struct client *client = (struct client *)context; + const struct http_request *http_req = http_server_request_get(req); + struct http_server_response *http_resp; + const char *ipport; + string_t *content; + + if (strcmp(http_req->method, "GET") != 0) { + /* Unsupported method */ + http_resp = http_server_response_create(req, 501, "Not Implemented"); + http_server_response_add_header(http_resp, "Allow", "GET"); + http_server_response_submit(http_resp); + return; + } + + /* Compose response payload */ + content = t_str_new(1024); + (void)net_ipport2str(&client->server_ip, client->server_port, &ipport); + str_printfa(content, "Server: %s\r\n", ipport); + (void)net_ipport2str(&client->ip, client->port, &ipport); + str_printfa(content, "Client: %s\r\n", ipport); + str_printfa(content, "Host: %s", http_req->target.url->host.name); + if (http_req->target.url->port != 0) + str_printfa(content, ":%u", http_req->target.url->port); + str_append(content, "\r\n"); + switch (http_req->target.format) { + case HTTP_REQUEST_TARGET_FORMAT_ORIGIN: + case HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE: + str_printfa(content, "Target: %s\r\n", + http_url_create(http_req->target.url)); + break; + case HTTP_REQUEST_TARGET_FORMAT_AUTHORITY: + str_printfa(content, "Target: %s\r\n", + http_url_create_authority(http_req->target.url)); + break; + case HTTP_REQUEST_TARGET_FORMAT_ASTERISK: + str_append(content, "Target: *\r\n"); + break; + } + + /* Just respond with the request target */ + http_resp = http_server_response_create(req, 200, "OK"); + http_server_response_add_header(http_resp, "Content-Type", "text/plain"); + http_server_response_set_payload_data(http_resp, + str_data(content), str_len(content)); + http_server_response_submit(http_resp); +} + +static void +client_http_connection_destroy(void *context, const char *reason) +{ + struct client *client = (struct client *)context; + + if (client->http_conn == NULL) { + /* already destroying client directly */ + return; + } + + /* HTTP connection is destroyed already now */ + client->http_conn = NULL; + + /* destroy the client itself */ + client_destroy(&client, reason); +} + +static const struct http_server_callbacks server_callbacks = { + .handle_request = client_http_handle_request, + .connection_destroy = client_http_connection_destroy +}; + +static void +client_init(int fd, const struct ip_addr *ip, in_port_t port) +{ + struct client *client; + struct http_request_limits req_limits; + + i_zero(&req_limits); + req_limits.max_target_length = 4096; + + client = i_new(struct client, 1); + client->ip = *ip; + client->port = port; + (void)net_getsockname(fd, &client->server_ip, &client->server_port); + client->http_conn = http_server_connection_create(http_server, + fd, fd, FALSE, &server_callbacks, client); + + DLLIST2_APPEND(&clients_head, &clients_tail, client); +} + +static void client_accept(void *context ATTR_UNUSED) +{ + struct ip_addr client_ip; + in_port_t client_port; + int fd; + + fd = net_accept(fd_listen, &client_ip, &client_port); + if (fd == -1) + return; + if (fd == -2) + i_fatal("accept() failed: %m"); + + client_init(fd, &client_ip, client_port); +} + +static void +sig_die(const siginfo_t *si ATTR_UNUSED, void *context ATTR_UNUSED) +{ + if (shut_down) { + i_info("Received SIGINT again - stopping immediately"); + io_loop_stop(current_ioloop); + return; + } + + i_info("Received SIGINT - shutting down gracefully"); + shut_down = TRUE; + http_server_shut_down(http_server); + if (clients_head == NULL) + io_loop_stop(ioloop); +} + +int main(int argc, char *argv[]) +{ + struct http_server_settings http_set; + bool debug = FALSE; + struct ip_addr my_ip; + in_port_t port; + int c; + + lib_init(); + + while ((c = getopt(argc, argv, "D")) > 0) { + switch (c) { + case 'D': + debug = TRUE; + break; + default: + i_fatal("Usage: %s [-D] <port> [<IP>]", argv[0]); + } + } + argc -= optind; + argv += optind; + + if (argc < 1 || net_str2port(argv[0], &port) < 0) + i_fatal("Port parameter missing"); + if (argc < 2) + my_ip = net_ip4_any; + else if (net_addr2ip(argv[1], &my_ip) < 0) + i_fatal("Invalid IP parameter"); + + i_zero(&http_set); + http_set.max_client_idle_time_msecs = 20*1000; /* defaults to indefinite! */ + http_set.max_pipelined_requests = 4; + http_set.debug = debug; + + ioloop = io_loop_create(); + + http_server = http_server_init(&http_set); + + lib_signals_init(); + lib_signals_ignore(SIGPIPE, TRUE); + lib_signals_set_handler(SIGTERM, LIBSIG_FLAG_DELAYED, sig_die, NULL); + lib_signals_set_handler(SIGINT, LIBSIG_FLAG_DELAYED, sig_die, NULL); + + fd_listen = net_listen(&my_ip, &port, 128); + if (fd_listen == -1) + i_fatal("listen(port=%u) failed: %m", port); + + io_listen = io_add(fd_listen, IO_READ, client_accept, NULL); + + io_loop_run(ioloop); + + io_remove(&io_listen); + i_close_fd(&fd_listen); + + clients_destroy_all(); /* just an example; avoid doing this */ + + http_server_deinit(&http_server); + lib_signals_deinit(); + io_loop_destroy(&ioloop); + lib_deinit(); +} diff --git a/src/lib-http/test-http-transfer.c b/src/lib-http/test-http-transfer.c new file mode 100644 index 0000000..39b08a6 --- /dev/null +++ b/src/lib-http/test-http-transfer.c @@ -0,0 +1,347 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "buffer.h" +#include "str.h" +#include "strfuncs.h" +#include "str-sanitize.h" +#include "istream.h" +#include "ostream.h" +#include "test-common.h" +#include "http-transfer.h" + +#include <time.h> + +struct http_transfer_chunked_input_test { + const char *in; + const char *out; +}; + +/* Valid transfer_chunked input tests */ +static struct http_transfer_chunked_input_test +valid_transfer_chunked_input_tests[] = { + { .in = "1E\r\n" + "This is a simple test payload." + "\r\n" + "0\r\n" + "\r\n", + .out = + "This is a simple test payload." + }, + { .in = "20\r\n" + "This is a longer test payload..." + "\r\n" + "23\r\n" + "...spread over two separate chunks." + "\r\n" + "0\r\n" + "\r\n", + .out = + "This is a longer test payload..." + "...spread over two separate chunks." + }, + { .in = "26\r\n" + "This is an even longer test payload..." + "\r\n" + "27\r\n" + "...spread over three separate chunks..." + "\r\n" + "1F\r\n" + "...and also includes a trailer." + "\r\n" + "0\r\n" + "Checksum: adgfef3fdaf3daf3dfaf3ff3fdag\r\n" + "X-Dovecot: Whatever\r\n" + "\r\n", + .out = + "This is an even longer test payload..." + "...spread over three separate chunks..." + "...and also includes a trailer." + }, + { .in = "26\n" + "This is an even longer test payload..." + "\n" + "27\n" + "...spread over three separate chunks..." + "\n" + "1F\n" + "...and also includes a trailer." + "\n" + "0\n" + "Checksum: adgfef3fdaf3daf3dfaf3ff3fdag\n" + "X-Dovecot: Whatever\n" + "\n", + .out = + "This is an even longer test payload..." + "...spread over three separate chunks..." + "...and also includes a trailer." + } +}; + +static unsigned int valid_transfer_chunked_input_test_count = + N_ELEMENTS(valid_transfer_chunked_input_tests); + +static void test_http_transfer_chunked_input_valid(void) +{ + struct istream *input, *chunked; + struct ostream *output; + buffer_t *payload_buffer; + unsigned int i; + + payload_buffer = buffer_create_dynamic(default_pool, 1024); + + for (i = 0; i < valid_transfer_chunked_input_test_count; i++) T_BEGIN { + const char *in, *out, *stream_out; + + in = valid_transfer_chunked_input_tests[i].in; + out = valid_transfer_chunked_input_tests[i].out; + + test_begin(t_strdup_printf("http transfer_chunked input valid [%d]", i)); + + input = i_stream_create_from_data(in, strlen(in)); + chunked = http_transfer_chunked_istream_create(input, 0); + i_stream_unref(&input); + + buffer_set_used_size(payload_buffer, 0); + output = o_stream_create_buffer(payload_buffer); + test_out("payload read", o_stream_send_istream(output, chunked) == OSTREAM_SEND_ISTREAM_RESULT_FINISHED + && chunked->stream_errno == 0); + o_stream_destroy(&output); + i_stream_unref(&chunked); + stream_out = str_c(payload_buffer); + + test_out(t_strdup_printf("response->payload = %s", + str_sanitize(stream_out, 80)), + strcmp(stream_out, out) == 0); + test_end(); + } T_END; + + buffer_free(&payload_buffer); +} + +/* Invalid transfer_chunked input tests */ +static const char * +invalid_transfer_chunked_input_tests[] = { + // invalid size + "1X\r\n" + "This is a simple test payload." + "\r\n" + "0\r\n" + "\r\n", + // invalid end + "1E\r\n" + "This is a simple test payload." + "\r\n" + "0\r\n" + "ah\r\n", + // invalid size + "20\r\n" + "This is a longer test payload..." + "\r\n" + "2q\r\n" + "...spread over two separate chunks." + "\r\n" + "0\r\n" + "\r\n", + // invalid end + "20\r\n" + "This is a longer test payload..." + "\r\n" + "23\r\n" + "...spread over two separate chunks." + "\r\n" + "0\r\n", + // invalid last chunk + "20\r\n" + "This is a longer test payload..." + "\r\n" + "23\r\n" + "...spread over two separate chunks." + "\r\n" + "4\r\n" + "\r\n", + // invalid trailer + "26\r\n" + "This is an even longer test payload..." + "\r\n" + "27\r\n" + "...spread over three separate chunks..." + "\r\n" + "1F\r\n" + "...and also includes a trailer." + "\r\n" + "0\r\n" + "Checksum adgfef3fdaf3daf3dfaf3ff3fdag\r\n" + "\r\n" +}; + +static unsigned int invalid_transfer_chunked_input_test_count = + N_ELEMENTS(invalid_transfer_chunked_input_tests); + +static void test_http_transfer_chunked_input_invalid(void) +{ + struct istream *input, *chunked; + struct ostream *output; + buffer_t *payload_buffer; + unsigned int i; + + payload_buffer = buffer_create_dynamic(default_pool, 1024); + + for (i = 0; i < invalid_transfer_chunked_input_test_count; i++) T_BEGIN { + const char *in; + + in = invalid_transfer_chunked_input_tests[i]; + + test_begin(t_strdup_printf("http transfer_chunked input invalid [%d]", i)); + + input = i_stream_create_from_data(in, strlen(in)); + chunked = http_transfer_chunked_istream_create(input, 0); + i_stream_unref(&input); + + buffer_set_used_size(payload_buffer, 0); + output = o_stream_create_buffer(payload_buffer); + o_stream_nsend_istream(output, chunked); + test_out("payload read failure", chunked->stream_errno != 0); + i_stream_unref(&chunked); + o_stream_destroy(&output); + + test_end(); + } T_END; + + buffer_free(&payload_buffer); +} + +/* Valid transfer_chunked output tests */ +static const char *valid_transfer_chunked_output_tests[] = { + /* The maximum chunk size is set to 16. These tests are tuned to some border + cases + */ + "A small payload", // 15 bytes + "A longer payload", // 16 bytes + "A lengthy payload", // 17 bytes + /* Others */ + "This is a test payload with lots of nonsense.", + "Yet another payload.", + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " + "This a very long repetitive payload. This a very long repetitive payload. " +}; + +static unsigned int valid_transfer_chunked_output_test_count = + N_ELEMENTS(valid_transfer_chunked_output_tests); + +static void test_http_transfer_chunked_output_valid(void) +{ + struct istream *input, *ichunked; + struct ostream *output, *ochunked; + buffer_t *chunked_buffer, *plain_buffer; + unsigned int i; + + chunked_buffer = buffer_create_dynamic(default_pool, 1024); + plain_buffer = buffer_create_dynamic(default_pool, 1024); + + for (i = 0; i < valid_transfer_chunked_output_test_count; i++) T_BEGIN { + const char *data, *stream_out; + const unsigned char *rdata; + size_t rsize; + ssize_t ret; + + data = valid_transfer_chunked_output_tests[i]; + + test_begin(t_strdup_printf("http transfer_chunked output valid [%d]", i)); + + /* create input stream */ + input = i_stream_create_from_data(data, strlen(data)); + + /* create buffer output stream */ + buffer_set_used_size(chunked_buffer, 0); + output = o_stream_create_buffer(chunked_buffer); + + /* create chunked output stream */ + ochunked = http_transfer_chunked_ostream_create(output); + + /* send input through chunked stream; chunk size is limited */ + for (;;) { + ret = i_stream_read_more(input, &rdata, &rsize); + if (ret < 0) { + if (input->eof) + ret = 1; + break; + } + if (rsize == 0) + break; + if (rsize > 16) + rsize = 16; + + ret = o_stream_send(ochunked, rdata, rsize); + if (ret < 0) + break; + + if ((size_t)ret != rsize) { + ret = -1; + break; + } + + i_stream_skip(input, ret); + } + + /* cleanup streams */ + test_out("payload chunk", ret > 0); + test_assert(o_stream_finish(ochunked) > 0); + o_stream_destroy(&ochunked); + o_stream_destroy(&output); + i_stream_destroy(&input); + + /* create chunked input stream */ + input = i_stream_create_from_data + (chunked_buffer->data, chunked_buffer->used); + ichunked = http_transfer_chunked_istream_create(input, 0); + + /* read back chunk */ + buffer_set_used_size(plain_buffer, 0); + output = o_stream_create_buffer(plain_buffer); + test_out("payload unchunk", + o_stream_send_istream(output, ichunked) == OSTREAM_SEND_ISTREAM_RESULT_FINISHED + && ichunked->stream_errno == 0); + o_stream_destroy(&output); + i_stream_destroy(&ichunked); + i_stream_destroy(&input); + + /* test output */ + stream_out = str_c(plain_buffer); + test_out(t_strdup_printf("response->payload = %s", + str_sanitize(stream_out, 80)), + strcmp(stream_out, data) == 0); + test_end(); + } T_END; + + buffer_free(&chunked_buffer); + buffer_free(&plain_buffer); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_http_transfer_chunked_input_valid, + test_http_transfer_chunked_input_invalid, + test_http_transfer_chunked_output_valid, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-http/test-http-url.c b/src/lib-http/test-http-url.c new file mode 100644 index 0000000..a9fdf24 --- /dev/null +++ b/src/lib-http/test-http-url.c @@ -0,0 +1,950 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "net.h" +#include "http-url.h" +#include "test-common.h" + +struct valid_http_url_test { + const char *url; + enum http_url_parse_flags flags; + struct http_url url_base; + + struct http_url url_parsed; +}; + +/* Valid HTTP URL tests */ +static struct valid_http_url_test valid_url_tests[] = { + /* Generic tests */ + { + .url = "http://localhost", + .url_parsed = { + .host = { .name = "localhost" }, + }, + }, + { + .url = "http://www.%65%78%61%6d%70%6c%65.com", + .url_parsed = { + .host = { .name = "www.example.com" }, + }, + }, + { + .url = "http://www.dovecot.org:8080", + .url_parsed = { + .host = { .name = "www.dovecot.org" }, + .port = 8080, + }, + }, + { + .url = "http://127.0.0.1", + .url_parsed = { + .host = { + .name = "127.0.0.1", + .ip = { .family = AF_INET }, + }, + }, + }, + { + .url = "http://[::1]", + .url_parsed = { + .host = { + .name = "[::1]", + .ip = { .family = AF_INET6 }, + }, + }, + }, + { + .url = "http://[::1]:8080", + .url_parsed = { + .host = { + .name = "[::1]", + .ip = { .family = AF_INET6 }, + }, + .port = 8080, + }, + }, + { + .url = "http://user@api.dovecot.org", + .flags = HTTP_URL_ALLOW_USERINFO_PART, + .url_parsed = { + .host = { .name = "api.dovecot.org" }, + .user = "user", + }, + }, + { + .url = "http://userid:secret@api.dovecot.org", + .flags = HTTP_URL_ALLOW_USERINFO_PART, + .url_parsed = { + .host = { .name = "api.dovecot.org" }, + .user = "userid", + .password = "secret", + }, + }, + { + .url = "http://su%3auserid:secret@api.dovecot.org", + .flags = HTTP_URL_ALLOW_USERINFO_PART, + .url_parsed = { + .host = { .name = "api.dovecot.org" }, + .user = "su:userid", + .password = "secret", + }, + }, + { + .url = "http://www.example.com/" + "?question=What%20are%20you%20doing%3f&answer=Nothing.", + .url_parsed = { + .path = "/", + .host = { .name = "www.example.com" }, + .enc_query = "question=What%20are%20you%20doing%3f&answer=Nothing.", + }, + }, + /* Empty path segments */ + { + .url = "http://target//index.php", + .url_parsed = { + .path = "//index.php", + .host = { .name = "target" }, + }, + }, + { + .url = "http://target//path//index.php", + .url_parsed = { + .path = "//path//index.php", + .host = { .name = "target" }, + }, + }, + { + .url = "http://target//path/", + .url_parsed = { + .path = "//path/", + .host = { .name = "target" }, + }, + }, + { + .url = "http://target//path//", + .url_parsed = { + .path = "//path//", + .host = { .name = "target" }, + }, + }, + { + .url = "http://target//path//to//./index.php", + .url_parsed = { + .path = "//path//to//index.php", + .host = { .name = "target" }, + }, + }, + { + .url = "http://target//path//to//../index.php", + .url_parsed = { + .path = "//path//to/index.php", + .host = { .name = "target" }, + }, + }, + { + .url = "/index.php", + .url_base = { + .host = { .name = "target" }, + }, + .url_parsed = { + .host = { .name = "target" }, + .path = "/index.php", + }, + }, + { + .url = "//index.php", + .url_base = { + .host = { .name = "target" }, + }, + .url_parsed = { + .host = { .name = "index.php" }, + }, + }, + { + .url = "/path/to/index.php", + .url_base = { + .host = { .name = "target" }, + }, + .url_parsed = { + .host = { .name = "target" }, + .path = "/path/to/index.php", + }, + }, + { + .url = "//path//to//index.php", + .url_base = { + .host = { .name = "target" }, + }, + .url_parsed = { + .host = { .name = "path" }, + .path = "//to//index.php", + }, + }, + /* These next 2 URLs don't follow the recommendations in + http://tools.ietf.org/html/rfc1034#section-3.5 and + http://tools.ietf.org/html/rfc3696 + However they satisfy the grammar in + http://tools.ietf.org/html/rfc1123#section-2 and + http://tools.ietf.org/html/rfc952 + so we should parse them. + */ + { + .url = "http://256.0.0.1/that/reverts/to/DNS", + .url_parsed = { + .path = "/that/reverts/to/DNS", + .host = { .name = "256.0.0.1" }, + }, + }, + { + .url = "http://127.0.0.284/this/also/reverts/to/DNS", + .url_parsed = { + .path = "/this/also/reverts/to/DNS", + .host = { .name = "127.0.0.284" }, + }, + }, + { + .url = "http://www.example.com/#Status%20of%20development", + .flags = HTTP_URL_ALLOW_FRAGMENT_PART, + .url_parsed = { + .path = "/", + .host = { .name = "www.example.com" }, + .enc_fragment = "Status%20of%20development", + }, + }, + /* RFC 3986, Section 5.4. Reference Resolution Examples + * + * Within a representation with a well defined base URI of + * + * http://a/b/c/d;p?q + * + * a relative reference is transformed to its target URI as follows. + * + * 5.4.1. Normal Examples + */ + { // "g" = "http://a/b/c/g" + .url = "g", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g", + }, + }, + { // "./g" = "http://a/b/c/g" + .url = "./g", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g", + }, + }, + { // "g/" = "http://a/b/c/g/" + .url = "g/", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g/", + }, + }, + { // "/g" = "http://a/g" + .url = "/g", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/g", + }, + }, + { // "//g" = "http://g" + .url = "//g", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "g" }, + }, + }, + { // "?y" = "http://a/b/c/d;p?y" + .url = "?y", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "y", + }, + }, + { // "g?y" = "http://a/b/c/g?y" + .url = "g?y", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g", + .enc_query = "y", + }, + }, + { // "#s" = "http://a/b/c/d;p?q#s" + .url = "#s", + .flags = HTTP_URL_ALLOW_FRAGMENT_PART, + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + .enc_fragment = "s", + }, + }, + { // "g#s" = "http://a/b/c/g#s" + .url = "g#s", + .flags = HTTP_URL_ALLOW_FRAGMENT_PART, + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g", + .enc_fragment = "s", + }, + }, + { // "g?y#s" = "http://a/b/c/g?y#s" + .url = "g?y#s", + .flags = HTTP_URL_ALLOW_FRAGMENT_PART, + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g", + .enc_query = "y", + .enc_fragment = "s", + }, + }, + { // ";x" = "http://a/b/c/;x" + .url = ";x", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/;x", + }, + }, + { // "g;x" = "http://a/b/c/g;x" + .url = "g;x", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g;x", + }, + + }, + { // "g;x?y#s" = "http://a/b/c/g;x?y#s" + .url = "g;x?y#s", + .flags = HTTP_URL_ALLOW_FRAGMENT_PART, + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g;x", + .enc_query = "y", + .enc_fragment = "s", + }, + }, + { // "" = "http://a/b/c/d;p?q" + .url = "", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + }, + { // "." = "http://a/b/c/" + .url = ".", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/", + }, + }, + { // "./" = "http://a/b/c/" + .url = "./", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/", + }, + }, + { // ".." = "http://a/b/" + .url = "..", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/", + }, + }, + { // "../" = "http://a/b/" + .url = "../", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/", + }, + }, + { // "../g" = "http://a/b/g" + .url = "../g", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/g", + }, + }, + { // "../.." = "http://a/" + .url = "../..", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/", + }, + }, + { // "../../" = "http://a/" + .url = "../../", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/", + }, + }, + { // "../../g" = "http://a/g" + .url = "../../g", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/g", + }, + }, + /* 5.4.2. Abnormal Examples + */ + { // "../../../g" = "http://a/g" + .url = "../../../g", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/g", + }, + }, + { // "../../../../g" = "http://a/g" + .url = "../../../../g", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/g", + }, + }, + { // "/./g" = "http://a/g" + .url = "/./g", + .url_base = { + .host = {.name = "a"}, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = {.name = "a"}, + .path = "/g", + }, + }, + { // "/../g" = "http://a/g" + .url = "/../g", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/g", + }, + }, + { // "g." = "http://a/b/c/g." + .url = "g.", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g.", + }, + }, + { // ".g" = "http://a/b/c/.g" + .url = ".g", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/.g", + }, + }, + { // "g.." = "http://a/b/c/g.." + .url = "g..", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g..", + }, + }, + { // "..g" = "http://a/b/c/..g" + .url = "..g", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/..g", + }, + }, + { // "./../g" = "http://a/b/g" + .url = "./../g", + .url_base = { + .host = {.name = "a"}, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = {.name = "a"}, + .path = "/b/g", + }, + }, + { // "./g/." = "http://a/b/c/g/" + .url = "./g/.", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g/", + }, + }, + { // "g/./h" = "http://a/b/c/g/h" + .url = "g/./h", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g/h", + }, + }, + { // "g/../h" = "http://a/b/c/h" + .url = "g/../h", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/h", + }, + }, + { // "g;x=1/./y" = "http://a/b/c/g;x=1/y" + .url = "g;x=1/./y", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g;x=1/y", + }, + }, + { // "g;x=1/../y" = "http://a/b/c/y" + .url = "g;x=1/../y", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/y", + }, + }, + { // "g?y/./x" = "http://a/b/c/g?y/./x" + .url = "g?y/./x", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g", + .enc_query = "y/./x", + }, + }, + { // "g?y/../x" = "http://a/b/c/g?y/../x" + .url = "g?y/../x", + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = { + .host = { .name = "a" }, + .path = "/b/c/g", + .enc_query = "y/../x", + }, + }, + { // "g#s/./x" = "http://a/b/c/g#s/./x" + .url = "g#s/./x", + .flags = HTTP_URL_ALLOW_FRAGMENT_PART, + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = + { + .host = { .name = "a" }, + .path = "/b/c/g", + .enc_fragment = "s/./x", + }, + }, + { // "g#s/../x" = "http://a/b/c/g#s/../x" + .url = "g#s/../x", + .flags = HTTP_URL_ALLOW_FRAGMENT_PART, + .url_base = { + .host = { .name = "a" }, + .path = "/b/c/d;p", + .enc_query = "q", + }, + .url_parsed = + { + .host = { .name = "a" }, + .path = "/b/c/g", + .enc_fragment = "s/../x", + }, + } +}; + +static unsigned int valid_url_test_count = N_ELEMENTS(valid_url_tests); + +static void +test_http_url_equal(struct http_url *urlt, struct http_url *urlp) +{ + if (urlp->host.name == NULL || urlt->host.name == NULL) { + test_assert(urlp->host.name == urlt->host.name); + } else { + test_assert(strcmp(urlp->host.name, urlt->host.name) == 0); + } + test_assert(urlp->port == urlt->port); + test_assert(urlp->host.ip.family == urlt->host.ip.family); + if (urlp->user == NULL || urlt->user == NULL) { + test_assert(urlp->user == urlt->user); + } else { + test_assert(strcmp(urlp->user, urlt->user) == 0); + } + if (urlp->password == NULL || urlt->password == NULL) { + test_assert(urlp->password == urlt->password); + } else { + test_assert(strcmp(urlp->password, urlt->password) == 0); + } + if (urlp->path == NULL || urlt->path == NULL) { + test_assert(urlp->path == urlt->path); + } else { + test_assert(strcmp(urlp->path, urlt->path) == 0); + } + if (urlp->enc_query == NULL || urlt->enc_query == NULL) { + test_assert(urlp->enc_query == urlt->enc_query); + } else { + test_assert(strcmp(urlp->enc_query, urlt->enc_query) == 0); + } + if (urlp->enc_fragment == NULL || urlt->enc_fragment == NULL) { + test_assert(urlp->enc_fragment == urlt->enc_fragment); + } else { + test_assert(strcmp(urlp->enc_fragment, + urlt->enc_fragment) == 0); + } +} + +static void test_http_url_valid(void) +{ + unsigned int i; + + for (i = 0; i < valid_url_test_count; i++) T_BEGIN { + const char *url = valid_url_tests[i].url; + enum http_url_parse_flags flags = valid_url_tests[i].flags; + struct http_url *urlt = &valid_url_tests[i].url_parsed; + struct http_url *urlb = &valid_url_tests[i].url_base; + struct http_url *urlp; + const char *error = NULL; + + test_begin(t_strdup_printf("http url valid [%d]", i)); + + if (urlb->host.name == NULL) urlb = NULL; + if (http_url_parse(url, urlb, flags, pool_datastack_create(), + &urlp, &error) < 0) + urlp = NULL; + + test_out_reason(t_strdup_printf("http_url_parse(%s)", + valid_url_tests[i].url), urlp != NULL, error); + if (urlp != NULL) + test_http_url_equal(urlt, urlp); + + test_end(); + } T_END; +} + +struct invalid_http_url_test { + const char *url; + enum http_url_parse_flags flags; + struct http_url url_base; +}; + +static struct invalid_http_url_test invalid_url_tests[] = { + { + .url = "imap://example.com/INBOX" + }, + { + .url = "http:/www.example.com" + }, + { + .url = "" + }, + { + .url = "/index.html" + }, + { + .url = "http://www.example.com/index.html\"" + }, + { + .url = "http:///dovecot.org" + }, + { + .url = "http://[]/index.html" + }, + { + .url = "http://[v08.234:232:234:234:2221]/index.html" + }, + { + .url = "http://[1::34a:34:234::6]/index.html" + }, + { + .url = "http://example%a.com/index.html" + }, + { + .url = "http://example.com%/index.html" + }, + { + .url = "http://example%00.com/index.html" + }, + { + .url = "http://example.com:65536/index.html" + }, + { + .url = "http://example.com:72817/index.html" + }, + { + .url = "http://example.com/settings/%00/" + }, + { + .url = "http://example.com/settings/%0r/" + }, + { + .url = "http://example.com/settings/misc/%/" + }, + { + .url = "http://example.com/?%00" + }, + { + .url = "http://www.example.com/network.html#IMAP_Server" + }, + { + .url = "http://example.com/#%00", + .flags = HTTP_URL_ALLOW_FRAGMENT_PART + } +}; + +static unsigned int invalid_url_test_count = N_ELEMENTS(invalid_url_tests); + +static void test_http_url_invalid(void) +{ + unsigned int i; + + for (i = 0; i < invalid_url_test_count; i++) T_BEGIN { + const char *url = invalid_url_tests[i].url; + enum http_url_parse_flags flags = invalid_url_tests[i].flags; + struct http_url *urlb = &invalid_url_tests[i].url_base; + struct http_url *urlp; + const char *error = NULL; + + if (urlb->host.name == NULL) + urlb = NULL; + + test_begin(t_strdup_printf("http url invalid [%d]", i)); + + if (http_url_parse(url, urlb, flags, + pool_datastack_create(), &urlp, &error) < 0) + urlp = NULL; + test_out_reason(t_strdup_printf("parse %s", url), + urlp == NULL, error); + + test_end(); + } T_END; + +} + +static const char *parse_create_url_tests[] = { + "http://www.example.com/", + "http://10.0.0.1/", + "http://[::1]/", + "http://www.example.com:993/", + "http://www.example.com/index.html", + "http://www.example.com/settings/index.html", + "http://www.example.com/%23shared/news", + "http://www.example.com/query.php?name=Hendrik%20Visser", + "http://www.example.com/network.html#IMAP%20Server", +}; + +static unsigned int +parse_create_url_test_count = N_ELEMENTS(parse_create_url_tests); + +static void test_http_url_parse_create(void) +{ + unsigned int i; + + for (i = 0; i < parse_create_url_test_count; i++) T_BEGIN { + const char *url = parse_create_url_tests[i]; + struct http_url *urlp; + const char *error = NULL; + + test_begin(t_strdup_printf("http url parse/create [%d]", i)); + + if (http_url_parse + (url, NULL, HTTP_URL_ALLOW_FRAGMENT_PART, + pool_datastack_create(), &urlp, &error) < 0) + urlp = NULL; + test_out_reason(t_strdup_printf("parse %s", url), + urlp != NULL, error); + if (urlp != NULL) { + const char *urlnew = http_url_create(urlp); + test_out(t_strdup_printf("create %s", urlnew), + strcmp(url, urlnew) == 0); + } + + test_end(); + } T_END; + +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_http_url_valid, + test_http_url_invalid, + test_http_url_parse_create, + NULL + }; + return test_run(test_functions); +} + |