Description: import http2 module from 2.4.41 There are too many changes in http2 module to distiguish CVE-2019-9517, CVE-2019-10082 and CVE-2019-10081 changes. Author: Apache authors Bug: https://security-tracker.debian.org/tracker/CVE-2019-9517 https://security-tracker.debian.org/tracker/CVE-2019-10082 https://security-tracker.debian.org/tracker/CVE-2019-10081 https://security-tracker.debian.org/tracker/CVE-2020-9490 https://security-tracker.debian.org/tracker/CVE-2020-11993 Forwarded: not-needed Reviewed-By: Xavier Guimard Last-Update: 2020-08-25 --- a/modules/http2/config2.m4 +++ b/modules/http2/config2.m4 @@ -31,7 +31,6 @@ h2_h2.lo dnl h2_headers.lo dnl h2_mplx.lo dnl -h2_ngn_shed.lo dnl h2_push.lo dnl h2_request.lo dnl h2_session.lo dnl --- a/modules/http2/h2.h +++ b/modules/http2/h2.h @@ -48,12 +48,12 @@ #define H2_HEADER_PATH_LEN 5 #define H2_CRLF "\r\n" -/* Max data size to write so it fits inside a TLS record */ -#define H2_DATA_CHUNK_SIZE ((16*1024) - 100 - 9) - /* Size of the frame header itself in HTTP/2 */ #define H2_FRAME_HDR_LEN 9 +/* Max data size to write so it fits inside a TLS record */ +#define H2_DATA_CHUNK_SIZE ((16*1024) - 100 - H2_FRAME_HDR_LEN) + /* Maximum number of padding bytes in a frame, rfc7540 */ #define H2_MAX_PADLEN 256 /* Initial default window size, RFC 7540 ch. 6.5.2 */ @@ -138,7 +138,7 @@ apr_table_t *headers; apr_time_t request_time; - unsigned int chunked : 1; /* iff requst body needs to be forwarded as chunked */ + unsigned int chunked : 1; /* iff request body needs to be forwarded as chunked */ unsigned int serialize : 1; /* iff this request is written in HTTP/1.1 serialization */ apr_off_t raw_bytes; /* RAW network bytes that generated this request - if known. */ }; @@ -162,5 +162,6 @@ #define H2_FILTER_DEBUG_NOTE "http2-debug" #define H2_HDR_CONFORMANCE "http2-hdr-conformance" #define H2_HDR_CONFORMANCE_UNSAFE "unsafe" +#define H2_PUSH_MODE_NOTE "http2-push-mode" #endif /* defined(__mod_h2__h2__) */ --- a/modules/http2/h2_alt_svc.c +++ b/modules/http2/h2_alt_svc.c @@ -75,7 +75,7 @@ static int h2_alt_svc_handler(request_rec *r) { - const h2_config *cfg; + apr_array_header_t *alt_svcs; int i; if (r->connection->keepalives > 0) { @@ -87,8 +87,8 @@ return DECLINED; } - cfg = h2_config_sget(r->server); - if (r->hostname && cfg && cfg->alt_svcs && cfg->alt_svcs->nelts > 0) { + alt_svcs = h2_config_alt_svcs(r); + if (r->hostname && alt_svcs && alt_svcs->nelts > 0) { const char *alt_svc_used = apr_table_get(r->headers_in, "Alt-Svc-Used"); if (!alt_svc_used) { /* We have alt-svcs defined and client is not already using @@ -99,7 +99,7 @@ const char *alt_svc = ""; const char *svc_ma = ""; int secure = h2_h2_is_tls(r->connection); - int ma = h2_config_geti(cfg, H2_CONF_ALT_SVC_MAX_AGE); + int ma = h2_config_rgeti(r, H2_CONF_ALT_SVC_MAX_AGE); if (ma >= 0) { svc_ma = apr_psprintf(r->pool, "; ma=%d", ma); } @@ -107,8 +107,8 @@ "h2_alt_svc: announce %s for %s:%d", (secure? "secure" : "insecure"), r->hostname, (int)r->server->port); - for (i = 0; i < cfg->alt_svcs->nelts; ++i) { - h2_alt_svc *as = h2_alt_svc_IDX(cfg->alt_svcs, i); + for (i = 0; i < alt_svcs->nelts; ++i) { + h2_alt_svc *as = h2_alt_svc_IDX(alt_svcs, i); const char *ahost = as->host; if (ahost && !apr_strnatcasecmp(ahost, r->hostname)) { ahost = NULL; --- a/modules/http2/h2_bucket_beam.c +++ b/modules/http2/h2_bucket_beam.c @@ -196,7 +196,7 @@ * bucket beam that can transport buckets across threads ******************************************************************************/ -static void mutex_leave(void *ctx, apr_thread_mutex_t *lock) +static void mutex_leave(apr_thread_mutex_t *lock) { apr_thread_mutex_unlock(lock); } @@ -217,7 +217,7 @@ static void leave_yellow(h2_bucket_beam *beam, h2_beam_lock *pbl) { if (pbl->leave) { - pbl->leave(pbl->leave_ctx, pbl->mutex); + pbl->leave(pbl->mutex); } } --- a/modules/http2/h2_bucket_beam.h +++ b/modules/http2/h2_bucket_beam.h @@ -126,12 +126,11 @@ * buffers until the transmission is complete. Star gates use a similar trick. */ -typedef void h2_beam_mutex_leave(void *ctx, struct apr_thread_mutex_t *lock); +typedef void h2_beam_mutex_leave(struct apr_thread_mutex_t *lock); typedef struct { apr_thread_mutex_t *mutex; h2_beam_mutex_leave *leave; - void *leave_ctx; } h2_beam_lock; typedef struct h2_bucket_beam h2_bucket_beam; --- a/modules/http2/h2_config.c +++ b/modules/http2/h2_config.c @@ -42,6 +42,55 @@ #define H2_CONFIG_GET(a, b, n) \ (((a)->n == DEF_VAL)? (b) : (a))->n +#define H2_CONFIG_SET(a, n, v) \ + ((a)->n = v) + +#define CONFIG_CMD_SET(cmd,dir,var,val) \ + h2_config_seti(((cmd)->path? (dir) : NULL), h2_config_sget((cmd)->server), var, val) + +#define CONFIG_CMD_SET64(cmd,dir,var,val) \ + h2_config_seti64(((cmd)->path? (dir) : NULL), h2_config_sget((cmd)->server), var, val) + +/* Apache httpd module configuration for h2. */ +typedef struct h2_config { + const char *name; + int h2_max_streams; /* max concurrent # streams (http2) */ + int h2_window_size; /* stream window size (http2) */ + int min_workers; /* min # of worker threads/child */ + int max_workers; /* max # of worker threads/child */ + int max_worker_idle_secs; /* max # of idle seconds for worker */ + int stream_max_mem_size; /* max # bytes held in memory/stream */ + apr_array_header_t *alt_svcs; /* h2_alt_svc specs for this server */ + int alt_svc_max_age; /* seconds clients can rely on alt-svc info*/ + int serialize_headers; /* Use serialized HTTP/1.1 headers for + processing, better compatibility */ + int h2_direct; /* if mod_h2 is active directly */ + int modern_tls_only; /* Accept only modern TLS in HTTP/2 connections */ + int h2_upgrade; /* Allow HTTP/1 upgrade to h2/h2c */ + apr_int64_t tls_warmup_size; /* Amount of TLS data to send before going full write size */ + int tls_cooldown_secs; /* Seconds of idle time before going back to small TLS records */ + int h2_push; /* if HTTP/2 server push is enabled */ + struct apr_hash_t *priorities;/* map of content-type to h2_priority records */ + + int push_diary_size; /* # of entries in push diary */ + int copy_files; /* if files shall be copied vs setaside on output */ + apr_array_header_t *push_list;/* list of h2_push_res configurations */ + int early_hints; /* support status code 103 */ + int padding_bits; + int padding_always; +} h2_config; + +typedef struct h2_dir_config { + const char *name; + apr_array_header_t *alt_svcs; /* h2_alt_svc specs for this server */ + int alt_svc_max_age; /* seconds clients can rely on alt-svc info*/ + int h2_upgrade; /* Allow HTTP/1 upgrade to h2/h2c */ + int h2_push; /* if HTTP/2 server push is enabled */ + apr_array_header_t *push_list;/* list of h2_push_res configurations */ + int early_hints; /* support status code 103 */ +} h2_dir_config; + + static h2_config defconf = { "default", 100, /* max_streams */ @@ -64,6 +113,18 @@ 0, /* copy files across threads */ NULL, /* push list */ 0, /* early hints, http status 103 */ + 0, /* padding bits */ + 1, /* padding always */ +}; + +static h2_dir_config defdconf = { + "default", + NULL, /* no alt-svcs */ + -1, /* alt-svc max age */ + -1, /* HTTP/1 Upgrade support */ + -1, /* HTTP/2 server push enabled */ + NULL, /* push list */ + -1, /* early hints, http status 103 */ }; void h2_config_init(apr_pool_t *pool) @@ -71,12 +132,10 @@ (void)pool; } -static void *h2_config_create(apr_pool_t *pool, - const char *prefix, const char *x) +void *h2_config_create_svr(apr_pool_t *pool, server_rec *s) { h2_config *conf = (h2_config *)apr_pcalloc(pool, sizeof(h2_config)); - const char *s = x? x : "unknown"; - char *name = apr_pstrcat(pool, prefix, "[", s, "]", NULL); + char *name = apr_pstrcat(pool, "srv[", s->defn_name, "]", NULL); conf->name = name; conf->h2_max_streams = DEF_VAL; @@ -98,19 +157,11 @@ conf->copy_files = DEF_VAL; conf->push_list = NULL; conf->early_hints = DEF_VAL; + conf->padding_bits = DEF_VAL; + conf->padding_always = DEF_VAL; return conf; } -void *h2_config_create_svr(apr_pool_t *pool, server_rec *s) -{ - return h2_config_create(pool, "srv", s->defn_name); -} - -void *h2_config_create_dir(apr_pool_t *pool, char *x) -{ - return h2_config_create(pool, "dir", x); -} - static void *h2_config_merge(apr_pool_t *pool, void *basev, void *addv) { h2_config *base = (h2_config *)basev; @@ -149,25 +200,52 @@ n->push_list = add->push_list? add->push_list : base->push_list; } n->early_hints = H2_CONFIG_GET(add, base, early_hints); + n->padding_bits = H2_CONFIG_GET(add, base, padding_bits); + n->padding_always = H2_CONFIG_GET(add, base, padding_always); return n; } -void *h2_config_merge_dir(apr_pool_t *pool, void *basev, void *addv) +void *h2_config_merge_svr(apr_pool_t *pool, void *basev, void *addv) { return h2_config_merge(pool, basev, addv); } -void *h2_config_merge_svr(apr_pool_t *pool, void *basev, void *addv) +void *h2_config_create_dir(apr_pool_t *pool, char *x) { - return h2_config_merge(pool, basev, addv); + h2_dir_config *conf = (h2_dir_config *)apr_pcalloc(pool, sizeof(h2_dir_config)); + const char *s = x? x : "unknown"; + char *name = apr_pstrcat(pool, "dir[", s, "]", NULL); + + conf->name = name; + conf->alt_svc_max_age = DEF_VAL; + conf->h2_upgrade = DEF_VAL; + conf->h2_push = DEF_VAL; + conf->early_hints = DEF_VAL; + return conf; } -int h2_config_geti(const h2_config *conf, h2_config_var_t var) +void *h2_config_merge_dir(apr_pool_t *pool, void *basev, void *addv) { - return (int)h2_config_geti64(conf, var); + h2_dir_config *base = (h2_dir_config *)basev; + h2_dir_config *add = (h2_dir_config *)addv; + h2_dir_config *n = (h2_dir_config *)apr_pcalloc(pool, sizeof(h2_dir_config)); + + n->name = apr_pstrcat(pool, "merged[", add->name, ", ", base->name, "]", NULL); + n->alt_svcs = add->alt_svcs? add->alt_svcs : base->alt_svcs; + n->alt_svc_max_age = H2_CONFIG_GET(add, base, alt_svc_max_age); + n->h2_upgrade = H2_CONFIG_GET(add, base, h2_upgrade); + n->h2_push = H2_CONFIG_GET(add, base, h2_push); + if (add->push_list && base->push_list) { + n->push_list = apr_array_append(pool, base->push_list, add->push_list); + } + else { + n->push_list = add->push_list? add->push_list : base->push_list; + } + n->early_hints = H2_CONFIG_GET(add, base, early_hints); + return n; } -apr_int64_t h2_config_geti64(const h2_config *conf, h2_config_var_t var) +static apr_int64_t h2_srv_config_geti64(const h2_config *conf, h2_config_var_t var) { switch(var) { case H2_CONF_MAX_STREAMS: @@ -204,12 +282,93 @@ return H2_CONFIG_GET(conf, &defconf, copy_files); case H2_CONF_EARLY_HINTS: return H2_CONFIG_GET(conf, &defconf, early_hints); + case H2_CONF_PADDING_BITS: + return H2_CONFIG_GET(conf, &defconf, padding_bits); + case H2_CONF_PADDING_ALWAYS: + return H2_CONFIG_GET(conf, &defconf, padding_always); default: return DEF_VAL; } } -const h2_config *h2_config_sget(server_rec *s) +static void h2_srv_config_seti(h2_config *conf, h2_config_var_t var, int val) +{ + switch(var) { + case H2_CONF_MAX_STREAMS: + H2_CONFIG_SET(conf, h2_max_streams, val); + break; + case H2_CONF_WIN_SIZE: + H2_CONFIG_SET(conf, h2_window_size, val); + break; + case H2_CONF_MIN_WORKERS: + H2_CONFIG_SET(conf, min_workers, val); + break; + case H2_CONF_MAX_WORKERS: + H2_CONFIG_SET(conf, max_workers, val); + break; + case H2_CONF_MAX_WORKER_IDLE_SECS: + H2_CONFIG_SET(conf, max_worker_idle_secs, val); + break; + case H2_CONF_STREAM_MAX_MEM: + H2_CONFIG_SET(conf, stream_max_mem_size, val); + break; + case H2_CONF_ALT_SVC_MAX_AGE: + H2_CONFIG_SET(conf, alt_svc_max_age, val); + break; + case H2_CONF_SER_HEADERS: + H2_CONFIG_SET(conf, serialize_headers, val); + break; + case H2_CONF_MODERN_TLS_ONLY: + H2_CONFIG_SET(conf, modern_tls_only, val); + break; + case H2_CONF_UPGRADE: + H2_CONFIG_SET(conf, h2_upgrade, val); + break; + case H2_CONF_DIRECT: + H2_CONFIG_SET(conf, h2_direct, val); + break; + case H2_CONF_TLS_WARMUP_SIZE: + H2_CONFIG_SET(conf, tls_warmup_size, val); + break; + case H2_CONF_TLS_COOLDOWN_SECS: + H2_CONFIG_SET(conf, tls_cooldown_secs, val); + break; + case H2_CONF_PUSH: + H2_CONFIG_SET(conf, h2_push, val); + break; + case H2_CONF_PUSH_DIARY_SIZE: + H2_CONFIG_SET(conf, push_diary_size, val); + break; + case H2_CONF_COPY_FILES: + H2_CONFIG_SET(conf, copy_files, val); + break; + case H2_CONF_EARLY_HINTS: + H2_CONFIG_SET(conf, early_hints, val); + break; + case H2_CONF_PADDING_BITS: + H2_CONFIG_SET(conf, padding_bits, val); + break; + case H2_CONF_PADDING_ALWAYS: + H2_CONFIG_SET(conf, padding_always, val); + break; + default: + break; + } +} + +static void h2_srv_config_seti64(h2_config *conf, h2_config_var_t var, apr_int64_t val) +{ + switch(var) { + case H2_CONF_TLS_WARMUP_SIZE: + H2_CONFIG_SET(conf, tls_warmup_size, val); + break; + default: + h2_srv_config_seti(conf, var, (int)val); + break; + } +} + +static h2_config *h2_config_sget(server_rec *s) { h2_config *cfg = (h2_config *)ap_get_module_config(s->module_config, &http2_module); @@ -217,9 +376,162 @@ return cfg; } -const struct h2_priority *h2_config_get_priority(const h2_config *conf, - const char *content_type) +static const h2_dir_config *h2_config_rget(request_rec *r) +{ + h2_dir_config *cfg = (h2_dir_config *)ap_get_module_config(r->per_dir_config, + &http2_module); + ap_assert(cfg); + return cfg; +} + +static apr_int64_t h2_dir_config_geti64(const h2_dir_config *conf, h2_config_var_t var) +{ + switch(var) { + case H2_CONF_ALT_SVC_MAX_AGE: + return H2_CONFIG_GET(conf, &defdconf, alt_svc_max_age); + case H2_CONF_UPGRADE: + return H2_CONFIG_GET(conf, &defdconf, h2_upgrade); + case H2_CONF_PUSH: + return H2_CONFIG_GET(conf, &defdconf, h2_push); + case H2_CONF_EARLY_HINTS: + return H2_CONFIG_GET(conf, &defdconf, early_hints); + + default: + return DEF_VAL; + } +} + +static void h2_config_seti(h2_dir_config *dconf, h2_config *conf, h2_config_var_t var, int val) +{ + int set_srv = !dconf; + if (dconf) { + switch(var) { + case H2_CONF_ALT_SVC_MAX_AGE: + H2_CONFIG_SET(dconf, alt_svc_max_age, val); + break; + case H2_CONF_UPGRADE: + H2_CONFIG_SET(dconf, h2_upgrade, val); + break; + case H2_CONF_PUSH: + H2_CONFIG_SET(dconf, h2_push, val); + break; + case H2_CONF_EARLY_HINTS: + H2_CONFIG_SET(dconf, early_hints, val); + break; + default: + /* not handled in dir_conf */ + set_srv = 1; + break; + } + } + + if (set_srv) { + h2_srv_config_seti(conf, var, val); + } +} + +static void h2_config_seti64(h2_dir_config *dconf, h2_config *conf, h2_config_var_t var, apr_int64_t val) { + int set_srv = !dconf; + if (dconf) { + switch(var) { + default: + /* not handled in dir_conf */ + set_srv = 1; + break; + } + } + + if (set_srv) { + h2_srv_config_seti64(conf, var, val); + } +} + +static const h2_config *h2_config_get(conn_rec *c) +{ + h2_ctx *ctx = h2_ctx_get(c, 0); + + if (ctx) { + if (ctx->config) { + return ctx->config; + } + else if (ctx->server) { + ctx->config = h2_config_sget(ctx->server); + return ctx->config; + } + } + + return h2_config_sget(c->base_server); +} + +int h2_config_cgeti(conn_rec *c, h2_config_var_t var) +{ + return (int)h2_srv_config_geti64(h2_config_get(c), var); +} + +apr_int64_t h2_config_cgeti64(conn_rec *c, h2_config_var_t var) +{ + return h2_srv_config_geti64(h2_config_get(c), var); +} + +int h2_config_sgeti(server_rec *s, h2_config_var_t var) +{ + return (int)h2_srv_config_geti64(h2_config_sget(s), var); +} + +apr_int64_t h2_config_sgeti64(server_rec *s, h2_config_var_t var) +{ + return h2_srv_config_geti64(h2_config_sget(s), var); +} + +int h2_config_geti(request_rec *r, server_rec *s, h2_config_var_t var) +{ + return (int)h2_config_geti64(r, s, var); +} + +apr_int64_t h2_config_geti64(request_rec *r, server_rec *s, h2_config_var_t var) +{ + apr_int64_t mode = r? (int)h2_dir_config_geti64(h2_config_rget(r), var) : DEF_VAL; + return (mode != DEF_VAL)? mode : h2_config_sgeti64(s, var); +} + +int h2_config_rgeti(request_rec *r, h2_config_var_t var) +{ + return h2_config_geti(r, r->server, var); +} + +apr_int64_t h2_config_rgeti64(request_rec *r, h2_config_var_t var) +{ + return h2_config_geti64(r, r->server, var); +} + +apr_array_header_t *h2_config_push_list(request_rec *r) +{ + const h2_config *sconf; + const h2_dir_config *conf = h2_config_rget(r); + + if (conf && conf->push_list) { + return conf->push_list; + } + sconf = h2_config_sget(r->server); + return sconf? sconf->push_list : NULL; +} + +apr_array_header_t *h2_config_alt_svcs(request_rec *r) +{ + const h2_config *sconf; + const h2_dir_config *conf = h2_config_rget(r); + + if (conf && conf->alt_svcs) { + return conf->alt_svcs; + } + sconf = h2_config_sget(r->server); + return sconf? sconf->alt_svcs : NULL; +} + +const struct h2_priority *h2_cconfig_get_priority(conn_rec *c, const char *content_type) +{ + const h2_config *conf = h2_config_get(c); if (content_type && conf->priorities) { size_t len = strcspn(content_type, "; \t"); h2_priority *prio = apr_hash_get(conf->priorities, content_type, len); @@ -228,166 +540,156 @@ return NULL; } -static const char *h2_conf_set_max_streams(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_max_streams(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); - cfg->h2_max_streams = (int)apr_atoi64(value); - (void)arg; - if (cfg->h2_max_streams < 1) { + apr_int64_t ival = (int)apr_atoi64(value); + if (ival < 1) { return "value must be > 0"; } + CONFIG_CMD_SET64(cmd, dirconf, H2_CONF_MAX_STREAMS, ival); return NULL; } -static const char *h2_conf_set_window_size(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_window_size(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); - cfg->h2_window_size = (int)apr_atoi64(value); - (void)arg; - if (cfg->h2_window_size < 1024) { + int val = (int)apr_atoi64(value); + if (val < 1024) { return "value must be >= 1024"; } + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_WIN_SIZE, val); return NULL; } -static const char *h2_conf_set_min_workers(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_min_workers(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); - cfg->min_workers = (int)apr_atoi64(value); - (void)arg; - if (cfg->min_workers < 1) { + int val = (int)apr_atoi64(value); + if (val < 1) { return "value must be > 0"; } + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_MIN_WORKERS, val); return NULL; } -static const char *h2_conf_set_max_workers(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_max_workers(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); - cfg->max_workers = (int)apr_atoi64(value); - (void)arg; - if (cfg->max_workers < 1) { + int val = (int)apr_atoi64(value); + if (val < 1) { return "value must be > 0"; } + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_MAX_WORKERS, val); return NULL; } -static const char *h2_conf_set_max_worker_idle_secs(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_max_worker_idle_secs(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); - cfg->max_worker_idle_secs = (int)apr_atoi64(value); - (void)arg; - if (cfg->max_worker_idle_secs < 1) { + int val = (int)apr_atoi64(value); + if (val < 1) { return "value must be > 0"; } + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_MAX_WORKER_IDLE_SECS, val); return NULL; } -static const char *h2_conf_set_stream_max_mem_size(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_stream_max_mem_size(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); - - - cfg->stream_max_mem_size = (int)apr_atoi64(value); - (void)arg; - if (cfg->stream_max_mem_size < 1024) { + int val = (int)apr_atoi64(value); + if (val < 1024) { return "value must be >= 1024"; } + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_STREAM_MAX_MEM, val); return NULL; } -static const char *h2_add_alt_svc(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_add_alt_svc(cmd_parms *cmd, + void *dirconf, const char *value) { if (value && *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); - h2_alt_svc *as = h2_alt_svc_parse(value, parms->pool); + h2_alt_svc *as = h2_alt_svc_parse(value, cmd->pool); if (!as) { return "unable to parse alt-svc specifier"; } - if (!cfg->alt_svcs) { - cfg->alt_svcs = apr_array_make(parms->pool, 5, sizeof(h2_alt_svc*)); + + if (cmd->path) { + h2_dir_config *dcfg = (h2_dir_config *)dirconf; + if (!dcfg->alt_svcs) { + dcfg->alt_svcs = apr_array_make(cmd->pool, 5, sizeof(h2_alt_svc*)); + } + APR_ARRAY_PUSH(dcfg->alt_svcs, h2_alt_svc*) = as; + } + else { + h2_config *cfg = (h2_config *)h2_config_sget(cmd->server); + if (!cfg->alt_svcs) { + cfg->alt_svcs = apr_array_make(cmd->pool, 5, sizeof(h2_alt_svc*)); + } + APR_ARRAY_PUSH(cfg->alt_svcs, h2_alt_svc*) = as; } - APR_ARRAY_PUSH(cfg->alt_svcs, h2_alt_svc*) = as; } - (void)arg; return NULL; } -static const char *h2_conf_set_alt_svc_max_age(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_alt_svc_max_age(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); - cfg->alt_svc_max_age = (int)apr_atoi64(value); - (void)arg; + int val = (int)apr_atoi64(value); + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_ALT_SVC_MAX_AGE, val); return NULL; } -static const char *h2_conf_set_session_extra_files(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_session_extra_files(cmd_parms *cmd, + void *dirconf, const char *value) { /* deprecated, ignore */ - (void)arg; + (void)dirconf; (void)value; - ap_log_perror(APLOG_MARK, APLOG_WARNING, 0, parms->pool, /* NO LOGNO */ + ap_log_perror(APLOG_MARK, APLOG_WARNING, 0, cmd->pool, /* NO LOGNO */ "H2SessionExtraFiles is obsolete and will be ignored"); return NULL; } -static const char *h2_conf_set_serialize_headers(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_serialize_headers(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); if (!strcasecmp(value, "On")) { - cfg->serialize_headers = 1; + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_SER_HEADERS, 1); return NULL; } else if (!strcasecmp(value, "Off")) { - cfg->serialize_headers = 0; + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_SER_HEADERS, 0); return NULL; } - - (void)arg; return "value must be On or Off"; } -static const char *h2_conf_set_direct(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_direct(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); if (!strcasecmp(value, "On")) { - cfg->h2_direct = 1; + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_DIRECT, 1); return NULL; } else if (!strcasecmp(value, "Off")) { - cfg->h2_direct = 0; + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_DIRECT, 0); return NULL; } - - (void)arg; return "value must be On or Off"; } -static const char *h2_conf_set_push(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_push(cmd_parms *cmd, void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); if (!strcasecmp(value, "On")) { - cfg->h2_push = 1; + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_PUSH, 1); return NULL; } else if (!strcasecmp(value, "Off")) { - cfg->h2_push = 0; + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_PUSH, 0); return NULL; } - - (void)arg; return "value must be On or Off"; } @@ -419,7 +721,7 @@ else if (!strcasecmp("BEFORE", sdependency)) { dependency = H2_DEPENDANT_BEFORE; if (sweight) { - return "dependency 'Before' does not allow a weight"; + return "dependecy 'Before' does not allow a weight"; } } else if (!strcasecmp("INTERLEAVED", sdependency)) { @@ -447,100 +749,88 @@ return NULL; } -static const char *h2_conf_set_modern_tls_only(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_modern_tls_only(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); if (!strcasecmp(value, "On")) { - cfg->modern_tls_only = 1; + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_MODERN_TLS_ONLY, 1); return NULL; } else if (!strcasecmp(value, "Off")) { - cfg->modern_tls_only = 0; + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_MODERN_TLS_ONLY, 0); return NULL; } - - (void)arg; return "value must be On or Off"; } -static const char *h2_conf_set_upgrade(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_upgrade(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); if (!strcasecmp(value, "On")) { - cfg->h2_upgrade = 1; + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_UPGRADE, 1); return NULL; } else if (!strcasecmp(value, "Off")) { - cfg->h2_upgrade = 0; + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_UPGRADE, 0); return NULL; } - - (void)arg; return "value must be On or Off"; } -static const char *h2_conf_set_tls_warmup_size(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_tls_warmup_size(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); - cfg->tls_warmup_size = apr_atoi64(value); - (void)arg; + apr_int64_t val = apr_atoi64(value); + CONFIG_CMD_SET64(cmd, dirconf, H2_CONF_TLS_WARMUP_SIZE, val); return NULL; } -static const char *h2_conf_set_tls_cooldown_secs(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_tls_cooldown_secs(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); - cfg->tls_cooldown_secs = (int)apr_atoi64(value); - (void)arg; + apr_int64_t val = (int)apr_atoi64(value); + CONFIG_CMD_SET64(cmd, dirconf, H2_CONF_TLS_COOLDOWN_SECS, val); return NULL; } -static const char *h2_conf_set_push_diary_size(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_push_diary_size(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); - (void)arg; - cfg->push_diary_size = (int)apr_atoi64(value); - if (cfg->push_diary_size < 0) { + int val = (int)apr_atoi64(value); + if (val < 0) { return "value must be >= 0"; } - if (cfg->push_diary_size > 0 && (cfg->push_diary_size & (cfg->push_diary_size-1))) { + if (val > 0 && (val & (val-1))) { return "value must a power of 2"; } - if (cfg->push_diary_size > (1 << 15)) { + if (val > (1 << 15)) { return "value must <= 65536"; } + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_PUSH_DIARY_SIZE, val); return NULL; } -static const char *h2_conf_set_copy_files(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_copy_files(cmd_parms *cmd, + void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)arg; if (!strcasecmp(value, "On")) { - cfg->copy_files = 1; + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_COPY_FILES, 1); return NULL; } else if (!strcasecmp(value, "Off")) { - cfg->copy_files = 0; + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_COPY_FILES, 0); return NULL; } - - (void)arg; return "value must be On or Off"; } -static void add_push(apr_pool_t *pool, h2_config *conf, h2_push_res *push) +static void add_push(apr_array_header_t **plist, apr_pool_t *pool, h2_push_res *push) { h2_push_res *new; - if (!conf->push_list) { - conf->push_list = apr_array_make(pool, 10, sizeof(*push)); + if (!*plist) { + *plist = apr_array_make(pool, 10, sizeof(*push)); } - new = apr_array_push(conf->push_list); + new = apr_array_push(*plist); new->uri_ref = push->uri_ref; new->critical = push->critical; } @@ -549,8 +839,6 @@ const char *arg1, const char *arg2, const char *arg3) { - h2_config *dconf = (h2_config*)dirconf ; - h2_config *sconf = (h2_config*)h2_config_sget(cmd->server); h2_push_res push; const char *last = arg3; @@ -575,42 +863,54 @@ } } - /* server command? set both */ - if (cmd->path == NULL) { - add_push(cmd->pool, sconf, &push); - add_push(cmd->pool, dconf, &push); + if (cmd->path) { + add_push(&(((h2_dir_config*)dirconf)->push_list), cmd->pool, &push); } else { - add_push(cmd->pool, dconf, &push); + add_push(&(h2_config_sget(cmd->server)->push_list), cmd->pool, &push); } + return NULL; +} +static const char *h2_conf_set_early_hints(cmd_parms *cmd, + void *dirconf, const char *value) +{ + int val; + + if (!strcasecmp(value, "On")) val = 1; + else if (!strcasecmp(value, "Off")) val = 0; + else return "value must be On or Off"; + + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_EARLY_HINTS, val); + if (cmd->path) { + ap_log_perror(APLOG_MARK, APLOG_WARNING, 0, cmd->pool, + "H2EarlyHints = %d on path %s", val, cmd->path); + } return NULL; } -static const char *h2_conf_set_early_hints(cmd_parms *parms, - void *arg, const char *value) +static const char *h2_conf_set_padding(cmd_parms *cmd, void *dirconf, const char *value) { - h2_config *cfg = (h2_config *)h2_config_sget(parms->server); - if (!strcasecmp(value, "On")) { - cfg->early_hints = 1; - return NULL; + int val; + + val = (int)apr_atoi64(value); + if (val < 0) { + return "number of bits must be >= 0"; } - else if (!strcasecmp(value, "Off")) { - cfg->early_hints = 0; - return NULL; + if (val > 8) { + return "number of bits must be <= 8"; } - - (void)arg; - return "value must be On or Off"; + CONFIG_CMD_SET(cmd, dirconf, H2_CONF_PADDING_BITS, val); + return NULL; } + void h2_get_num_workers(server_rec *s, int *minw, int *maxw) { int threads_per_child = 0; - const h2_config *config = h2_config_sget(s); - *minw = h2_config_geti(config, H2_CONF_MIN_WORKERS); - *maxw = h2_config_geti(config, H2_CONF_MAX_WORKERS); + *minw = h2_config_sgeti(s, H2_CONF_MIN_WORKERS); + *maxw = h2_config_sgeti(s, H2_CONF_MAX_WORKERS); ap_mpm_query(AP_MPMQ_MAX_THREADS, &threads_per_child); if (*minw <= 0) { @@ -652,7 +952,7 @@ AP_INIT_TAKE1("H2ModernTLSOnly", h2_conf_set_modern_tls_only, NULL, RSRC_CONF, "off to not impose RFC 7540 restrictions on TLS"), AP_INIT_TAKE1("H2Upgrade", h2_conf_set_upgrade, NULL, - RSRC_CONF, "on to allow HTTP/1 Upgrades to h2/h2c"), + RSRC_CONF|OR_AUTHCFG, "on to allow HTTP/1 Upgrades to h2/h2c"), AP_INIT_TAKE1("H2Direct", h2_conf_set_direct, NULL, RSRC_CONF, "on to enable direct HTTP/2 mode"), AP_INIT_TAKE1("H2SessionExtraFiles", h2_conf_set_session_extra_files, NULL, @@ -662,7 +962,7 @@ AP_INIT_TAKE1("H2TLSCoolDownSecs", h2_conf_set_tls_cooldown_secs, NULL, RSRC_CONF, "seconds of idle time on TLS before shrinking writes"), AP_INIT_TAKE1("H2Push", h2_conf_set_push, NULL, - RSRC_CONF, "off to disable HTTP/2 server push"), + RSRC_CONF|OR_AUTHCFG, "off to disable HTTP/2 server push"), AP_INIT_TAKE23("H2PushPriority", h2_conf_add_push_priority, NULL, RSRC_CONF, "define priority of PUSHed resources per content type"), AP_INIT_TAKE1("H2PushDiarySize", h2_conf_set_push_diary_size, NULL, @@ -670,33 +970,12 @@ AP_INIT_TAKE1("H2CopyFiles", h2_conf_set_copy_files, NULL, OR_FILEINFO, "on to perform copy of file data"), AP_INIT_TAKE123("H2PushResource", h2_conf_add_push_res, NULL, - OR_FILEINFO, "add a resource to be pushed in this location/on this server."), + OR_FILEINFO|OR_AUTHCFG, "add a resource to be pushed in this location/on this server."), AP_INIT_TAKE1("H2EarlyHints", h2_conf_set_early_hints, NULL, RSRC_CONF, "on to enable interim status 103 responses"), + AP_INIT_TAKE1("H2Padding", h2_conf_set_padding, NULL, + RSRC_CONF, "set payload padding"), AP_END_CMD }; -const h2_config *h2_config_rget(request_rec *r) -{ - h2_config *cfg = (h2_config *)ap_get_module_config(r->per_dir_config, - &http2_module); - return cfg? cfg : h2_config_sget(r->server); -} - -const h2_config *h2_config_get(conn_rec *c) -{ - h2_ctx *ctx = h2_ctx_get(c, 0); - - if (ctx) { - if (ctx->config) { - return ctx->config; - } - else if (ctx->server) { - ctx->config = h2_config_sget(ctx->server); - return ctx->config; - } - } - - return h2_config_sget(c->base_server); -} --- a/modules/http2/h2_config.h +++ b/modules/http2/h2_config.h @@ -42,6 +42,8 @@ H2_CONF_PUSH_DIARY_SIZE, H2_CONF_COPY_FILES, H2_CONF_EARLY_HINTS, + H2_CONF_PADDING_BITS, + H2_CONF_PADDING_ALWAYS, } h2_config_var_t; struct apr_hash_t; @@ -53,33 +55,6 @@ int critical; } h2_push_res; -/* Apache httpd module configuration for h2. */ -typedef struct h2_config { - const char *name; - int h2_max_streams; /* max concurrent # streams (http2) */ - int h2_window_size; /* stream window size (http2) */ - int min_workers; /* min # of worker threads/child */ - int max_workers; /* max # of worker threads/child */ - int max_worker_idle_secs; /* max # of idle seconds for worker */ - int stream_max_mem_size; /* max # bytes held in memory/stream */ - apr_array_header_t *alt_svcs; /* h2_alt_svc specs for this server */ - int alt_svc_max_age; /* seconds clients can rely on alt-svc info*/ - int serialize_headers; /* Use serialized HTTP/1.1 headers for - processing, better compatibility */ - int h2_direct; /* if mod_h2 is active directly */ - int modern_tls_only; /* Accept only modern TLS in HTTP/2 connections */ - int h2_upgrade; /* Allow HTTP/1 upgrade to h2/h2c */ - apr_int64_t tls_warmup_size; /* Amount of TLS data to send before going full write size */ - int tls_cooldown_secs; /* Seconds of idle time before going back to small TLS records */ - int h2_push; /* if HTTP/2 server push is enabled */ - struct apr_hash_t *priorities;/* map of content-type to h2_priority records */ - - int push_diary_size; /* # of entries in push diary */ - int copy_files; /* if files shall be copied vs setaside on output */ - apr_array_header_t *push_list;/* list of h2_push_res configurations */ - int early_hints; /* support status code 103 */ -} h2_config; - void *h2_config_create_dir(apr_pool_t *pool, char *x); void *h2_config_merge_dir(apr_pool_t *pool, void *basev, void *addv); @@ -88,19 +63,37 @@ extern const command_rec h2_cmds[]; -const h2_config *h2_config_get(conn_rec *c); -const h2_config *h2_config_sget(server_rec *s); -const h2_config *h2_config_rget(request_rec *r); +int h2_config_geti(request_rec *r, server_rec *s, h2_config_var_t var); +apr_int64_t h2_config_geti64(request_rec *r, server_rec *s, h2_config_var_t var); -int h2_config_geti(const h2_config *conf, h2_config_var_t var); -apr_int64_t h2_config_geti64(const h2_config *conf, h2_config_var_t var); +/** + * Get the configured value for variable at the given connection. + */ +int h2_config_cgeti(conn_rec *c, h2_config_var_t var); +apr_int64_t h2_config_cgeti64(conn_rec *c, h2_config_var_t var); + +/** + * Get the configured value for variable at the given server. + */ +int h2_config_sgeti(server_rec *s, h2_config_var_t var); +apr_int64_t h2_config_sgeti64(server_rec *s, h2_config_var_t var); + +/** + * Get the configured value for variable at the given request, + * if configured for the request location. + * Fallback to request server config otherwise. + */ +int h2_config_rgeti(request_rec *r, h2_config_var_t var); +apr_int64_t h2_config_rgeti64(request_rec *r, h2_config_var_t var); -void h2_get_num_workers(server_rec *s, int *minw, int *maxw); +apr_array_header_t *h2_config_push_list(request_rec *r); +apr_array_header_t *h2_config_alt_svcs(request_rec *r); + +void h2_get_num_workers(server_rec *s, int *minw, int *maxw); void h2_config_init(apr_pool_t *pool); -const struct h2_priority *h2_config_get_priority(const h2_config *conf, - const char *content_type); +const struct h2_priority *h2_cconfig_get_priority(conn_rec *c, const char *content_type); #endif /* __mod_h2__h2_config_h__ */ --- a/modules/http2/h2_conn.c +++ b/modules/http2/h2_conn.c @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -79,7 +80,7 @@ mpm_type = H2_MPM_PREFORK; mpm_module = m; /* While http2 can work really well on prefork, it collides - * today's use case for prefork: runnning single-thread app engines + * today's use case for prefork: running single-thread app engines * like php. If we restrict h2_workers to 1 per process, php will * work fine, but browser will be limited to 1 active request at a * time. */ @@ -109,7 +110,6 @@ apr_status_t h2_conn_child_init(apr_pool_t *pool, server_rec *s) { - const h2_config *config = h2_config_sget(s); apr_status_t status = APR_SUCCESS; int minw, maxw; int max_threads_per_child = 0; @@ -129,7 +129,7 @@ h2_get_num_workers(s, &minw, &maxw); - idle_secs = h2_config_geti(config, H2_CONF_MAX_WORKER_IDLE_SECS); + idle_secs = h2_config_sgeti(s, H2_CONF_MAX_WORKER_IDLE_SECS); ap_log_error(APLOG_MARK, APLOG_TRACE3, 0, s, "h2_workers: min=%d max=%d, mthrpchild=%d, idle_secs=%d", minw, maxw, max_threads_per_child, idle_secs); @@ -138,7 +138,7 @@ ap_register_input_filter("H2_IN", h2_filter_core_input, NULL, AP_FTYPE_CONNECTION); - status = h2_mplx_child_init(pool, s); + status = h2_mplx_m_child_init(pool, s); if (status == APR_SUCCESS) { status = apr_socket_create(&dummy_socket, APR_INET, SOCK_STREAM, @@ -172,9 +172,10 @@ return mpm_module; } -apr_status_t h2_conn_setup(h2_ctx *ctx, conn_rec *c, request_rec *r) +apr_status_t h2_conn_setup(conn_rec *c, request_rec *r, server_rec *s) { h2_session *session; + h2_ctx *ctx; apr_status_t status; if (!workers) { @@ -183,24 +184,25 @@ return APR_EGENERAL; } - if (r) { - status = h2_session_rcreate(&session, r, ctx, workers); - } - else { - status = h2_session_create(&session, c, ctx, workers); - } - - if (status == APR_SUCCESS) { + if (APR_SUCCESS == (status = h2_session_create(&session, c, r, s, workers))) { + ctx = h2_ctx_get(c, 1); h2_ctx_session_set(ctx, session); + + /* remove the input filter of mod_reqtimeout, now that the connection + * is established and we have swtiched to h2. reqtimeout has supervised + * possibly configured handshake timeouts and needs to get out of the way + * now since the rest of its state handling assumes http/1.x to take place. */ + ap_remove_input_filter_byhandle(c->input_filters, "reqtimeout"); } + return status; } -apr_status_t h2_conn_run(struct h2_ctx *ctx, conn_rec *c) +apr_status_t h2_conn_run(conn_rec *c) { apr_status_t status; int mpm_state = 0; - h2_session *session = h2_ctx_session_get(ctx); + h2_session *session = h2_ctx_get_session(c); ap_assert(session); do { @@ -235,6 +237,13 @@ case H2_SESSION_ST_BUSY: case H2_SESSION_ST_WAIT: c->cs->state = CONN_STATE_WRITE_COMPLETION; + if (c->cs && (session->open_streams || !session->remote.emitted_count)) { + /* let the MPM know that we are not done and want + * the Timeout behaviour instead of a KeepAliveTimeout + * See PR 63534. + */ + c->cs->sense = CONN_SENSE_WANT_READ; + } break; case H2_SESSION_ST_CLEANUP: case H2_SESSION_ST_DONE: @@ -249,7 +258,7 @@ apr_status_t h2_conn_pre_close(struct h2_ctx *ctx, conn_rec *c) { - h2_session *session = h2_ctx_session_get(ctx); + h2_session *session = h2_ctx_get_session(c); if (session) { apr_status_t status = h2_session_pre_close(session, async_mpm); return (status == APR_SUCCESS)? DONE : status; @@ -257,7 +266,7 @@ return DONE; } -conn_rec *h2_slave_create(conn_rec *master, int slave_id, apr_pool_t *parent) +conn_rec *h2_secondary_create(conn_rec *master, int sec_id, apr_pool_t *parent) { apr_allocator_t *allocator; apr_status_t status; @@ -268,11 +277,11 @@ ap_assert(master); ap_log_cerror(APLOG_MARK, APLOG_TRACE3, 0, master, - "h2_stream(%ld-%d): create slave", master->id, slave_id); + "h2_stream(%ld-%d): create secondary", master->id, sec_id); /* We create a pool with its own allocator to be used for * processing a request. This is the only way to have the processing - * independant of its parent pool in the sense that it can work in + * independent of its parent pool in the sense that it can work in * another thread. Also, the new allocator needs its own mutex to * synchronize sub-pools. */ @@ -281,18 +290,18 @@ status = apr_pool_create_ex(&pool, parent, NULL, allocator); if (status != APR_SUCCESS) { ap_log_cerror(APLOG_MARK, APLOG_ERR, status, master, - APLOGNO(10004) "h2_session(%ld-%d): create slave pool", - master->id, slave_id); + APLOGNO(10004) "h2_session(%ld-%d): create secondary pool", + master->id, sec_id); return NULL; } apr_allocator_owner_set(allocator, pool); - apr_pool_tag(pool, "h2_slave_conn"); + apr_pool_tag(pool, "h2_secondary_conn"); c = (conn_rec *) apr_palloc(pool, sizeof(conn_rec)); if (c == NULL) { ap_log_cerror(APLOG_MARK, APLOG_ERR, APR_ENOMEM, master, - APLOGNO(02913) "h2_session(%ld-%d): create slave", - master->id, slave_id); + APLOGNO(02913) "h2_session(%ld-%d): create secondary", + master->id, sec_id); apr_pool_destroy(pool); return NULL; } @@ -310,26 +319,28 @@ c->filter_conn_ctx = NULL; #endif c->bucket_alloc = apr_bucket_alloc_create(pool); +#if !AP_MODULE_MAGIC_AT_LEAST(20180720, 1) c->data_in_input_filters = 0; c->data_in_output_filters = 0; +#endif /* prevent mpm_event from making wrong assumptions about this connection, * like e.g. using its socket for an async read check. */ c->clogging_input_filters = 1; c->log = NULL; c->log_id = apr_psprintf(pool, "%ld-%d", - master->id, slave_id); + master->id, sec_id); c->aborted = 0; - /* We cannot install the master connection socket on the slaves, as + /* We cannot install the master connection socket on the secondary, as * modules mess with timeouts/blocking of the socket, with * unwanted side effects to the master connection processing. - * Fortunately, since we never use the slave socket, we can just install + * Fortunately, since we never use the secondary socket, we can just install * a single, process-wide dummy and everyone is happy. */ ap_set_module_config(c->conn_config, &core_module, dummy_socket); /* TODO: these should be unique to this thread */ c->sbh = master->sbh; - /* TODO: not all mpm modules have learned about slave connections yet. - * copy their config from master to slave. + /* TODO: not all mpm modules have learned about secondary connections yet. + * copy their config from master to secondary. */ if ((mpm = h2_conn_mpm_module()) != NULL) { cfg = ap_get_module_config(master->conn_config, mpm); @@ -337,38 +348,38 @@ } ap_log_cerror(APLOG_MARK, APLOG_TRACE3, 0, c, - "h2_slave(%s): created", c->log_id); + "h2_secondary(%s): created", c->log_id); return c; } -void h2_slave_destroy(conn_rec *slave) +void h2_secondary_destroy(conn_rec *secondary) { - ap_log_cerror(APLOG_MARK, APLOG_TRACE3, 0, slave, - "h2_slave(%s): destroy", slave->log_id); - slave->sbh = NULL; - apr_pool_destroy(slave->pool); + ap_log_cerror(APLOG_MARK, APLOG_TRACE3, 0, secondary, + "h2_secondary(%s): destroy", secondary->log_id); + secondary->sbh = NULL; + apr_pool_destroy(secondary->pool); } -apr_status_t h2_slave_run_pre_connection(conn_rec *slave, apr_socket_t *csd) +apr_status_t h2_secondary_run_pre_connection(conn_rec *secondary, apr_socket_t *csd) { - if (slave->keepalives == 0) { + if (secondary->keepalives == 0) { /* Simulate that we had already a request on this connection. Some * hooks trigger special behaviour when keepalives is 0. * (Not necessarily in pre_connection, but later. Set it here, so it * is in place.) */ - slave->keepalives = 1; + secondary->keepalives = 1; /* We signal that this connection will be closed after the request. * Which is true in that sense that we throw away all traffic data - * on this slave connection after each requests. Although we might + * on this secondary connection after each requests. Although we might * reuse internal structures like memory pools. * The wanted effect of this is that httpd does not try to clean up * any dangling data on this connection when a request is done. Which - * is unneccessary on a h2 stream. + * is unnecessary on a h2 stream. */ - slave->keepalive = AP_CONN_CLOSE; - return ap_run_pre_connection(slave, csd); + secondary->keepalive = AP_CONN_CLOSE; + return ap_run_pre_connection(secondary, csd); } - ap_assert(slave->output_filters); + ap_assert(secondary->output_filters); return APR_SUCCESS; } --- a/modules/http2/h2_conn.h +++ b/modules/http2/h2_conn.h @@ -23,21 +23,21 @@ /** * Setup the connection and our context for HTTP/2 processing * - * @param ctx the http2 context to setup * @param c the connection HTTP/2 is starting on * @param r the upgrade request that still awaits an answer, optional + * @param s the server selected for this connection (can be != c->base_server) */ -apr_status_t h2_conn_setup(struct h2_ctx *ctx, conn_rec *c, request_rec *r); +apr_status_t h2_conn_setup(conn_rec *c, request_rec *r, server_rec *s); /** * Run the HTTP/2 connection in synchronous fashion. * Return when the HTTP/2 session is done * and the connection will close or a fatal error occurred. * - * @param ctx the http2 context to run + * @param c the http2 connection to run * @return APR_SUCCESS when session is done. */ -apr_status_t h2_conn_run(struct h2_ctx *ctx, conn_rec *c); +apr_status_t h2_conn_run(conn_rec *c); /** * The connection is about to close. If we have not send a GOAWAY @@ -68,10 +68,10 @@ const char *h2_conn_mpm_name(void); int h2_mpm_supported(void); -conn_rec *h2_slave_create(conn_rec *master, int slave_id, apr_pool_t *parent); -void h2_slave_destroy(conn_rec *slave); +conn_rec *h2_secondary_create(conn_rec *master, int sec_id, apr_pool_t *parent); +void h2_secondary_destroy(conn_rec *secondary); -apr_status_t h2_slave_run_pre_connection(conn_rec *slave, apr_socket_t *csd); -void h2_slave_run_connection(conn_rec *slave); +apr_status_t h2_secondary_run_pre_connection(conn_rec *secondary, apr_socket_t *csd); +void h2_secondary_run_connection(conn_rec *secondary); #endif /* defined(__mod_h2__h2_conn__) */ --- a/modules/http2/h2_conn_io.c +++ b/modules/http2/h2_conn_io.c @@ -40,12 +40,17 @@ * ~= 1300 bytes */ #define WRITE_SIZE_INITIAL 1300 -/* Calculated like this: max TLS record size 16*1024 - * - 40 (IP) - 20 (TCP) - 40 (TCP options) - * - TLS overhead (60-100) - * which seems to create less TCP packets overall +/* The maximum we'd like to write in one chunk is + * the max size of a TLS record. When pushing + * many frames down the h2 connection, this might + * align differently because of headers and other + * frames or simply as not sufficient data is + * in a response body. + * However keeping frames at or below this limit + * should make optimizations at the layer that writes + * to TLS easier. */ -#define WRITE_SIZE_MAX (TLS_DATA_MAX - 100) +#define WRITE_SIZE_MAX (TLS_DATA_MAX) static void h2_conn_io_bb_log(conn_rec *c, int stream_id, int level, @@ -123,21 +128,20 @@ } -apr_status_t h2_conn_io_init(h2_conn_io *io, conn_rec *c, - const h2_config *cfg) +apr_status_t h2_conn_io_init(h2_conn_io *io, conn_rec *c, server_rec *s) { io->c = c; io->output = apr_brigade_create(c->pool, c->bucket_alloc); io->is_tls = h2_h2_is_tls(c); io->buffer_output = io->is_tls; - io->flush_threshold = (apr_size_t)h2_config_geti64(cfg, H2_CONF_STREAM_MAX_MEM); + io->flush_threshold = (apr_size_t)h2_config_sgeti64(s, H2_CONF_STREAM_MAX_MEM); if (io->is_tls) { /* This is what we start with, * see https://issues.apache.org/jira/browse/TS-2503 */ - io->warmup_size = h2_config_geti64(cfg, H2_CONF_TLS_WARMUP_SIZE); - io->cooldown_usecs = (h2_config_geti(cfg, H2_CONF_TLS_COOLDOWN_SECS) + io->warmup_size = h2_config_sgeti64(s, H2_CONF_TLS_WARMUP_SIZE); + io->cooldown_usecs = (h2_config_sgeti(s, H2_CONF_TLS_COOLDOWN_SECS) * APR_USEC_PER_SEC); io->write_size = (io->cooldown_usecs > 0? WRITE_SIZE_INITIAL : WRITE_SIZE_MAX); --- a/modules/http2/h2_conn_io.h +++ b/modules/http2/h2_conn_io.h @@ -48,8 +48,7 @@ apr_size_t slen; } h2_conn_io; -apr_status_t h2_conn_io_init(h2_conn_io *io, conn_rec *c, - const struct h2_config *cfg); +apr_status_t h2_conn_io_init(h2_conn_io *io, conn_rec *c, server_rec *s); /** * Append data to the buffered output. --- a/modules/http2/h2_ctx.c +++ b/modules/http2/h2_ctx.c @@ -29,8 +29,8 @@ { h2_ctx *ctx = apr_pcalloc(c->pool, sizeof(h2_ctx)); ap_assert(ctx); + h2_ctx_server_update(ctx, c->base_server); ap_set_module_config(c->conn_config, &http2_module, ctx); - h2_ctx_server_set(ctx, c->base_server); return ctx; } @@ -79,8 +79,9 @@ return ctx; } -h2_session *h2_ctx_session_get(h2_ctx *ctx) +h2_session *h2_ctx_get_session(conn_rec *c) { + h2_ctx *ctx = h2_ctx_get(c, 0); return ctx? ctx->session : NULL; } @@ -89,33 +90,17 @@ ctx->session = session; } -server_rec *h2_ctx_server_get(h2_ctx *ctx) +h2_ctx *h2_ctx_server_update(h2_ctx *ctx, server_rec *s) { - return ctx? ctx->server : NULL; -} - -h2_ctx *h2_ctx_server_set(h2_ctx *ctx, server_rec *s) -{ - ctx->server = s; + if (ctx->server != s) { + ctx->server = s; + } return ctx; } -int h2_ctx_is_task(h2_ctx *ctx) -{ - return ctx && ctx->task; -} - -h2_task *h2_ctx_get_task(h2_ctx *ctx) +h2_task *h2_ctx_get_task(conn_rec *c) { + h2_ctx *ctx = h2_ctx_get(c, 0); return ctx? ctx->task : NULL; } -h2_task *h2_ctx_cget_task(conn_rec *c) -{ - return h2_ctx_get_task(h2_ctx_get(c, 0)); -} - -h2_task *h2_ctx_rget_task(request_rec *r) -{ - return h2_ctx_get_task(h2_ctx_rget(r)); -} --- a/modules/http2/h2_ctx.h +++ b/modules/http2/h2_ctx.h @@ -56,12 +56,11 @@ */ h2_ctx *h2_ctx_protocol_set(h2_ctx *ctx, const char *proto); -/* Set the server_rec relevant for this context. +/* Update the server_rec relevant for this context. A server for + * a connection may change during SNI handling, for example. */ -h2_ctx *h2_ctx_server_set(h2_ctx *ctx, server_rec *s); -server_rec *h2_ctx_server_get(h2_ctx *ctx); +h2_ctx *h2_ctx_server_update(h2_ctx *ctx, server_rec *s); -struct h2_session *h2_ctx_session_get(h2_ctx *ctx); void h2_ctx_session_set(h2_ctx *ctx, struct h2_session *session); /** @@ -69,10 +68,8 @@ */ const char *h2_ctx_protocol_get(const conn_rec *c); -int h2_ctx_is_task(h2_ctx *ctx); +struct h2_session *h2_ctx_get_session(conn_rec *c); +struct h2_task *h2_ctx_get_task(conn_rec *c); -struct h2_task *h2_ctx_get_task(h2_ctx *ctx); -struct h2_task *h2_ctx_cget_task(conn_rec *c); -struct h2_task *h2_ctx_rget_task(request_rec *r); #endif /* defined(__mod_h2__h2_ctx__) */ --- a/modules/http2/h2_filter.c +++ b/modules/http2/h2_filter.c @@ -54,6 +54,7 @@ const char *data; ssize_t n; + (void)c; status = apr_bucket_read(b, &data, &len, block); while (status == APR_SUCCESS && len > 0) { @@ -71,10 +72,10 @@ } else { session->io.bytes_read += n; - if (len <= n) { + if ((apr_ssize_t)len <= n) { break; } - len -= n; + len -= (apr_size_t)n; data += n; } } @@ -277,6 +278,7 @@ apr_bucket_brigade *dest, const apr_bucket *src) { + (void)beam; if (H2_BUCKET_IS_OBSERVER(src)) { h2_bucket_observer *l = (h2_bucket_observer *)src->data; apr_bucket *b = h2_bucket_observer_create(dest->bucket_alloc, @@ -311,8 +313,7 @@ bbout(bb, " \"settings\": {\n"); bbout(bb, " \"SETTINGS_MAX_CONCURRENT_STREAMS\": %d,\n", m->max_streams); bbout(bb, " \"SETTINGS_MAX_FRAME_SIZE\": %d,\n", 16*1024); - bbout(bb, " \"SETTINGS_INITIAL_WINDOW_SIZE\": %d,\n", - h2_config_geti(s->config, H2_CONF_WIN_SIZE)); + bbout(bb, " \"SETTINGS_INITIAL_WINDOW_SIZE\": %d,\n", h2_config_sgeti(s->s, H2_CONF_WIN_SIZE)); bbout(bb, " \"SETTINGS_ENABLE_PUSH\": %d\n", h2_session_push_enabled(s)); bbout(bb, " }%s\n", last? "" : ","); } @@ -369,7 +370,7 @@ x.s = s; x.idx = 0; bbout(bb, " \"streams\": {"); - h2_mplx_stream_do(s->mplx, add_stream, &x); + h2_mplx_m_stream_do(s->mplx, add_stream, &x); bbout(bb, "\n }%s\n", last? "" : ","); } @@ -431,41 +432,38 @@ static apr_status_t h2_status_insert(h2_task *task, apr_bucket *b) { - conn_rec *c = task->c->master; - h2_ctx *h2ctx = h2_ctx_get(c, 0); - h2_session *session; - h2_stream *stream; + h2_mplx *m = task->mplx; + h2_stream *stream = h2_mplx_t_stream_get(m, task); + h2_session *s; + conn_rec *c; + apr_bucket_brigade *bb; apr_bucket *e; int32_t connFlowIn, connFlowOut; - - if (!h2ctx || (session = h2_ctx_session_get(h2ctx)) == NULL) { - return APR_SUCCESS; - } - - stream = h2_session_stream_get(session, task->stream_id); if (!stream) { /* stream already done */ return APR_SUCCESS; } + s = stream->session; + c = s->c; bb = apr_brigade_create(stream->pool, c->bucket_alloc); - connFlowIn = nghttp2_session_get_effective_local_window_size(session->ngh2); - connFlowOut = nghttp2_session_get_remote_window_size(session->ngh2); + connFlowIn = nghttp2_session_get_effective_local_window_size(s->ngh2); + connFlowOut = nghttp2_session_get_remote_window_size(s->ngh2); bbout(bb, "{\n"); bbout(bb, " \"version\": \"draft-01\",\n"); - add_settings(bb, session, 0); - add_peer_settings(bb, session, 0); + add_settings(bb, s, 0); + add_peer_settings(bb, s, 0); bbout(bb, " \"connFlowIn\": %d,\n", connFlowIn); bbout(bb, " \"connFlowOut\": %d,\n", connFlowOut); - bbout(bb, " \"sentGoAway\": %d,\n", session->local.shutdown); + bbout(bb, " \"sentGoAway\": %d,\n", s->local.shutdown); - add_streams(bb, session, 0); + add_streams(bb, s, 0); - add_stats(bb, session, stream, 1); + add_stats(bb, s, stream, 1); bbout(bb, "}\n"); while ((e = APR_BRIGADE_FIRST(bb)) != APR_BRIGADE_SENTINEL(bb)) { @@ -495,9 +493,54 @@ return APR_SUCCESS; } +static apr_status_t discard_body(request_rec *r, apr_off_t maxlen) +{ + apr_bucket_brigade *bb; + int seen_eos; + apr_status_t rv; + + bb = apr_brigade_create(r->pool, r->connection->bucket_alloc); + seen_eos = 0; + do { + apr_bucket *bucket; + + rv = ap_get_brigade(r->input_filters, bb, AP_MODE_READBYTES, + APR_BLOCK_READ, HUGE_STRING_LEN); + + if (rv != APR_SUCCESS) { + apr_brigade_destroy(bb); + return rv; + } + + for (bucket = APR_BRIGADE_FIRST(bb); + bucket != APR_BRIGADE_SENTINEL(bb); + bucket = APR_BUCKET_NEXT(bucket)) + { + const char *data; + apr_size_t len; + + if (APR_BUCKET_IS_EOS(bucket)) { + seen_eos = 1; + break; + } + if (bucket->length == 0) { + continue; + } + rv = apr_bucket_read(bucket, &data, &len, APR_BLOCK_READ); + if (rv != APR_SUCCESS) { + apr_brigade_destroy(bb); + return rv; + } + maxlen -= bucket->length; + } + apr_brigade_cleanup(bb); + } while (!seen_eos && maxlen >= 0); + + return APR_SUCCESS; +} + int h2_filter_h2_status_handler(request_rec *r) { - h2_ctx *ctx = h2_ctx_rget(r); conn_rec *c = r->connection; h2_task *task; apr_bucket_brigade *bb; @@ -511,10 +554,12 @@ return DECLINED; } - task = ctx? h2_ctx_get_task(ctx) : NULL; + task = h2_ctx_get_task(r->connection); if (task) { - - if ((status = ap_discard_request_body(r)) != OK) { + /* In this handler, we do some special sauce to send footers back, + * IFF we received footers in the request. This is used in our test + * cases, since CGI has no way of handling those. */ + if ((status = discard_body(r, 1024)) != OK) { return status; } --- a/modules/http2/h2_from_h1.c +++ b/modules/http2/h2_from_h1.c @@ -315,6 +315,7 @@ int http_status; apr_array_header_t *hlines; apr_bucket_brigade *tmp; + apr_bucket_brigade *saveto; } h2_response_parser; static apr_status_t parse_header(h2_response_parser *parser, char *line) { @@ -351,13 +352,17 @@ parser->tmp = apr_brigade_create(task->pool, task->c->bucket_alloc); } status = apr_brigade_split_line(parser->tmp, bb, APR_BLOCK_READ, - HUGE_STRING_LEN); + len); if (status == APR_SUCCESS) { --len; status = apr_brigade_flatten(parser->tmp, line, &len); if (status == APR_SUCCESS) { /* we assume a non-0 containing line and remove trailing crlf. */ line[len] = '\0'; + /* + * XXX: What to do if there is an LF but no CRLF? + * Should we error out? + */ if (len >= 2 && !strcmp(H2_CRLF, line + len - 2)) { len -= 2; line[len] = '\0'; @@ -367,10 +372,47 @@ task->id, line); } else { + apr_off_t brigade_length; + + /* + * If the brigade parser->tmp becomes longer than our buffer + * for flattening we never have a chance to get a complete + * line. This can happen if we are called multiple times after + * previous calls did not find a H2_CRLF and we returned + * APR_EAGAIN. In this case parser->tmp (correctly) grows + * with each call to apr_brigade_split_line. + * + * XXX: Currently a stack based buffer of HUGE_STRING_LEN is + * used. This means we cannot cope with lines larger than + * HUGE_STRING_LEN which might be an issue. + */ + status = apr_brigade_length(parser->tmp, 0, &brigade_length); + if ((status != APR_SUCCESS) || (brigade_length > len)) { + ap_log_cerror(APLOG_MARK, APLOG_ERR, 0, task->c, APLOGNO(10257) + "h2_task(%s): read response, line too long", + task->id); + return APR_ENOSPC; + } /* this does not look like a complete line yet */ ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, task->c, "h2_task(%s): read response, incomplete line: %s", task->id, line); + if (!parser->saveto) { + parser->saveto = apr_brigade_create(task->pool, + task->c->bucket_alloc); + } + /* + * Be on the save side and save the parser->tmp brigade + * as it could contain transient buckets which could be + * invalid next time we are here. + * + * NULL for the filter parameter is ok since we + * provide our own brigade as second parameter + * and ap_save_brigade does not need to create one. + */ + ap_save_brigade(NULL, &(parser->saveto), &(parser->tmp), + parser->tmp->p); + APR_BRIGADE_CONCAT(parser->tmp, parser->saveto); return APR_EAGAIN; } } @@ -594,18 +636,20 @@ } } - if (r->header_only) { + if (r->header_only || AP_STATUS_IS_HEADER_ONLY(r->status)) { ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, f->c, - "h2_task(%s): header_only, cleanup output brigade", + "h2_task(%s): headers only, cleanup output brigade", task->id); b = body_bucket? body_bucket : APR_BRIGADE_FIRST(bb); while (b != APR_BRIGADE_SENTINEL(bb)) { next = APR_BUCKET_NEXT(b); if (APR_BUCKET_IS_EOS(b) || AP_BUCKET_IS_EOR(b)) { break; - } - APR_BUCKET_REMOVE(b); - apr_bucket_destroy(b); + } + if (!H2_BUCKET_IS_HEADERS(b)) { + APR_BUCKET_REMOVE(b); + apr_bucket_destroy(b); + } b = next; } } --- a/modules/http2/h2_h2.c +++ b/modules/http2/h2_h2.c @@ -463,19 +463,18 @@ return opt_ssl_is_https && opt_ssl_is_https(c); } -int h2_is_acceptable_connection(conn_rec *c, int require_all) +int h2_is_acceptable_connection(conn_rec *c, request_rec *r, int require_all) { int is_tls = h2_h2_is_tls(c); - const h2_config *cfg = h2_config_get(c); - if (is_tls && h2_config_geti(cfg, H2_CONF_MODERN_TLS_ONLY) > 0) { + if (is_tls && h2_config_cgeti(c, H2_CONF_MODERN_TLS_ONLY) > 0) { /* Check TLS connection for modern TLS parameters, as defined in * RFC 7540 and https://wiki.mozilla.org/Security/Server_Side_TLS#Modern_compatibility */ apr_pool_t *pool = c->pool; server_rec *s = c->base_server; char *val; - + if (!opt_ssl_var_lookup) { /* unable to check */ return 0; @@ -521,33 +520,29 @@ return 1; } -int h2_allows_h2_direct(conn_rec *c) +static int h2_allows_h2_direct(conn_rec *c) { - const h2_config *cfg = h2_config_get(c); int is_tls = h2_h2_is_tls(c); const char *needed_protocol = is_tls? "h2" : "h2c"; - int h2_direct = h2_config_geti(cfg, H2_CONF_DIRECT); + int h2_direct = h2_config_cgeti(c, H2_CONF_DIRECT); if (h2_direct < 0) { h2_direct = is_tls? 0 : 1; } - return (h2_direct - && ap_is_allowed_protocol(c, NULL, NULL, needed_protocol)); + return (h2_direct && ap_is_allowed_protocol(c, NULL, NULL, needed_protocol)); } -int h2_allows_h2_upgrade(conn_rec *c) +int h2_allows_h2_upgrade(request_rec *r) { - const h2_config *cfg = h2_config_get(c); - int h2_upgrade = h2_config_geti(cfg, H2_CONF_UPGRADE); - - return h2_upgrade > 0 || (h2_upgrade < 0 && !h2_h2_is_tls(c)); + int h2_upgrade = h2_config_rgeti(r, H2_CONF_UPGRADE); + return h2_upgrade > 0 || (h2_upgrade < 0 && !h2_h2_is_tls(r->connection)); } /******************************************************************************* * Register various hooks */ static const char* const mod_ssl[] = { "mod_ssl.c", NULL}; -static const char* const mod_reqtimeout[] = { "mod_reqtimeout.c", NULL}; +static const char* const mod_reqtimeout[] = { "mod_ssl.c", "mod_reqtimeout.c", NULL}; void h2_h2_register_hooks(void) { @@ -558,7 +553,7 @@ * a chance to take over before it. */ ap_hook_process_connection(h2_h2_process_conn, - mod_ssl, mod_reqtimeout, APR_HOOK_LAST); + mod_reqtimeout, NULL, APR_HOOK_LAST); /* One last chance to properly say goodbye if we have not done so * already. */ @@ -581,14 +576,17 @@ { apr_status_t status; h2_ctx *ctx; + server_rec *s; if (c->master) { return DECLINED; } ctx = h2_ctx_get(c, 0); + s = ctx? ctx->server : c->base_server; + ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, "h2_h2, process_conn"); - if (h2_ctx_is_task(ctx)) { + if (ctx && ctx->task) { /* our stream pseudo connection */ ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, c, "h2_h2, task, declined"); return DECLINED; @@ -601,19 +599,19 @@ ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, "h2_h2, process_conn, " "new connection using protocol '%s', direct=%d, " "tls acceptable=%d", proto, h2_allows_h2_direct(c), - h2_is_acceptable_connection(c, 1)); + h2_is_acceptable_connection(c, NULL, 1)); } if (!strcmp(AP_PROTOCOL_HTTP1, proto) && h2_allows_h2_direct(c) - && h2_is_acceptable_connection(c, 1)) { + && h2_is_acceptable_connection(c, NULL, 1)) { /* Fresh connection still is on http/1.1 and H2Direct is enabled. * Otherwise connection is in a fully acceptable state. * -> peek at the first 24 incoming bytes */ apr_bucket_brigade *temp; - char *s = NULL; - apr_size_t slen; + char *peek = NULL; + apr_size_t peeklen; temp = apr_brigade_create(c->pool, c->bucket_alloc); status = ap_get_brigade(c->input_filters, temp, @@ -626,8 +624,8 @@ return DECLINED; } - apr_brigade_pflatten(temp, &s, &slen, c->pool); - if ((slen >= 24) && !memcmp(H2_MAGIC_TOKEN, s, 24)) { + apr_brigade_pflatten(temp, &peek, &peeklen, c->pool); + if ((peeklen >= 24) && !memcmp(H2_MAGIC_TOKEN, peek, 24)) { ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, "h2_h2, direct mode detected"); if (!ctx) { @@ -638,7 +636,7 @@ else if (APLOGctrace2(c)) { ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, c, "h2_h2, not detected in %d bytes(base64): %s", - (int)slen, h2_util_base64url_encode(s, slen, c->pool)); + (int)peeklen, h2_util_base64url_encode(peek, peeklen, c->pool)); } apr_brigade_destroy(temp); @@ -647,15 +645,16 @@ if (ctx) { ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, "process_conn"); - if (!h2_ctx_session_get(ctx)) { - status = h2_conn_setup(ctx, c, NULL); + + if (!h2_ctx_get_session(c)) { + status = h2_conn_setup(c, NULL, s); ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, c, "conn_setup"); if (status != APR_SUCCESS) { h2_ctx_clear(c); return !OK; } } - h2_conn_run(ctx, c); + h2_conn_run(c); return OK; } @@ -667,7 +666,7 @@ { h2_ctx *ctx; - /* slave connection? */ + /* secondary connection? */ if (c->master) { return DECLINED; } @@ -684,16 +683,17 @@ static void check_push(request_rec *r, const char *tag) { - const h2_config *conf = h2_config_rget(r); - if (!r->expecting_100 - && conf && conf->push_list && conf->push_list->nelts > 0) { + apr_array_header_t *push_list = h2_config_push_list(r); + + if (!r->expecting_100 && push_list && push_list->nelts > 0) { int i, old_status; const char *old_line; + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, "%s, early announcing %d resources for push", - tag, conf->push_list->nelts); - for (i = 0; i < conf->push_list->nelts; ++i) { - h2_push_res *push = &APR_ARRAY_IDX(conf->push_list, i, h2_push_res); + tag, push_list->nelts); + for (i = 0; i < push_list->nelts; ++i) { + h2_push_res *push = &APR_ARRAY_IDX(push_list, i, h2_push_res); apr_table_add(r->headers_out, "Link", apr_psprintf(r->pool, "<%s>; rel=preload%s", push->uri_ref, push->critical? "; critical" : "")); @@ -710,10 +710,9 @@ static int h2_h2_post_read_req(request_rec *r) { - /* slave connection? */ + /* secondary connection? */ if (r->connection->master) { - h2_ctx *ctx = h2_ctx_rget(r); - struct h2_task *task = h2_ctx_get_task(ctx); + struct h2_task *task = h2_ctx_get_task(r->connection); /* This hook will get called twice on internal redirects. Take care * that we manipulate filters only once. */ if (task && !task->filters_set) { @@ -730,7 +729,7 @@ ap_add_output_filter("H2_RESPONSE", task, r, r->connection); for (f = r->input_filters; f; f = f->next) { - if (!strcmp("H2_SLAVE_IN", f->frec->name)) { + if (!strcmp("H2_SECONDARY_IN", f->frec->name)) { f->r = r; break; } @@ -744,17 +743,15 @@ static int h2_h2_late_fixups(request_rec *r) { - /* slave connection? */ + /* secondary connection? */ if (r->connection->master) { - h2_ctx *ctx = h2_ctx_rget(r); - struct h2_task *task = h2_ctx_get_task(ctx); + struct h2_task *task = h2_ctx_get_task(r->connection); if (task) { /* check if we copy vs. setaside files in this location */ - task->output.copy_files = h2_config_geti(h2_config_rget(r), - H2_CONF_COPY_FILES); + task->output.copy_files = h2_config_rgeti(r, H2_CONF_COPY_FILES); if (task->output.copy_files) { ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, task->c, - "h2_slave_out(%s): copy_files on", task->id); + "h2_secondary_out(%s): copy_files on", task->id); h2_beam_on_file_beam(task->output.beam, h2_beam_no_files, NULL); } check_push(r, "late_fixup"); --- a/modules/http2/h2_h2.h +++ b/modules/http2/h2_h2.h @@ -57,23 +57,15 @@ * the handshake is still ongoing. * @return != 0 iff connection requirements are met */ -int h2_is_acceptable_connection(conn_rec *c, int require_all); - -/** - * Check if the "direct" HTTP/2 mode of protocol handling is enabled - * for the given connection. - * @param c the connection to check - * @return != 0 iff direct mode is enabled - */ -int h2_allows_h2_direct(conn_rec *c); +int h2_is_acceptable_connection(conn_rec *c, request_rec *r, int require_all); /** * Check if the "Upgrade" HTTP/1.1 mode of protocol switching is enabled - * for the given connection. - * @param c the connection to check + * for the given request. + * @param r the request to check * @return != 0 iff Upgrade switching is enabled */ -int h2_allows_h2_upgrade(conn_rec *c); +int h2_allows_h2_upgrade(request_rec *r); #endif /* defined(__mod_h2__h2_h2__) */ --- a/modules/http2/h2_headers.c +++ b/modules/http2/h2_headers.c @@ -28,6 +28,7 @@ #include "h2_private.h" #include "h2_h2.h" +#include "h2_config.h" #include "h2_util.h" #include "h2_request.h" #include "h2_headers.h" @@ -101,8 +102,9 @@ const apr_bucket *src) { if (H2_BUCKET_IS_HEADERS(src)) { - h2_headers *r = ((h2_bucket_headers *)src->data)->headers; - apr_bucket *b = h2_bucket_headers_create(dest->bucket_alloc, r); + h2_headers *src_headers = ((h2_bucket_headers *)src->data)->headers; + apr_bucket *b = h2_bucket_headers_create(dest->bucket_alloc, + h2_headers_clone(dest->p, src_headers)); APR_BRIGADE_INSERT_TAIL(dest, b); return b; } @@ -128,28 +130,41 @@ { h2_headers *headers = h2_headers_create(status, header, r->notes, 0, pool); if (headers->status == HTTP_FORBIDDEN) { - const char *cause = apr_table_get(r->notes, "ssl-renegotiate-forbidden"); - if (cause) { - /* This request triggered a TLS renegotiation that is now allowed - * in HTTP/2. Tell the client that it should use HTTP/1.1 for this. - */ - ap_log_rerror(APLOG_MARK, APLOG_DEBUG, headers->status, r, - APLOGNO(03061) - "h2_headers(%ld): renegotiate forbidden, cause: %s", - (long)r->connection->id, cause); - headers->status = H2_ERR_HTTP_1_1_REQUIRED; + request_rec *r_prev; + for (r_prev = r; r_prev != NULL; r_prev = r_prev->prev) { + const char *cause = apr_table_get(r_prev->notes, "ssl-renegotiate-forbidden"); + if (cause) { + /* This request triggered a TLS renegotiation that is not allowed + * in HTTP/2. Tell the client that it should use HTTP/1.1 for this. + */ + ap_log_rerror(APLOG_MARK, APLOG_DEBUG, headers->status, r, + APLOGNO(03061) + "h2_headers(%ld): renegotiate forbidden, cause: %s", + (long)r->connection->id, cause); + headers->status = H2_ERR_HTTP_1_1_REQUIRED; + break; + } } } if (is_unsafe(r->server)) { - apr_table_setn(headers->notes, H2_HDR_CONFORMANCE, - H2_HDR_CONFORMANCE_UNSAFE); + apr_table_setn(headers->notes, H2_HDR_CONFORMANCE, H2_HDR_CONFORMANCE_UNSAFE); + } + if (h2_config_rgeti(r, H2_CONF_PUSH) == 0 && h2_config_sgeti(r->server, H2_CONF_PUSH) != 0) { + apr_table_setn(headers->notes, H2_PUSH_MODE_NOTE, "0"); } return headers; } h2_headers *h2_headers_copy(apr_pool_t *pool, h2_headers *h) { - return h2_headers_create(h->status, h->headers, h->notes, h->raw_bytes, pool); + return h2_headers_create(h->status, apr_table_copy(pool, h->headers), + apr_table_copy(pool, h->notes), h->raw_bytes, pool); +} + +h2_headers *h2_headers_clone(apr_pool_t *pool, h2_headers *h) +{ + return h2_headers_create(h->status, apr_table_clone(pool, h->headers), + apr_table_clone(pool, h->notes), h->raw_bytes, pool); } h2_headers *h2_headers_die(apr_status_t type, --- a/modules/http2/h2_headers.h +++ b/modules/http2/h2_headers.h @@ -59,12 +59,18 @@ apr_table_t *header, apr_pool_t *pool); /** - * Clone the headers into another pool. This will not copy any + * Copy the headers into another pool. This will not copy any * header strings. */ h2_headers *h2_headers_copy(apr_pool_t *pool, h2_headers *h); /** + * Clone the headers into another pool. This will also clone any + * header strings. + */ +h2_headers *h2_headers_clone(apr_pool_t *pool, h2_headers *h); + +/** * Create the headers for the given error. * @param stream_id id of the stream to create the headers for * @param type the error code --- a/modules/http2/h2_mplx.c +++ b/modules/http2/h2_mplx.c @@ -40,7 +40,6 @@ #include "h2_ctx.h" #include "h2_h2.h" #include "h2_mplx.h" -#include "h2_ngn_shed.h" #include "h2_request.h" #include "h2_stream.h" #include "h2_session.h" @@ -54,9 +53,21 @@ h2_mplx *m; h2_stream *stream; apr_time_t now; + apr_size_t count; } stream_iter_ctx; -apr_status_t h2_mplx_child_init(apr_pool_t *pool, server_rec *s) +/** + * Naming convention for static functions: + * - m_*: function only called from the master connection + * - s_*: function only called from a secondary connection + * - t_*: function only called from a h2_task holder + * - mst_*: function called from everyone + */ + +static apr_status_t s_mplx_be_happy(h2_mplx *m, h2_task *task); +static apr_status_t m_be_annoyed(h2_mplx *m); + +apr_status_t h2_mplx_m_child_init(apr_pool_t *pool, server_rec *s) { return APR_SUCCESS; } @@ -72,46 +83,40 @@ #define H2_MPLX_ENTER_ALWAYS(m) \ apr_thread_mutex_lock(m->lock) -#define H2_MPLX_ENTER_MAYBE(m, lock) \ - if (lock) apr_thread_mutex_lock(m->lock) +#define H2_MPLX_ENTER_MAYBE(m, dolock) \ + if (dolock) apr_thread_mutex_lock(m->lock) -#define H2_MPLX_LEAVE_MAYBE(m, lock) \ - if (lock) apr_thread_mutex_unlock(m->lock) +#define H2_MPLX_LEAVE_MAYBE(m, dolock) \ + if (dolock) apr_thread_mutex_unlock(m->lock) -static void check_data_for(h2_mplx *m, h2_stream *stream, int lock); +static void mst_check_data_for(h2_mplx *m, h2_stream *stream, int mplx_is_locked); -static void stream_output_consumed(void *ctx, - h2_bucket_beam *beam, apr_off_t length) +static void mst_stream_output_consumed(void *ctx, h2_bucket_beam *beam, apr_off_t length) { - h2_stream *stream = ctx; - h2_task *task = stream->task; - - if (length > 0 && task && task->assigned) { - h2_req_engine_out_consumed(task->assigned, task->c, length); - } } -static void stream_input_ev(void *ctx, h2_bucket_beam *beam) +static void mst_stream_input_ev(void *ctx, h2_bucket_beam *beam) { h2_stream *stream = ctx; h2_mplx *m = stream->session->mplx; apr_atomic_set32(&m->event_pending, 1); } -static void stream_input_consumed(void *ctx, h2_bucket_beam *beam, apr_off_t length) +static void m_stream_input_consumed(void *ctx, h2_bucket_beam *beam, apr_off_t length) { h2_stream_in_consumed(ctx, length); } -static void stream_joined(h2_mplx *m, h2_stream *stream) +static void ms_stream_joined(h2_mplx *m, h2_stream *stream) { - ap_assert(!stream->task || stream->task->worker_done); + ap_assert(!h2_task_has_started(stream->task) || stream->task->worker_done); + h2_ififo_remove(m->readyq, stream->id); h2_ihash_remove(m->shold, stream->id); h2_ihash_add(m->spurge, stream); } -static void stream_cleanup(h2_mplx *m, h2_stream *stream) +static void m_stream_cleanup(h2_mplx *m, h2_stream *stream) { ap_assert(stream->state == H2_SS_CLEANUP); @@ -128,15 +133,16 @@ h2_ihash_remove(m->streams, stream->id); h2_iq_remove(m->q, stream->id); - h2_ififo_remove(m->readyq, stream->id); - h2_ihash_add(m->shold, stream); - if (!stream->task || stream->task->worker_done) { - stream_joined(m, stream); + if (!h2_task_has_started(stream->task) || stream->task->done_done) { + ms_stream_joined(m, stream); } - else if (stream->task) { - stream->task->c->aborted = 1; - apr_thread_cond_broadcast(m->task_thawed); + else { + h2_ififo_remove(m->readyq, stream->id); + h2_ihash_add(m->shold, stream); + if (stream->task) { + stream->task->c->aborted = 1; + } } } @@ -151,29 +157,23 @@ * their HTTP/1 cousins, the separate allocator seems to work better * than protecting a shared h2_session one with an own lock. */ -h2_mplx *h2_mplx_create(conn_rec *c, apr_pool_t *parent, - const h2_config *conf, - h2_workers *workers) +h2_mplx *h2_mplx_m_create(conn_rec *c, server_rec *s, apr_pool_t *parent, + h2_workers *workers) { apr_status_t status = APR_SUCCESS; apr_allocator_t *allocator; apr_thread_mutex_t *mutex; h2_mplx *m; - h2_ctx *ctx = h2_ctx_get(c, 0); - ap_assert(conf); m = apr_pcalloc(parent, sizeof(h2_mplx)); if (m) { m->id = c->id; m->c = c; - m->s = (ctx? h2_ctx_server_get(ctx) : NULL); - if (!m->s) { - m->s = c->base_server; - } + m->s = s; /* We create a pool with its own allocator to be used for - * processing slave connections. This is the only way to have the - * processing independant of its parent pool in the sense that it + * processing secondary connections. This is the only way to have the + * processing independent of its parent pool in the sense that it * can work in another thread. Also, the new allocator needs its own * mutex to synchronize sub-pools. */ @@ -204,17 +204,10 @@ return NULL; } - status = apr_thread_cond_create(&m->task_thawed, m->pool); - if (status != APR_SUCCESS) { - apr_pool_destroy(m->pool); - return NULL; - } - - m->max_streams = h2_config_geti(conf, H2_CONF_MAX_STREAMS); - m->stream_max_mem = h2_config_geti(conf, H2_CONF_STREAM_MAX_MEM); + m->max_streams = h2_config_sgeti(s, H2_CONF_MAX_STREAMS); + m->stream_max_mem = h2_config_sgeti(s, H2_CONF_STREAM_MAX_MEM); m->streams = h2_ihash_create(m->pool, offsetof(h2_stream,id)); - m->sredo = h2_ihash_create(m->pool, offsetof(h2_stream,id)); m->shold = h2_ihash_create(m->pool, offsetof(h2_stream,id)); m->spurge = h2_ihash_create(m->pool, offsetof(h2_stream,id)); m->q = h2_iq_create(m->pool, m->max_streams); @@ -228,19 +221,15 @@ m->workers = workers; m->max_active = workers->max_workers; m->limit_active = 6; /* the original h1 max parallel connections */ - m->last_limit_change = m->last_idle_block = apr_time_now(); - m->limit_change_interval = apr_time_from_msec(100); - - m->spare_slaves = apr_array_make(m->pool, 10, sizeof(conn_rec*)); + m->last_mood_change = apr_time_now(); + m->mood_update_interval = apr_time_from_msec(100); - m->ngn_shed = h2_ngn_shed_create(m->pool, m->c, m->max_streams, - m->stream_max_mem); - h2_ngn_shed_set_ctx(m->ngn_shed , m); + m->spare_secondary = apr_array_make(m->pool, 10, sizeof(conn_rec*)); } return m; } -int h2_mplx_shutdown(h2_mplx *m) +int h2_mplx_m_shutdown(h2_mplx *m) { int max_stream_started = 0; @@ -254,7 +243,7 @@ return max_stream_started; } -static int input_consumed_signal(h2_mplx *m, h2_stream *stream) +static int m_input_consumed_signal(h2_mplx *m, h2_stream *stream) { if (stream->input) { return h2_beam_report_consumption(stream->input); @@ -262,12 +251,12 @@ return 0; } -static int report_consumption_iter(void *ctx, void *val) +static int m_report_consumption_iter(void *ctx, void *val) { h2_stream *stream = val; h2_mplx *m = ctx; - input_consumed_signal(m, stream); + m_input_consumed_signal(m, stream); if (stream->state == H2_SS_CLOSED_L && (!stream->task || stream->task->worker_done)) { ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, m->c, @@ -278,7 +267,7 @@ return 1; } -static int output_consumed_signal(h2_mplx *m, h2_task *task) +static int s_output_consumed_signal(h2_mplx *m, h2_task *task) { if (task->output.beam) { return h2_beam_report_consumption(task->output.beam); @@ -286,7 +275,7 @@ return 0; } -static int stream_destroy_iter(void *ctx, void *val) +static int m_stream_destroy_iter(void *ctx, void *val) { h2_mplx *m = ctx; h2_stream *stream = val; @@ -296,7 +285,7 @@ if (stream->input) { /* Process outstanding events before destruction */ - input_consumed_signal(m, stream); + m_input_consumed_signal(m, stream); h2_beam_log(stream->input, m->c, APLOG_TRACE2, "stream_destroy"); h2_beam_destroy(stream->input); stream->input = NULL; @@ -304,12 +293,12 @@ if (stream->task) { h2_task *task = stream->task; - conn_rec *slave; - int reuse_slave = 0; + conn_rec *secondary; + int reuse_secondary = 0; stream->task = NULL; - slave = task->c; - if (slave) { + secondary = task->c; + if (secondary) { /* On non-serialized requests, the IO logging has not accounted for any * meta data send over the network: response headers and h2 frame headers. we * counted this on the stream and need to add this now. @@ -318,26 +307,25 @@ if (task->request && !task->request->serialize && h2_task_logio_add_bytes_out) { apr_off_t unaccounted = stream->out_frame_octets - stream->out_data_octets; if (unaccounted > 0) { - h2_task_logio_add_bytes_out(slave, unaccounted); + h2_task_logio_add_bytes_out(secondary, unaccounted); } } - if (m->s->keep_alive_max == 0 || slave->keepalives < m->s->keep_alive_max) { - reuse_slave = ((m->spare_slaves->nelts < (m->limit_active * 3 / 2)) - && !task->rst_error); + if (m->s->keep_alive_max == 0 || secondary->keepalives < m->s->keep_alive_max) { + reuse_secondary = ((m->spare_secondary->nelts < (m->limit_active * 3 / 2)) + && !task->rst_error); } - task->c = NULL; - if (reuse_slave) { + if (reuse_secondary) { h2_beam_log(task->output.beam, m->c, APLOG_DEBUG, - APLOGNO(03385) "h2_task_destroy, reuse slave"); + APLOGNO(03385) "h2_task_destroy, reuse secondary"); h2_task_destroy(task); - APR_ARRAY_PUSH(m->spare_slaves, conn_rec*) = slave; + APR_ARRAY_PUSH(m->spare_secondary, conn_rec*) = secondary; } else { h2_beam_log(task->output.beam, m->c, APLOG_TRACE1, - "h2_task_destroy, destroy slave"); - h2_slave_destroy(slave); + "h2_task_destroy, destroy secondary"); + h2_secondary_destroy(secondary); } } } @@ -345,11 +333,11 @@ return 0; } -static void purge_streams(h2_mplx *m, int lock) +static void m_purge_streams(h2_mplx *m, int lock) { if (!h2_ihash_empty(m->spurge)) { H2_MPLX_ENTER_MAYBE(m, lock); - while (!h2_ihash_iter(m->spurge, stream_destroy_iter, m)) { + while (!h2_ihash_iter(m->spurge, m_stream_destroy_iter, m)) { /* repeat until empty */ } H2_MPLX_LEAVE_MAYBE(m, lock); @@ -361,13 +349,13 @@ void *ctx; } stream_iter_ctx_t; -static int stream_iter_wrap(void *ctx, void *stream) +static int m_stream_iter_wrap(void *ctx, void *stream) { stream_iter_ctx_t *x = ctx; return x->cb(stream, x->ctx); } -apr_status_t h2_mplx_stream_do(h2_mplx *m, h2_mplx_stream_cb *cb, void *ctx) +apr_status_t h2_mplx_m_stream_do(h2_mplx *m, h2_mplx_stream_cb *cb, void *ctx) { stream_iter_ctx_t x; @@ -375,13 +363,13 @@ x.cb = cb; x.ctx = ctx; - h2_ihash_iter(m->streams, stream_iter_wrap, &x); + h2_ihash_iter(m->streams, m_stream_iter_wrap, &x); H2_MPLX_LEAVE(m); return APR_SUCCESS; } -static int report_stream_iter(void *ctx, void *val) { +static int m_report_stream_iter(void *ctx, void *val) { h2_mplx *m = ctx; h2_stream *stream = val; h2_task *task = stream->task; @@ -394,10 +382,10 @@ if (task) { ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, m->c, /* NO APLOGNO */ H2_STRM_MSG(stream, "->03198: %s %s %s" - "[started=%d/done=%d/frozen=%d]"), + "[started=%d/done=%d]"), task->request->method, task->request->authority, task->request->path, task->worker_started, - task->worker_done, task->frozen); + task->worker_done); } else { ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, m->c, /* NO APLOGNO */ @@ -406,7 +394,7 @@ return 1; } -static int unexpected_stream_iter(void *ctx, void *val) { +static int m_unexpected_stream_iter(void *ctx, void *val) { h2_mplx *m = ctx; h2_stream *stream = val; ap_log_cerror(APLOG_MARK, APLOG_WARNING, 0, m->c, /* NO APLOGNO */ @@ -415,7 +403,7 @@ return 1; } -static int stream_cancel_iter(void *ctx, void *val) { +static int m_stream_cancel_iter(void *ctx, void *val) { h2_mplx *m = ctx; h2_stream *stream = val; @@ -429,14 +417,14 @@ h2_stream_rst(stream, H2_ERR_NO_ERROR); /* All connection data has been sent, simulate cleanup */ h2_stream_dispatch(stream, H2_SEV_EOS_SENT); - stream_cleanup(m, stream); + m_stream_cleanup(m, stream); return 0; } -void h2_mplx_release_and_join(h2_mplx *m, apr_thread_cond_t *wait) +void h2_mplx_m_release_and_join(h2_mplx *m, apr_thread_cond_t *wait) { apr_status_t status; - int i, wait_secs = 60; + int i, wait_secs = 60, old_aborted; ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, m->c, "h2_mplx(%ld): start release", m->id); @@ -447,15 +435,23 @@ H2_MPLX_ENTER_ALWAYS(m); + /* While really terminating any secondary connections, treat the master + * connection as aborted. It's not as if we could send any more data + * at this point. */ + old_aborted = m->c->aborted; + m->c->aborted = 1; + /* How to shut down a h2 connection: * 1. cancel all streams still active */ - while (!h2_ihash_iter(m->streams, stream_cancel_iter, m)) { + ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, m->c, + "h2_mplx(%ld): release, %d/%d/%d streams (total/hold/purge), %d active tasks", + m->id, (int)h2_ihash_count(m->streams), + (int)h2_ihash_count(m->shold), (int)h2_ihash_count(m->spurge), m->tasks_active); + while (!h2_ihash_iter(m->streams, m_stream_cancel_iter, m)) { /* until empty */ } - /* 2. terminate ngn_shed, no more streams - * should be scheduled or in the active set */ - h2_ngn_shed_abort(m->ngn_shed); + /* 2. no more streams should be scheduled or in the active set */ ap_assert(h2_ihash_empty(m->streams)); ap_assert(h2_iq_empty(m->q)); @@ -473,65 +469,60 @@ ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, m->c, APLOGNO(03198) "h2_mplx(%ld): waited %d sec for %d tasks", m->id, i*wait_secs, (int)h2_ihash_count(m->shold)); - h2_ihash_iter(m->shold, report_stream_iter, m); + h2_ihash_iter(m->shold, m_report_stream_iter, m); } } - ap_assert(m->tasks_active == 0); m->join_wait = NULL; - - /* 4. close the h2_req_enginge shed */ - h2_ngn_shed_destroy(m->ngn_shed); - m->ngn_shed = NULL; - + /* 4. With all workers done, all streams should be in spurge */ + ap_assert(m->tasks_active == 0); if (!h2_ihash_empty(m->shold)) { ap_log_cerror(APLOG_MARK, APLOG_WARNING, 0, m->c, APLOGNO(03516) "h2_mplx(%ld): unexpected %d streams in hold", m->id, (int)h2_ihash_count(m->shold)); - h2_ihash_iter(m->shold, unexpected_stream_iter, m); + h2_ihash_iter(m->shold, m_unexpected_stream_iter, m); } + m->c->aborted = old_aborted; H2_MPLX_LEAVE(m); - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, m->c, - "h2_mplx(%ld): released", m->id); + ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, m->c, "h2_mplx(%ld): released", m->id); } -apr_status_t h2_mplx_stream_cleanup(h2_mplx *m, h2_stream *stream) +apr_status_t h2_mplx_m_stream_cleanup(h2_mplx *m, h2_stream *stream) { H2_MPLX_ENTER(m); ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, m->c, H2_STRM_MSG(stream, "cleanup")); - stream_cleanup(m, stream); + m_stream_cleanup(m, stream); H2_MPLX_LEAVE(m); return APR_SUCCESS; } -h2_stream *h2_mplx_stream_get(h2_mplx *m, int id) +h2_stream *h2_mplx_t_stream_get(h2_mplx *m, h2_task *task) { h2_stream *s = NULL; H2_MPLX_ENTER_ALWAYS(m); - s = h2_ihash_get(m->streams, id); + s = h2_ihash_get(m->streams, task->stream_id); H2_MPLX_LEAVE(m); return s; } -static void output_produced(void *ctx, h2_bucket_beam *beam, apr_off_t bytes) +static void mst_output_produced(void *ctx, h2_bucket_beam *beam, apr_off_t bytes) { h2_stream *stream = ctx; h2_mplx *m = stream->session->mplx; - check_data_for(m, stream, 1); + mst_check_data_for(m, stream, 0); } -static apr_status_t out_open(h2_mplx *m, int stream_id, h2_bucket_beam *beam) +static apr_status_t t_out_open(h2_mplx *m, int stream_id, h2_bucket_beam *beam) { - apr_status_t status = APR_SUCCESS; h2_stream *stream = h2_ihash_get(m->streams, stream_id); if (!stream || !stream->task || m->aborted) { @@ -542,26 +533,26 @@ stream->output = beam; if (APLOGctrace2(m->c)) { - h2_beam_log(beam, m->c, APLOG_TRACE2, "out_open"); + h2_beam_log(beam, stream->task->c, APLOG_TRACE2, "out_open"); } else { - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, m->c, + ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, stream->task->c, "h2_mplx(%s): out open", stream->task->id); } - h2_beam_on_consumed(stream->output, NULL, stream_output_consumed, stream); - h2_beam_on_produced(stream->output, output_produced, stream); + h2_beam_on_consumed(stream->output, NULL, mst_stream_output_consumed, stream); + h2_beam_on_produced(stream->output, mst_output_produced, stream); if (stream->task->output.copy_files) { h2_beam_on_file_beam(stream->output, h2_beam_no_files, NULL); } /* we might see some file buckets in the output, see * if we have enough handles reserved. */ - check_data_for(m, stream, 0); - return status; + mst_check_data_for(m, stream, 1); + return APR_SUCCESS; } -apr_status_t h2_mplx_out_open(h2_mplx *m, int stream_id, h2_bucket_beam *beam) +apr_status_t h2_mplx_t_out_open(h2_mplx *m, int stream_id, h2_bucket_beam *beam) { apr_status_t status; @@ -571,14 +562,14 @@ status = APR_ECONNABORTED; } else { - status = out_open(m, stream_id, beam); + status = t_out_open(m, stream_id, beam); } H2_MPLX_LEAVE(m); return status; } -static apr_status_t out_close(h2_mplx *m, h2_task *task) +static apr_status_t s_out_close(h2_mplx *m, h2_task *task) { apr_status_t status = APR_SUCCESS; h2_stream *stream; @@ -595,17 +586,17 @@ return APR_ECONNABORTED; } - ap_log_cerror(APLOG_MARK, APLOG_TRACE2, status, m->c, + ap_log_cerror(APLOG_MARK, APLOG_TRACE2, status, task->c, "h2_mplx(%s): close", task->id); status = h2_beam_close(task->output.beam); - h2_beam_log(task->output.beam, m->c, APLOG_TRACE2, "out_close"); - output_consumed_signal(m, task); - check_data_for(m, stream, 0); + h2_beam_log(task->output.beam, task->c, APLOG_TRACE2, "out_close"); + s_output_consumed_signal(m, task); + mst_check_data_for(m, stream, 1); return status; } -apr_status_t h2_mplx_out_trywait(h2_mplx *m, apr_interval_time_t timeout, - apr_thread_cond_t *iowait) +apr_status_t h2_mplx_m_out_trywait(h2_mplx *m, apr_interval_time_t timeout, + apr_thread_cond_t *iowait) { apr_status_t status; @@ -614,12 +605,12 @@ if (m->aborted) { status = APR_ECONNABORTED; } - else if (h2_mplx_has_master_events(m)) { + else if (h2_mplx_m_has_master_events(m)) { status = APR_SUCCESS; } else { - purge_streams(m, 0); - h2_ihash_iter(m->streams, report_consumption_iter, m); + m_purge_streams(m, 0); + h2_ihash_iter(m->streams, m_report_consumption_iter, m); m->added_output = iowait; status = apr_thread_cond_timedwait(m->added_output, m->lock, timeout); if (APLOGctrace2(m->c)) { @@ -634,19 +625,27 @@ return status; } -static void check_data_for(h2_mplx *m, h2_stream *stream, int lock) +static void mst_check_data_for(h2_mplx *m, h2_stream *stream, int mplx_is_locked) { + /* If m->lock is already held, we must release during h2_ififo_push() + * which can wait on its not_full condition, causing a deadlock because + * no one would then be able to acquire m->lock to empty the fifo. + */ + H2_MPLX_LEAVE_MAYBE(m, mplx_is_locked); if (h2_ififo_push(m->readyq, stream->id) == APR_SUCCESS) { + H2_MPLX_ENTER_ALWAYS(m); apr_atomic_set32(&m->event_pending, 1); - H2_MPLX_ENTER_MAYBE(m, lock); if (m->added_output) { apr_thread_cond_signal(m->added_output); } - H2_MPLX_LEAVE_MAYBE(m, lock); + H2_MPLX_LEAVE_MAYBE(m, !mplx_is_locked); + } + else { + H2_MPLX_ENTER_MAYBE(m, mplx_is_locked); } } -apr_status_t h2_mplx_reprioritize(h2_mplx *m, h2_stream_pri_cmp *cmp, void *ctx) +apr_status_t h2_mplx_m_reprioritize(h2_mplx *m, h2_stream_pri_cmp *cmp, void *ctx) { apr_status_t status; @@ -666,22 +665,22 @@ return status; } -static void register_if_needed(h2_mplx *m) +static void ms_register_if_needed(h2_mplx *m, int from_master) { if (!m->aborted && !m->is_registered && !h2_iq_empty(m->q)) { apr_status_t status = h2_workers_register(m->workers, m); if (status == APR_SUCCESS) { m->is_registered = 1; } - else { + else if (from_master) { ap_log_cerror(APLOG_MARK, APLOG_ERR, status, m->c, APLOGNO(10021) "h2_mplx(%ld): register at workers", m->id); } } } -apr_status_t h2_mplx_process(h2_mplx *m, struct h2_stream *stream, - h2_stream_pri_cmp *cmp, void *ctx) +apr_status_t h2_mplx_m_process(h2_mplx *m, struct h2_stream *stream, + h2_stream_pri_cmp *cmp, void *ctx) { apr_status_t status; @@ -695,13 +694,13 @@ h2_ihash_add(m->streams, stream); if (h2_stream_is_ready(stream)) { /* already have a response */ - check_data_for(m, stream, 0); + mst_check_data_for(m, stream, 1); ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, m->c, H2_STRM_MSG(stream, "process, add to readyq")); } else { h2_iq_add(m->q, stream->id, cmp, ctx); - register_if_needed(m); + ms_register_if_needed(m, 1); ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, m->c, H2_STRM_MSG(stream, "process, added to q")); } @@ -711,7 +710,7 @@ return status; } -static h2_task *next_stream_task(h2_mplx *m) +static h2_task *s_next_stream_task(h2_mplx *m) { h2_stream *stream; int sid; @@ -720,40 +719,39 @@ stream = h2_ihash_get(m->streams, sid); if (stream) { - conn_rec *slave, **pslave; + conn_rec *secondary, **psecondary; - pslave = (conn_rec **)apr_array_pop(m->spare_slaves); - if (pslave) { - slave = *pslave; - slave->aborted = 0; + psecondary = (conn_rec **)apr_array_pop(m->spare_secondary); + if (psecondary) { + secondary = *psecondary; + secondary->aborted = 0; } else { - slave = h2_slave_create(m->c, stream->id, m->pool); + secondary = h2_secondary_create(m->c, stream->id, m->pool); } if (!stream->task) { - if (sid > m->max_stream_started) { m->max_stream_started = sid; } if (stream->input) { - h2_beam_on_consumed(stream->input, stream_input_ev, - stream_input_consumed, stream); + h2_beam_on_consumed(stream->input, mst_stream_input_ev, + m_stream_input_consumed, stream); } - stream->task = h2_task_create(slave, stream->id, + stream->task = h2_task_create(secondary, stream->id, stream->request, m, stream->input, stream->session->s->timeout, m->stream_max_mem); if (!stream->task) { - ap_log_cerror(APLOG_MARK, APLOG_ERR, APR_ENOMEM, slave, + ap_log_cerror(APLOG_MARK, APLOG_ERR, APR_ENOMEM, secondary, H2_STRM_LOG(APLOGNO(02941), stream, "create task")); return NULL; } - } + stream->task->started_at = apr_time_now(); ++m->tasks_active; return stream->task; } @@ -761,7 +759,7 @@ return NULL; } -apr_status_t h2_mplx_pop_task(h2_mplx *m, h2_task **ptask) +apr_status_t h2_mplx_s_pop_task(h2_mplx *m, h2_task **ptask) { apr_status_t rv = APR_EOF; @@ -777,7 +775,7 @@ rv = APR_EOF; } else { - *ptask = next_stream_task(m); + *ptask = s_next_stream_task(m); rv = (*ptask != NULL && !h2_iq_empty(m->q))? APR_EAGAIN : APR_SUCCESS; } if (APR_EAGAIN != rv) { @@ -787,127 +785,87 @@ return rv; } -static void task_done(h2_mplx *m, h2_task *task, h2_req_engine *ngn) +static void s_task_done(h2_mplx *m, h2_task *task) { h2_stream *stream; - if (task->frozen) { - /* this task was handed over to an engine for processing - * and the original worker has finished. That means the - * engine may start processing now. */ - h2_task_thaw(task); - apr_thread_cond_broadcast(m->task_thawed); - return; - } - - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, m->c, + ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, task->c, "h2_mplx(%ld): task(%s) done", m->id, task->id); - out_close(m, task); - - if (ngn) { - apr_off_t bytes = 0; - h2_beam_send(task->output.beam, NULL, APR_NONBLOCK_READ); - bytes += h2_beam_get_buffered(task->output.beam); - if (bytes > 0) { - /* we need to report consumed and current buffered output - * to the engine. The request will be streamed out or cancelled, - * no more data is coming from it and the engine should update - * its calculations before we destroy this information. */ - h2_req_engine_out_consumed(ngn, task->c, bytes); - } - } - - if (task->engine) { - if (!m->aborted && !task->c->aborted - && !h2_req_engine_is_shutdown(task->engine)) { - ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, m->c, APLOGNO(10022) - "h2_mplx(%ld): task(%s) has not-shutdown " - "engine(%s)", m->id, task->id, - h2_req_engine_get_id(task->engine)); - } - h2_ngn_shed_done_ngn(m->ngn_shed, task->engine); - } + s_out_close(m, task); task->worker_done = 1; task->done_at = apr_time_now(); - ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, m->c, + ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, task->c, "h2_mplx(%s): request done, %f ms elapsed", task->id, (task->done_at - task->started_at) / 1000.0); - if (task->started_at > m->last_idle_block) { - /* this task finished without causing an 'idle block', e.g. - * a block by flow control. - */ - if (task->done_at- m->last_limit_change >= m->limit_change_interval - && m->limit_active < m->max_active) { - /* Well behaving stream, allow it more workers */ - m->limit_active = H2MIN(m->limit_active * 2, - m->max_active); - m->last_limit_change = task->done_at; - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, m->c, - "h2_mplx(%ld): increase worker limit to %d", - m->id, m->limit_active); - } + if (task->c && !task->c->aborted && task->started_at > m->last_mood_change) { + s_mplx_be_happy(m, task); } + ap_assert(task->done_done == 0); + stream = h2_ihash_get(m->streams, task->stream_id); if (stream) { /* stream not done yet. */ - if (!m->aborted && h2_ihash_get(m->sredo, stream->id)) { + if (!m->aborted && task->redo) { /* reset and schedule again */ h2_task_redo(task); - h2_ihash_remove(m->sredo, stream->id); h2_iq_add(m->q, stream->id, NULL, NULL); + ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, task->c, + H2_STRM_MSG(stream, "redo, added to q")); } else { /* stream not cleaned up, stay around */ - ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, m->c, + task->done_done = 1; + ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, task->c, H2_STRM_MSG(stream, "task_done, stream open")); if (stream->input) { h2_beam_leave(stream->input); } /* more data will not arrive, resume the stream */ - check_data_for(m, stream, 0); + mst_check_data_for(m, stream, 1); } } else if ((stream = h2_ihash_get(m->shold, task->stream_id)) != NULL) { /* stream is done, was just waiting for this. */ - ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, m->c, + task->done_done = 1; + ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, task->c, H2_STRM_MSG(stream, "task_done, in hold")); if (stream->input) { h2_beam_leave(stream->input); } - stream_joined(m, stream); + ms_stream_joined(m, stream); } else if ((stream = h2_ihash_get(m->spurge, task->stream_id)) != NULL) { - ap_log_cerror(APLOG_MARK, APLOG_WARNING, 0, m->c, + ap_log_cerror(APLOG_MARK, APLOG_WARNING, 0, task->c, H2_STRM_LOG(APLOGNO(03517), stream, "already in spurge")); ap_assert("stream should not be in spurge" == NULL); } else { - ap_log_cerror(APLOG_MARK, APLOG_WARNING, 0, m->c, APLOGNO(03518) + ap_log_cerror(APLOG_MARK, APLOG_WARNING, 0, task->c, APLOGNO(03518) "h2_mplx(%s): task_done, stream not found", task->id); ap_assert("stream should still be available" == NULL); } } -void h2_mplx_task_done(h2_mplx *m, h2_task *task, h2_task **ptask) +void h2_mplx_s_task_done(h2_mplx *m, h2_task *task, h2_task **ptask) { H2_MPLX_ENTER_ALWAYS(m); - task_done(m, task, NULL); --m->tasks_active; + s_task_done(m, task); if (m->join_wait) { apr_thread_cond_signal(m->join_wait); } if (ptask) { /* caller wants another task */ - *ptask = next_stream_task(m); + *ptask = s_next_stream_task(m); } - register_if_needed(m); + ms_register_if_needed(m, 0); H2_MPLX_LEAVE(m); } @@ -916,94 +874,161 @@ * h2_mplx DoS protection ******************************************************************************/ -static int latest_repeatable_unsubmitted_iter(void *data, void *val) +static int m_timed_out_busy_iter(void *data, void *val) { stream_iter_ctx *ctx = data; h2_stream *stream = val; - - if (stream->task && !stream->task->worker_done - && h2_task_can_redo(stream->task) - && !h2_ihash_get(ctx->m->sredo, stream->id)) { - if (!h2_stream_is_ready(stream)) { - /* this task occupies a worker, the response has not been submitted - * yet, not been cancelled and it is a repeatable request - * -> it can be re-scheduled later */ - if (!ctx->stream - || (ctx->stream->task->started_at < stream->task->started_at)) { - /* we did not have one or this one was started later */ - ctx->stream = stream; - } - } + if (h2_task_has_started(stream->task) && !stream->task->worker_done + && (ctx->now - stream->task->started_at) > stream->task->timeout) { + /* timed out stream occupying a worker, found */ + ctx->stream = stream; + return 0; } return 1; } -static h2_stream *get_latest_repeatable_unsubmitted_stream(h2_mplx *m) +static h2_stream *m_get_timed_out_busy_stream(h2_mplx *m) { stream_iter_ctx ctx; ctx.m = m; ctx.stream = NULL; - h2_ihash_iter(m->streams, latest_repeatable_unsubmitted_iter, &ctx); + ctx.now = apr_time_now(); + h2_ihash_iter(m->streams, m_timed_out_busy_iter, &ctx); return ctx.stream; } -static int timed_out_busy_iter(void *data, void *val) +static int m_latest_repeatable_unsubmitted_iter(void *data, void *val) { stream_iter_ctx *ctx = data; h2_stream *stream = val; - if (stream->task && !stream->task->worker_done - && (ctx->now - stream->task->started_at) > stream->task->timeout) { - /* timed out stream occupying a worker, found */ - ctx->stream = stream; - return 0; + + if (!stream->task) goto leave; + if (!h2_task_has_started(stream->task) || stream->task->worker_done) goto leave; + if (h2_stream_is_ready(stream)) goto leave; + if (stream->task->redo) { + ++ctx->count; + goto leave; + } + if (h2_task_can_redo(stream->task)) { + /* this task occupies a worker, the response has not been submitted + * yet, not been cancelled and it is a repeatable request + * -> we could redo it later */ + if (!ctx->stream + || (ctx->stream->task->started_at < stream->task->started_at)) { + /* we did not have one or this one was started later */ + ctx->stream = stream; + } } +leave: return 1; } -static h2_stream *get_timed_out_busy_stream(h2_mplx *m) +static apr_status_t m_assess_task_to_throttle(h2_task **ptask, h2_mplx *m) { stream_iter_ctx ctx; + + /* count the running tasks already marked for redo and get one that could + * be throttled */ + *ptask = NULL; ctx.m = m; ctx.stream = NULL; - ctx.now = apr_time_now(); - h2_ihash_iter(m->streams, timed_out_busy_iter, &ctx); - return ctx.stream; + ctx.count = 0; + h2_ihash_iter(m->streams, m_latest_repeatable_unsubmitted_iter, &ctx); + if (m->tasks_active - ctx.count > m->limit_active) { + /* we are above the limit of running tasks, accounting for the ones + * already throttled. */ + if (ctx.stream && ctx.stream->task) { + *ptask = ctx.stream->task; + return APR_EAGAIN; + } + /* above limit, be seeing no candidate for easy throttling */ + if (m_get_timed_out_busy_stream(m)) { + /* Too many busy workers, unable to cancel enough streams + * and with a busy, timed out stream, we tell the client + * to go away... */ + return APR_TIMEUP; + } + } + return APR_SUCCESS; } -static apr_status_t unschedule_slow_tasks(h2_mplx *m) +static apr_status_t m_unschedule_slow_tasks(h2_mplx *m) { - h2_stream *stream; - int n; + h2_task *task; + apr_status_t rv; /* Try to get rid of streams that occupy workers. Look for safe requests * that are repeatable. If none found, fail the connection. */ - n = (m->tasks_active - m->limit_active - (int)h2_ihash_count(m->sredo)); - while (n > 0 && (stream = get_latest_repeatable_unsubmitted_stream(m))) { + while (APR_EAGAIN == (rv = m_assess_task_to_throttle(&task, m))) { ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, m->c, "h2_mplx(%s): unschedule, resetting task for redo later", - stream->task->id); - h2_task_rst(stream->task, H2_ERR_CANCEL); - h2_ihash_add(m->sredo, stream); - --n; + task->id); + task->redo = 1; + h2_task_rst(task, H2_ERR_CANCEL); } - if ((m->tasks_active - h2_ihash_count(m->sredo)) > m->limit_active) { - h2_stream *stream = get_timed_out_busy_stream(m); - if (stream) { - /* Too many busy workers, unable to cancel enough streams - * and with a busy, timed out stream, we tell the client - * to go away... */ - return APR_TIMEUP; - } + return rv; +} + +static apr_status_t s_mplx_be_happy(h2_mplx *m, h2_task *task) +{ + apr_time_t now; + + --m->irritations_since; + now = apr_time_now(); + if (m->limit_active < m->max_active + && (now - m->last_mood_change >= m->mood_update_interval + || m->irritations_since < -m->limit_active)) { + m->limit_active = H2MIN(m->limit_active * 2, m->max_active); + m->last_mood_change = now; + m->irritations_since = 0; + ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, task->c, + "h2_mplx(%ld): mood update, increasing worker limit to %d", + m->id, m->limit_active); } return APR_SUCCESS; } -apr_status_t h2_mplx_idle(h2_mplx *m) +static apr_status_t m_be_annoyed(h2_mplx *m) { apr_status_t status = APR_SUCCESS; apr_time_t now; + + ++m->irritations_since; + now = apr_time_now(); + if (m->limit_active > 2 && + ((now - m->last_mood_change >= m->mood_update_interval) + || (m->irritations_since >= m->limit_active))) { + + if (m->limit_active > 16) { + m->limit_active = 16; + } + else if (m->limit_active > 8) { + m->limit_active = 8; + } + else if (m->limit_active > 4) { + m->limit_active = 4; + } + else if (m->limit_active > 2) { + m->limit_active = 2; + } + m->last_mood_change = now; + m->irritations_since = 0; + ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, m->c, + "h2_mplx(%ld): mood update, decreasing worker limit to %d", + m->id, m->limit_active); + } + + if (m->tasks_active > m->limit_active) { + status = m_unschedule_slow_tasks(m); + } + return status; +} + +apr_status_t h2_mplx_m_idle(h2_mplx *m) +{ + apr_status_t status = APR_SUCCESS; apr_size_t scount; H2_MPLX_ENTER(m); @@ -1023,31 +1048,7 @@ * of busy workers we allow for this connection until it * well behaves. */ - now = apr_time_now(); - m->last_idle_block = now; - if (m->limit_active > 2 - && now - m->last_limit_change >= m->limit_change_interval) { - if (m->limit_active > 16) { - m->limit_active = 16; - } - else if (m->limit_active > 8) { - m->limit_active = 8; - } - else if (m->limit_active > 4) { - m->limit_active = 4; - } - else if (m->limit_active > 2) { - m->limit_active = 2; - } - m->last_limit_change = now; - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, m->c, - "h2_mplx(%ld): decrease worker limit to %d", - m->id, m->limit_active); - } - - if (m->tasks_active > m->limit_active) { - status = unschedule_slow_tasks(m); - } + status = m_be_annoyed(m); } else if (!h2_iq_empty(m->q)) { ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, m->c, @@ -1077,167 +1078,30 @@ h2_beam_is_closed(stream->output), (long)h2_beam_get_buffered(stream->output)); h2_ihash_add(m->streams, stream); - check_data_for(m, stream, 0); + mst_check_data_for(m, stream, 1); stream->out_checked = 1; status = APR_EAGAIN; } } } } - register_if_needed(m); + ms_register_if_needed(m, 1); H2_MPLX_LEAVE(m); return status; } /******************************************************************************* - * HTTP/2 request engines - ******************************************************************************/ - -typedef struct { - h2_mplx * m; - h2_req_engine *ngn; - int streams_updated; -} ngn_update_ctx; - -static int ngn_update_window(void *ctx, void *val) -{ - ngn_update_ctx *uctx = ctx; - h2_stream *stream = val; - if (stream->task && stream->task->assigned == uctx->ngn - && output_consumed_signal(uctx->m, stream->task)) { - ++uctx->streams_updated; - } - return 1; -} - -static apr_status_t ngn_out_update_windows(h2_mplx *m, h2_req_engine *ngn) -{ - ngn_update_ctx ctx; - - ctx.m = m; - ctx.ngn = ngn; - ctx.streams_updated = 0; - h2_ihash_iter(m->streams, ngn_update_window, &ctx); - - return ctx.streams_updated? APR_SUCCESS : APR_EAGAIN; -} - -apr_status_t h2_mplx_req_engine_push(const char *ngn_type, - request_rec *r, - http2_req_engine_init *einit) -{ - apr_status_t status; - h2_mplx *m; - h2_task *task; - h2_stream *stream; - - task = h2_ctx_rget_task(r); - if (!task) { - return APR_ECONNABORTED; - } - m = task->mplx; - - H2_MPLX_ENTER(m); - - stream = h2_ihash_get(m->streams, task->stream_id); - if (stream) { - status = h2_ngn_shed_push_request(m->ngn_shed, ngn_type, r, einit); - } - else { - status = APR_ECONNABORTED; - } - - H2_MPLX_LEAVE(m); - return status; -} - -apr_status_t h2_mplx_req_engine_pull(h2_req_engine *ngn, - apr_read_type_e block, - int capacity, - request_rec **pr) -{ - h2_ngn_shed *shed = h2_ngn_shed_get_shed(ngn); - h2_mplx *m = h2_ngn_shed_get_ctx(shed); - apr_status_t status; - int want_shutdown; - - H2_MPLX_ENTER(m); - - want_shutdown = (block == APR_BLOCK_READ); - - /* Take this opportunity to update output consummation - * for this engine */ - ngn_out_update_windows(m, ngn); - - if (want_shutdown && !h2_iq_empty(m->q)) { - /* For a blocking read, check first if requests are to be - * had and, if not, wait a short while before doing the - * blocking, and if unsuccessful, terminating read. - */ - status = h2_ngn_shed_pull_request(shed, ngn, capacity, 1, pr); - if (APR_STATUS_IS_EAGAIN(status)) { - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, m->c, - "h2_mplx(%ld): start block engine pull", m->id); - apr_thread_cond_timedwait(m->task_thawed, m->lock, - apr_time_from_msec(20)); - status = h2_ngn_shed_pull_request(shed, ngn, capacity, 1, pr); - } - } - else { - status = h2_ngn_shed_pull_request(shed, ngn, capacity, - want_shutdown, pr); - } - - H2_MPLX_LEAVE(m); - return status; -} - -void h2_mplx_req_engine_done(h2_req_engine *ngn, conn_rec *r_conn, - apr_status_t status) -{ - h2_task *task = h2_ctx_cget_task(r_conn); - - if (task) { - h2_mplx *m = task->mplx; - h2_stream *stream; - - H2_MPLX_ENTER_ALWAYS(m); - - stream = h2_ihash_get(m->streams, task->stream_id); - - ngn_out_update_windows(m, ngn); - h2_ngn_shed_done_task(m->ngn_shed, ngn, task); - - if (status != APR_SUCCESS && stream - && h2_task_can_redo(task) - && !h2_ihash_get(m->sredo, stream->id)) { - h2_ihash_add(m->sredo, stream); - } - - if (task->engine) { - /* cannot report that as done until engine returns */ - } - else { - task_done(m, task, ngn); - } - - H2_MPLX_LEAVE(m); - } -} - -/******************************************************************************* * mplx master events dispatching ******************************************************************************/ -int h2_mplx_has_master_events(h2_mplx *m) +int h2_mplx_m_has_master_events(h2_mplx *m) { return apr_atomic_read32(&m->event_pending) > 0; } -apr_status_t h2_mplx_dispatch_master_events(h2_mplx *m, - stream_ev_callback *on_resume, - void *on_ctx) +apr_status_t h2_mplx_m_dispatch_master_events(h2_mplx *m, stream_ev_callback *on_resume, + void *on_ctx) { h2_stream *stream; int n, id; @@ -1247,8 +1111,8 @@ apr_atomic_set32(&m->event_pending, 0); /* update input windows for streams */ - h2_ihash_iter(m->streams, report_consumption_iter, m); - purge_streams(m, 1); + h2_ihash_iter(m->streams, m_report_consumption_iter, m); + m_purge_streams(m, 1); n = h2_ififo_count(m->readyq); while (n > 0 @@ -1263,13 +1127,13 @@ return APR_SUCCESS; } -apr_status_t h2_mplx_keep_active(h2_mplx *m, h2_stream *stream) +apr_status_t h2_mplx_m_keep_active(h2_mplx *m, h2_stream *stream) { - check_data_for(m, stream, 1); + mst_check_data_for(m, stream, 0); return APR_SUCCESS; } -int h2_mplx_awaits_data(h2_mplx *m) +int h2_mplx_m_awaits_data(h2_mplx *m) { int waiting = 1; @@ -1278,11 +1142,24 @@ if (h2_ihash_empty(m->streams)) { waiting = 0; } - else if (!m->tasks_active && !h2_ififo_count(m->readyq) - && h2_iq_empty(m->q)) { + else if (!m->tasks_active && !h2_ififo_count(m->readyq) && h2_iq_empty(m->q)) { waiting = 0; } H2_MPLX_LEAVE(m); return waiting; } + +apr_status_t h2_mplx_m_client_rst(h2_mplx *m, int stream_id) +{ + h2_stream *stream; + apr_status_t status = APR_SUCCESS; + + H2_MPLX_ENTER_ALWAYS(m); + stream = h2_ihash_get(m->streams, stream_id); + if (stream && stream->task) { + status = m_be_annoyed(m); + } + H2_MPLX_LEAVE(m); + return status; +} --- a/modules/http2/h2_mplx.h +++ b/modules/http2/h2_mplx.h @@ -31,8 +31,10 @@ * queued in the multiplexer. If a task thread tries to write more * data, it is blocked until space becomes available. * - * Writing input is never blocked. In order to use flow control on the input, - * the mplx can be polled for input data consumption. + * Naming Convention: + * "h2_mplx_m_" are methods only to be called by the main connection + * "h2_mplx_s_" are method only to be called by a secondary connection + * "h2_mplx_t_" are method only to be called by a task handler (can be master or secondary) */ struct apr_pool_t; @@ -47,8 +49,6 @@ struct apr_thread_cond_t; struct h2_workers; struct h2_iqueue; -struct h2_ngn_shed; -struct h2_req_engine; #include @@ -65,7 +65,6 @@ unsigned int is_registered; /* is registered at h2_workers */ struct h2_ihash_t *streams; /* all streams currently processing */ - struct h2_ihash_t *sredo; /* all streams that need to be re-started */ struct h2_ihash_t *shold; /* all streams done with task ongoing */ struct h2_ihash_t *spurge; /* all streams done, ready for destroy */ @@ -79,41 +78,35 @@ int tasks_active; /* # of tasks being processed from this mplx */ int limit_active; /* current limit on active tasks, dynamic */ int max_active; /* max, hard limit # of active tasks in a process */ - apr_time_t last_idle_block; /* last time, this mplx entered IDLE while - * streams were ready */ - apr_time_t last_limit_change; /* last time, worker limit changed */ - apr_interval_time_t limit_change_interval; + + apr_time_t last_mood_change; /* last time, we worker limit changed */ + apr_interval_time_t mood_update_interval; /* how frequent we update at most */ + int irritations_since; /* irritations (>0) or happy events (<0) since last mood change */ apr_thread_mutex_t *lock; struct apr_thread_cond_t *added_output; - struct apr_thread_cond_t *task_thawed; struct apr_thread_cond_t *join_wait; apr_size_t stream_max_mem; apr_pool_t *spare_io_pool; - apr_array_header_t *spare_slaves; /* spare slave connections */ + apr_array_header_t *spare_secondary; /* spare secondary connections */ struct h2_workers *workers; - - struct h2_ngn_shed *ngn_shed; }; - - /******************************************************************************* - * Object lifecycle and information. + * From the main connection processing: h2_mplx_m_* ******************************************************************************/ -apr_status_t h2_mplx_child_init(apr_pool_t *pool, server_rec *s); +apr_status_t h2_mplx_m_child_init(apr_pool_t *pool, server_rec *s); /** * Create the multiplexer for the given HTTP2 session. * Implicitly has reference count 1. */ -h2_mplx *h2_mplx_create(conn_rec *c, apr_pool_t *master, - const struct h2_config *conf, - struct h2_workers *workers); +h2_mplx *h2_mplx_m_create(conn_rec *c, server_rec *s, apr_pool_t *master, + struct h2_workers *workers); /** * Decreases the reference counter of this mplx and waits for it @@ -123,26 +116,14 @@ * @param m the mplx to be released and destroyed * @param wait condition var to wait on for ref counter == 0 */ -void h2_mplx_release_and_join(h2_mplx *m, struct apr_thread_cond_t *wait); - -apr_status_t h2_mplx_pop_task(h2_mplx *m, struct h2_task **ptask); - -void h2_mplx_task_done(h2_mplx *m, struct h2_task *task, struct h2_task **ptask); +void h2_mplx_m_release_and_join(h2_mplx *m, struct apr_thread_cond_t *wait); /** * Shut down the multiplexer gracefully. Will no longer schedule new streams * but let the ongoing ones finish normally. * @return the highest stream id being/been processed */ -int h2_mplx_shutdown(h2_mplx *m); - -int h2_mplx_is_busy(h2_mplx *m); - -/******************************************************************************* - * IO lifetime of streams. - ******************************************************************************/ - -struct h2_stream *h2_mplx_stream_get(h2_mplx *m, int id); +int h2_mplx_m_shutdown(h2_mplx *m); /** * Notifies mplx that a stream has been completely handled on the main @@ -151,20 +132,16 @@ * @param m the mplx itself * @param stream the stream ready for cleanup */ -apr_status_t h2_mplx_stream_cleanup(h2_mplx *m, struct h2_stream *stream); +apr_status_t h2_mplx_m_stream_cleanup(h2_mplx *m, struct h2_stream *stream); /** * Waits on output data from any stream in this session to become available. * Returns APR_TIMEUP if no data arrived in the given time. */ -apr_status_t h2_mplx_out_trywait(h2_mplx *m, apr_interval_time_t timeout, - struct apr_thread_cond_t *iowait); +apr_status_t h2_mplx_m_out_trywait(h2_mplx *m, apr_interval_time_t timeout, + struct apr_thread_cond_t *iowait); -apr_status_t h2_mplx_keep_active(h2_mplx *m, struct h2_stream *stream); - -/******************************************************************************* - * Stream processing. - ******************************************************************************/ +apr_status_t h2_mplx_m_keep_active(h2_mplx *m, struct h2_stream *stream); /** * Process a stream request. @@ -175,8 +152,8 @@ * @param cmp the stream priority compare function * @param ctx context data for the compare function */ -apr_status_t h2_mplx_process(h2_mplx *m, struct h2_stream *stream, - h2_stream_pri_cmp *cmp, void *ctx); +apr_status_t h2_mplx_m_process(h2_mplx *m, struct h2_stream *stream, + h2_stream_pri_cmp *cmp, void *ctx); /** * Stream priorities have changed, reschedule pending requests. @@ -185,7 +162,7 @@ * @param cmp the stream priority compare function * @param ctx context data for the compare function */ -apr_status_t h2_mplx_reprioritize(h2_mplx *m, h2_stream_pri_cmp *cmp, void *ctx); +apr_status_t h2_mplx_m_reprioritize(h2_mplx *m, h2_stream_pri_cmp *cmp, void *ctx); typedef apr_status_t stream_ev_callback(void *ctx, struct h2_stream *stream); @@ -193,7 +170,7 @@ * Check if the multiplexer has events for the master connection pending. * @return != 0 iff there are events pending */ -int h2_mplx_has_master_events(h2_mplx *m); +int h2_mplx_m_has_master_events(h2_mplx *m); /** * Dispatch events for the master connection, such as @@ -201,130 +178,46 @@ * @param on_resume new output data has arrived for a suspended stream * @param ctx user supplied argument to invocation. */ -apr_status_t h2_mplx_dispatch_master_events(h2_mplx *m, - stream_ev_callback *on_resume, - void *ctx); +apr_status_t h2_mplx_m_dispatch_master_events(h2_mplx *m, stream_ev_callback *on_resume, + void *ctx); -int h2_mplx_awaits_data(h2_mplx *m); +int h2_mplx_m_awaits_data(h2_mplx *m); typedef int h2_mplx_stream_cb(struct h2_stream *s, void *ctx); -apr_status_t h2_mplx_stream_do(h2_mplx *m, h2_mplx_stream_cb *cb, void *ctx); +apr_status_t h2_mplx_m_stream_do(h2_mplx *m, h2_mplx_stream_cb *cb, void *ctx); -/******************************************************************************* - * Output handling of streams. - ******************************************************************************/ +apr_status_t h2_mplx_m_client_rst(h2_mplx *m, int stream_id); /** - * Opens the output for the given stream with the specified response. + * Master connection has entered idle mode. + * @param m the mplx instance of the master connection + * @return != SUCCESS iff connection should be terminated */ -apr_status_t h2_mplx_out_open(h2_mplx *mplx, int stream_id, - struct h2_bucket_beam *beam); +apr_status_t h2_mplx_m_idle(h2_mplx *m); /******************************************************************************* - * h2_mplx list Manipulation. + * From a secondary connection processing: h2_mplx_s_* ******************************************************************************/ - -/** - * The magic pointer value that indicates the head of a h2_mplx list - * @param b The mplx list - * @return The magic pointer value - */ -#define H2_MPLX_LIST_SENTINEL(b) APR_RING_SENTINEL((b), h2_mplx, link) - -/** - * Determine if the mplx list is empty - * @param b The list to check - * @return true or false - */ -#define H2_MPLX_LIST_EMPTY(b) APR_RING_EMPTY((b), h2_mplx, link) - -/** - * Return the first mplx in a list - * @param b The list to query - * @return The first mplx in the list - */ -#define H2_MPLX_LIST_FIRST(b) APR_RING_FIRST(b) - -/** - * Return the last mplx in a list - * @param b The list to query - * @return The last mplx int he list - */ -#define H2_MPLX_LIST_LAST(b) APR_RING_LAST(b) - -/** - * Insert a single mplx at the front of a list - * @param b The list to add to - * @param e The mplx to insert - */ -#define H2_MPLX_LIST_INSERT_HEAD(b, e) do { \ -h2_mplx *ap__b = (e); \ -APR_RING_INSERT_HEAD((b), ap__b, h2_mplx, link); \ -} while (0) - -/** - * Insert a single mplx at the end of a list - * @param b The list to add to - * @param e The mplx to insert - */ -#define H2_MPLX_LIST_INSERT_TAIL(b, e) do { \ -h2_mplx *ap__b = (e); \ -APR_RING_INSERT_TAIL((b), ap__b, h2_mplx, link); \ -} while (0) - -/** - * Get the next mplx in the list - * @param e The current mplx - * @return The next mplx - */ -#define H2_MPLX_NEXT(e) APR_RING_NEXT((e), link) -/** - * Get the previous mplx in the list - * @param e The current mplx - * @return The previous mplx - */ -#define H2_MPLX_PREV(e) APR_RING_PREV((e), link) - -/** - * Remove a mplx from its list - * @param e The mplx to remove - */ -#define H2_MPLX_REMOVE(e) APR_RING_REMOVE((e), link) +apr_status_t h2_mplx_s_pop_task(h2_mplx *m, struct h2_task **ptask); +void h2_mplx_s_task_done(h2_mplx *m, struct h2_task *task, struct h2_task **ptask); /******************************************************************************* - * h2_mplx DoS protection + * From a h2_task owner: h2_mplx_s_* + * (a task is transfered from master to secondary connection and back in + * its normal lifetime). ******************************************************************************/ /** - * Master connection has entered idle mode. - * @param m the mplx instance of the master connection - * @return != SUCCESS iff connection should be terminated + * Opens the output for the given stream with the specified response. */ -apr_status_t h2_mplx_idle(h2_mplx *m); +apr_status_t h2_mplx_t_out_open(h2_mplx *mplx, int stream_id, + struct h2_bucket_beam *beam); -/******************************************************************************* - * h2_req_engine handling - ******************************************************************************/ +/** + * Get the stream that belongs to the given task. + */ +struct h2_stream *h2_mplx_t_stream_get(h2_mplx *m, struct h2_task *task); -typedef void h2_output_consumed(void *ctx, conn_rec *c, apr_off_t consumed); -typedef apr_status_t h2_mplx_req_engine_init(struct h2_req_engine *engine, - const char *id, - const char *type, - apr_pool_t *pool, - apr_size_t req_buffer_size, - request_rec *r, - h2_output_consumed **pconsumed, - void **pbaton); - -apr_status_t h2_mplx_req_engine_push(const char *ngn_type, - request_rec *r, - h2_mplx_req_engine_init *einit); -apr_status_t h2_mplx_req_engine_pull(struct h2_req_engine *ngn, - apr_read_type_e block, - int capacity, - request_rec **pr); -void h2_mplx_req_engine_done(struct h2_req_engine *ngn, conn_rec *r_conn, - apr_status_t status); #endif /* defined(__mod_h2__h2_mplx__) */ --- a/modules/http2/h2_proxy_session.c +++ b/modules/http2/h2_proxy_session.c @@ -45,6 +45,7 @@ unsigned int suspended : 1; unsigned int waiting_on_100 : 1; unsigned int waiting_on_ping : 1; + unsigned int headers_ended : 1; uint32_t error_code; apr_bucket_brigade *input; @@ -61,7 +62,123 @@ static void ping_arrived(h2_proxy_session *session); static apr_status_t check_suspended(h2_proxy_session *session); static void stream_resume(h2_proxy_stream *stream); +static apr_status_t submit_trailers(h2_proxy_stream *stream); +/* + * The H2_PING connection sub-state: a state independant of the H2_SESSION state + * of the connection: + * - H2_PING_ST_NONE: no interference with request handling, ProxyTimeout in effect. + * When entered, all suspended streams are unsuspended again. + * - H2_PING_ST_AWAIT_ANY: new requests are suspended, a possibly configured "ping" + * timeout is in effect. Any frame received transits to H2_PING_ST_NONE. + * - H2_PING_ST_AWAIT_PING: same as above, but only a PING frame transits + * to H2_PING_ST_NONE. + * + * An AWAIT state is entered on a new connection or when re-using a connection and + * the last frame received has been some time ago. The latter sends a PING frame + * and insists on an answer, the former is satisfied by any frame received from the + * backend. + * + * This works for new connections as there is always at least one SETTINGS frame + * that the backend sends. When re-using connection, we send a PING and insist on + * receiving one back, as there might be frames in our connection buffers from + * some time ago. Since some servers have protections against PING flooding, we + * only ever have one PING unanswered. + * + * Requests are suspended while in a PING state, as we do not want to send data + * before we can be reasonably sure that the connection is working (at least on + * the h2 protocol level). This also means that the session can do blocking reads + * when expecting PING answers. + */ +static void set_ping_timeout(h2_proxy_session *session) +{ + if (session->ping_timeout != -1 && session->save_timeout == -1) { + apr_socket_t *socket = NULL; + + socket = ap_get_conn_socket(session->c); + if (socket) { + apr_socket_timeout_get(socket, &session->save_timeout); + apr_socket_timeout_set(socket, session->ping_timeout); + } + } +} + +static void unset_ping_timeout(h2_proxy_session *session) +{ + if (session->save_timeout != -1) { + apr_socket_t *socket = NULL; + + socket = ap_get_conn_socket(session->c); + if (socket) { + apr_socket_timeout_set(socket, session->save_timeout); + session->save_timeout = -1; + } + } +} + +static void enter_ping_state(h2_proxy_session *session, h2_ping_state_t state) +{ + if (session->ping_state == state) return; + switch (session->ping_state) { + case H2_PING_ST_NONE: + /* leaving NONE, enforce timeout, send frame maybe */ + if (H2_PING_ST_AWAIT_PING == state) { + unset_ping_timeout(session); + nghttp2_submit_ping(session->ngh2, 0, (const uint8_t *)"nevergonnagiveyouup"); + } + set_ping_timeout(session); + session->ping_state = state; + break; + default: + /* no switching between the != NONE states */ + if (H2_PING_ST_NONE == state) { + session->ping_state = state; + unset_ping_timeout(session); + ping_arrived(session); + } + break; + } +} + +static void ping_new_session(h2_proxy_session *session, proxy_conn_rec *p_conn) +{ + session->save_timeout = -1; + session->ping_timeout = (p_conn->worker->s->ping_timeout_set? + p_conn->worker->s->ping_timeout : -1); + session->ping_state = H2_PING_ST_NONE; + enter_ping_state(session, H2_PING_ST_AWAIT_ANY); +} + +static void ping_reuse_session(h2_proxy_session *session) +{ + if (H2_PING_ST_NONE == session->ping_state) { + apr_interval_time_t age = apr_time_now() - session->last_frame_received; + if (age > apr_time_from_sec(1)) { + enter_ping_state(session, H2_PING_ST_AWAIT_PING); + } + } +} + +static void ping_ev_frame_received(h2_proxy_session *session, const nghttp2_frame *frame) +{ + session->last_frame_received = apr_time_now(); + switch (session->ping_state) { + case H2_PING_ST_NONE: + /* nop */ + break; + case H2_PING_ST_AWAIT_ANY: + enter_ping_state(session, H2_PING_ST_NONE); + break; + case H2_PING_ST_AWAIT_PING: + if (NGHTTP2_PING == frame->hd.type) { + enter_ping_state(session, H2_PING_ST_NONE); + } + /* we may receive many other frames while we are waiting for the + * PING answer. They may come all from our connection buffers and + * say nothing about the current state of the backend. */ + break; + } +} static apr_status_t proxy_session_pre_close(void *theconn) { @@ -152,7 +269,8 @@ session->id, buffer); } - session->last_frame_received = apr_time_now(); + ping_ev_frame_received(session, frame); + /* Action for frame types: */ switch (frame->hd.type) { case NGHTTP2_HEADERS: stream = nghttp2_session_get_stream_user_data(ngh2, frame->hd.stream_id); @@ -193,10 +311,6 @@ stream_resume(stream); break; case NGHTTP2_PING: - if (session->check_ping) { - session->check_ping = 0; - ping_arrived(session); - } break; case NGHTTP2_PUSH_PROMISE: break; @@ -241,7 +355,8 @@ return 1; } -static void process_proxy_header(h2_proxy_stream *stream, const char *n, const char *v) +static void process_proxy_header(apr_table_t *headers, h2_proxy_stream *stream, + const char *n, const char *v) { static const struct { const char *name; @@ -262,20 +377,18 @@ if (!dconf->preserve_host) { for (i = 0; transform_hdrs[i].name; ++i) { if (!ap_cstr_casecmp(transform_hdrs[i].name, n)) { - apr_table_add(r->headers_out, n, - (*transform_hdrs[i].func)(r, dconf, v)); + apr_table_add(headers, n, (*transform_hdrs[i].func)(r, dconf, v)); return; } } if (!ap_cstr_casecmp("Link", n)) { dconf = ap_get_module_config(r->per_dir_config, &proxy_module); - apr_table_add(r->headers_out, n, - h2_proxy_link_reverse_map(r, dconf, - stream->real_server_uri, stream->p_server_uri, v)); + apr_table_add(headers, n, h2_proxy_link_reverse_map(r, dconf, + stream->real_server_uri, stream->p_server_uri, v)); return; } } - apr_table_add(r->headers_out, n, v); + apr_table_add(headers, n, v); } static apr_status_t h2_proxy_stream_add_header_out(h2_proxy_stream *stream, @@ -299,8 +412,13 @@ return APR_SUCCESS; } + ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, stream->session->c, + "h2_proxy_stream(%s-%d): on_header %s: %s", + stream->session->id, stream->id, n, v); if (!h2_proxy_res_ignore_header(n, nlen)) { char *hname, *hvalue; + apr_table_t *headers = (stream->headers_ended? + stream->r->trailers_out : stream->r->headers_out); hname = apr_pstrndup(stream->pool, n, nlen); h2_proxy_util_camel_case_header(hname, nlen); @@ -309,7 +427,7 @@ ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, stream->session->c, "h2_proxy_stream(%s-%d): got header %s: %s", stream->session->id, stream->id, hname, hvalue); - process_proxy_header(stream, hname, hvalue); + process_proxy_header(headers, stream, hname, hvalue); } return APR_SUCCESS; } @@ -374,6 +492,7 @@ server_name, portstr) ); } + if (r->status >= 200) stream->headers_ended = 1; if (APLOGrtrace2(stream->r)) { ap_log_rerror(APLOG_MARK, APLOG_TRACE2, 0, stream->r, @@ -429,12 +548,6 @@ stream_id, NGHTTP2_STREAM_CLOSED); return NGHTTP2_ERR_STREAM_CLOSING; } - if (stream->standalone) { - nghttp2_session_consume(ngh2, stream_id, len); - ap_log_rerror(APLOG_MARK, APLOG_TRACE2, 0, stream->r, - "h2_proxy_session(%s): stream %d, win_update %d bytes", - session->id, stream_id, (int)len); - } return 0; } @@ -493,12 +606,12 @@ stream = nghttp2_session_get_stream_user_data(ngh2, stream_id); if (!stream) { ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf, APLOGNO(03361) - "h2_proxy_stream(%s): data_read, stream %d not found", - stream->session->id, stream_id); + "h2_proxy_stream(NULL): data_read, stream %d not found", + stream_id); return NGHTTP2_ERR_CALLBACK_FAILURE; } - if (stream->session->check_ping) { + if (stream->session->ping_state != H2_PING_ST_NONE) { /* suspend until we hear from the other side */ stream->waiting_on_ping = 1; status = APR_EAGAIN; @@ -553,9 +666,14 @@ stream->data_sent += readlen; ap_log_rerror(APLOG_MARK, APLOG_DEBUG, status, stream->r, APLOGNO(03468) "h2_proxy_stream(%d): request DATA %ld, %ld" - " total, flags=%d", - stream->id, (long)readlen, (long)stream->data_sent, + " total, flags=%d", stream->id, (long)readlen, (long)stream->data_sent, (int)*data_flags); + if ((*data_flags & NGHTTP2_DATA_FLAG_EOF) && !apr_is_empty_table(stream->r->trailers_in)) { + ap_log_rerror(APLOG_MARK, APLOG_DEBUG, status, stream->r, APLOGNO(10179) + "h2_proxy_stream(%d): submit trailers", stream->id); + *data_flags |= NGHTTP2_DATA_FLAG_NO_END_STREAM; + submit_trailers(stream); + } return readlen; } else if (APR_STATUS_IS_EAGAIN(status)) { @@ -641,23 +759,20 @@ nghttp2_option_new(&option); nghttp2_option_set_peer_max_concurrent_streams(option, 100); - nghttp2_option_set_no_auto_window_update(option, 1); + nghttp2_option_set_no_auto_window_update(option, 0); nghttp2_session_client_new2(&session->ngh2, cbs, session, option); nghttp2_option_del(option); nghttp2_session_callbacks_del(cbs); + ping_new_session(session, p_conn); ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c, APLOGNO(03362) "setup session for %s", p_conn->hostname); } else { h2_proxy_session *session = p_conn->data; - apr_interval_time_t age = apr_time_now() - session->last_frame_received; - if (age > apr_time_from_sec(1)) { - session->check_ping = 1; - nghttp2_submit_ping(session->ngh2, 0, (const uint8_t *)"nevergonnagiveyouup"); - } + ping_reuse_session(session); } return p_conn->data; } @@ -740,6 +855,8 @@ stream->real_server_uri = apr_psprintf(stream->pool, "%s://%s", scheme, authority); stream->p_server_uri = apr_psprintf(stream->pool, "%s://%s", puri.scheme, authority); path = apr_uri_unparse(stream->pool, &puri, APR_URI_UNP_OMITSITEPART); + + h2_proxy_req_make(stream->req, stream->pool, r->method, scheme, authority, path, r->headers_in); @@ -826,6 +943,16 @@ return APR_EGENERAL; } +static apr_status_t submit_trailers(h2_proxy_stream *stream) +{ + h2_proxy_ngheader *hd; + int rv; + + hd = h2_proxy_util_nghd_make(stream->pool, stream->r->trailers_in); + rv = nghttp2_submit_trailer(stream->session->ngh2, stream->id, hd->nv, hd->nvlen); + return rv == 0? APR_SUCCESS: APR_EGENERAL; +} + static apr_status_t feed_brigade(h2_proxy_session *session, apr_bucket_brigade *bb) { apr_status_t status = APR_SUCCESS; @@ -882,7 +1009,7 @@ apr_socket_t *socket = NULL; apr_time_t save_timeout = -1; - if (block) { + if (block && timeout > 0) { socket = ap_get_conn_socket(session->c); if (socket) { apr_socket_timeout_get(socket, &save_timeout); @@ -954,6 +1081,14 @@ dispatch_event(session, H2_PROXYS_EV_STREAM_RESUMED, 0, NULL); } +static int is_waiting_for_backend(h2_proxy_session *session) +{ + return ((session->ping_state != H2_PING_ST_NONE) + || ((session->suspended->nelts <= 0) + && !nghttp2_session_want_write(session->ngh2) + && nghttp2_session_want_read(session->ngh2))); +} + static apr_status_t check_suspended(h2_proxy_session *session) { h2_proxy_stream *stream; @@ -1408,7 +1543,22 @@ break; case H2_PROXYS_ST_WAIT: - if (check_suspended(session) == APR_EAGAIN) { + if (is_waiting_for_backend(session)) { + /* we can do a blocking read with the default timeout (as + * configured via ProxyTimeout in our socket. There is + * nothing we want to send or check until we get more data + * from the backend. */ + status = h2_proxy_session_read(session, 1, 0); + if (status == APR_SUCCESS) { + have_read = 1; + dispatch_event(session, H2_PROXYS_EV_DATA_READ, 0, NULL); + } + else { + dispatch_event(session, H2_PROXYS_EV_CONN_ERROR, status, NULL); + return status; + } + } + else if (check_suspended(session) == APR_EAGAIN) { /* no stream has become resumed. Do a blocking read with * ever increasing timeouts... */ if (session->wait_timeout < 25) { @@ -1423,7 +1573,7 @@ ap_log_cerror(APLOG_MARK, APLOG_TRACE3, status, session->c, APLOGNO(03365) "h2_proxy_session(%s): WAIT read, timeout=%fms", - session->id, (float)session->wait_timeout/1000.0); + session->id, session->wait_timeout/1000.0); if (status == APR_SUCCESS) { have_read = 1; dispatch_event(session, H2_PROXYS_EV_DATA_READ, 0, NULL); @@ -1543,42 +1693,3 @@ int updated; } win_update_ctx; -static int win_update_iter(void *udata, void *val) -{ - win_update_ctx *ctx = udata; - h2_proxy_stream *stream = val; - - if (stream->r && stream->r->connection == ctx->c) { - ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, ctx->session->c, - "h2_proxy_session(%s-%d): win_update %ld bytes", - ctx->session->id, (int)stream->id, (long)ctx->bytes); - nghttp2_session_consume(ctx->session->ngh2, stream->id, ctx->bytes); - ctx->updated = 1; - return 0; - } - return 1; -} - - -void h2_proxy_session_update_window(h2_proxy_session *session, - conn_rec *c, apr_off_t bytes) -{ - if (!h2_proxy_ihash_empty(session->streams)) { - win_update_ctx ctx; - ctx.session = session; - ctx.c = c; - ctx.bytes = bytes; - ctx.updated = 0; - h2_proxy_ihash_iter(session->streams, win_update_iter, &ctx); - - if (!ctx.updated) { - /* could not find the stream any more, possibly closed, update - * the connection window at least */ - ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, session->c, - "h2_proxy_session(%s): win_update conn %ld bytes", - session->id, (long)bytes); - nghttp2_session_consume_connection(session->ngh2, (size_t)bytes); - } - } -} - --- a/modules/http2/h2_proxy_session.h +++ b/modules/http2/h2_proxy_session.h @@ -60,6 +60,11 @@ H2_PROXYS_EV_PRE_CLOSE, /* connection will close after this */ } h2_proxys_event_t; +typedef enum { + H2_PING_ST_NONE, /* normal connection mode, ProxyTimeout rules */ + H2_PING_ST_AWAIT_ANY, /* waiting for any frame from backend */ + H2_PING_ST_AWAIT_PING, /* waiting for PING frame from backend */ +} h2_ping_state_t; typedef struct h2_proxy_session h2_proxy_session; typedef void h2_proxy_request_done(h2_proxy_session *s, request_rec *r, @@ -74,7 +79,6 @@ nghttp2_session *ngh2; /* the nghttp2 session itself */ unsigned int aborted : 1; - unsigned int check_ping : 1; unsigned int h2_front : 1; /* if front-end connection is HTTP/2 */ h2_proxy_request_done *done; @@ -94,6 +98,10 @@ apr_bucket_brigade *input; apr_bucket_brigade *output; + + h2_ping_state_t ping_state; + apr_time_t ping_timeout; + apr_time_t save_timeout; }; h2_proxy_session *h2_proxy_session_setup(const char *id, proxy_conn_rec *p_conn, @@ -120,9 +128,6 @@ void h2_proxy_session_cleanup(h2_proxy_session *s, h2_proxy_request_done *done); -void h2_proxy_session_update_window(h2_proxy_session *s, - conn_rec *c, apr_off_t bytes); - #define H2_PROXY_REQ_URL_NOTE "h2-proxy-req-url" #endif /* h2_proxy_session_h */ --- a/modules/http2/h2_proxy_util.c +++ b/modules/http2/h2_proxy_util.c @@ -452,6 +452,22 @@ return ngh; } +h2_proxy_ngheader *h2_proxy_util_nghd_make(apr_pool_t *p, apr_table_t *headers) +{ + + h2_proxy_ngheader *ngh; + size_t n; + + n = 0; + apr_table_do(count_header, &n, headers, NULL); + + ngh = apr_pcalloc(p, sizeof(h2_proxy_ngheader)); + ngh->nv = apr_pcalloc(p, n * sizeof(nghttp2_nv)); + apr_table_do(add_table_header, ngh, headers, NULL); + + return ngh; +} + /******************************************************************************* * header HTTP/1 <-> HTTP/2 conversions ******************************************************************************/ @@ -609,6 +625,7 @@ apr_table_t *headers) { h1_ctx x; + const char *val; req->method = method; req->scheme = scheme; @@ -623,6 +640,11 @@ x.pool = pool; x.headers = req->headers; apr_table_do(set_h1_header, &x, headers, NULL); + if ((val = apr_table_get(headers, "TE")) && ap_find_token(pool, val, "trailers")) { + /* client accepts trailers, forward this information */ + apr_table_addn(req->headers, "TE", "trailers"); + } + apr_table_setn(req->headers, "te", "trailers"); return APR_SUCCESS; } @@ -915,12 +937,12 @@ nlen = (int)strlen(ns); delta = nlen - olen; plen = ctx->slen + delta + 1; - p = apr_pcalloc(ctx->pool, plen); + p = apr_palloc(ctx->pool, plen); memcpy(p, ctx->s, start); memcpy(p + start, ns, nlen); strcpy(p + start + nlen, ctx->s + end); ctx->s = p; - ctx->slen = (int)strlen(p); + ctx->slen = plen - 1; /* (int)strlen(p) */ if (ctx->i >= end) { ctx->i += delta; } --- a/modules/http2/h2_proxy_util.h +++ b/modules/http2/h2_proxy_util.h @@ -168,6 +168,8 @@ h2_proxy_ngheader *h2_proxy_util_nghd_make_req(apr_pool_t *p, const struct h2_proxy_request *req); +h2_proxy_ngheader *h2_proxy_util_nghd_make(apr_pool_t *p, apr_table_t *headers); + /******************************************************************************* * h2_proxy_request helpers ******************************************************************************/ @@ -183,7 +185,7 @@ apr_time_t request_time; - unsigned int chunked : 1; /* iff requst body needs to be forwarded as chunked */ + unsigned int chunked : 1; /* iff request body needs to be forwarded as chunked */ unsigned int serialize : 1; /* iff this request is written in HTTP/1.1 serialization */ }; --- a/modules/http2/h2_push.c +++ b/modules/http2/h2_push.c @@ -464,33 +464,6 @@ return NULL; } -/******************************************************************************* - * push diary - * - * - The push diary keeps track of resources already PUSHed via HTTP/2 on this - * connection. It records a hash value from the absolute URL of the resource - * pushed. - * - Lacking openssl, it uses 'apr_hashfunc_default' for the value - * - with openssl, it uses SHA256 to calculate the hash value - * - whatever the method to generate the hash, the diary keeps a maximum of 64 - * bits per hash, limiting the memory consumption to about - * H2PushDiarySize * 8 - * bytes. Entries are sorted by most recently used and oldest entries are - * forgotten first. - * - Clients can initialize/replace the push diary by sending a 'Cache-Digest' - * header. Currently, this is the base64url encoded value of the cache digest - * as specified in https://datatracker.ietf.org/doc/draft-kazuho-h2-cache-digest/ - * This draft can be expected to evolve and the definition of the header - * will be added there and refined. - * - The cache digest header is a Golomb Coded Set of hash values, but it may - * limit the amount of bits per hash value even further. For a good description - * of GCS, read here: - * http://giovanni.bajo.it/post/47119962313/golomb-coded-sets-smaller-than-bloom-filters - * - The means that the push diary might be initialized with hash values of much - * less than 64 bits, leading to more false positives, but smaller digest size. - ******************************************************************************/ - - #define GCSLOG_LEVEL APLOG_TRACE1 typedef struct h2_push_diary_entry { @@ -617,38 +590,48 @@ return -1; } -static h2_push_diary_entry *move_to_last(h2_push_diary *diary, apr_size_t idx) +static void move_to_last(h2_push_diary *diary, apr_size_t idx) { h2_push_diary_entry *entries = (h2_push_diary_entry*)diary->entries->elts; h2_push_diary_entry e; - apr_size_t lastidx = diary->entries->nelts-1; + int lastidx; + /* Move an existing entry to the last place */ + if (diary->entries->nelts <= 0) + return; + /* move entry[idx] to the end */ + lastidx = diary->entries->nelts - 1; if (idx < lastidx) { e = entries[idx]; - memmove(entries+idx, entries+idx+1, sizeof(e) * (lastidx - idx)); + memmove(entries+idx, entries+idx+1, sizeof(h2_push_diary_entry) * (lastidx - idx)); entries[lastidx] = e; } - return &entries[lastidx]; } -static void h2_push_diary_append(h2_push_diary *diary, h2_push_diary_entry *e) +static void remove_first(h2_push_diary *diary) { - h2_push_diary_entry *ne; + h2_push_diary_entry *entries = (h2_push_diary_entry*)diary->entries->elts; + int lastidx; - if (diary->entries->nelts < diary->N) { - /* append a new diary entry at the end */ - APR_ARRAY_PUSH(diary->entries, h2_push_diary_entry) = *e; - ne = &APR_ARRAY_IDX(diary->entries, diary->entries->nelts-1, h2_push_diary_entry); - } - else { - /* replace content with new digest. keeps memory usage constant once diary is full */ - ne = move_to_last(diary, 0); - *ne = *e; + /* move remaining entries to index 0 */ + lastidx = diary->entries->nelts - 1; + if (lastidx > 0) { + --diary->entries->nelts; + memmove(entries, entries+1, sizeof(h2_push_diary_entry) * diary->entries->nelts); } +} + +static void h2_push_diary_append(h2_push_diary *diary, h2_push_diary_entry *e) +{ + while (diary->entries->nelts >= diary->N) { + remove_first(diary); + } + /* append a new diary entry at the end */ + APR_ARRAY_PUSH(diary->entries, h2_push_diary_entry) = *e; /* Intentional no APLOGNO */ ap_log_perror(APLOG_MARK, GCSLOG_LEVEL, 0, diary->entries->pool, - "push_diary_append: %"APR_UINT64_T_HEX_FMT, ne->hash); + "push_diary_append: %"APR_UINT64_T_HEX_FMT, e->hash); } apr_array_header_t *h2_push_diary_update(h2_session *session, apr_array_header_t *pushes) @@ -691,30 +674,12 @@ const struct h2_request *req, const struct h2_headers *res) { - h2_session *session = stream->session; - const char *cache_digest = apr_table_get(req->headers, "Cache-Digest"); apr_array_header_t *pushes; - apr_status_t status; - if (cache_digest && session->push_diary) { - status = h2_push_diary_digest64_set(session->push_diary, req->authority, - cache_digest, stream->pool); - if (status != APR_SUCCESS) { - ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, session->c, - H2_SSSN_LOG(APLOGNO(03057), session, - "push diary set from Cache-Digest: %s"), cache_digest); - } - } pushes = h2_push_collect(stream->pool, req, stream->push_policy, res); return h2_push_diary_update(stream->session, pushes); } -static apr_int32_t h2_log2inv(unsigned char log2) -{ - return log2? (1 << log2) : 1; -} - - typedef struct { h2_push_diary *diary; unsigned char log2p; @@ -829,16 +794,11 @@ apr_size_t hash_count; nelts = diary->entries->nelts; - - if (nelts > APR_UINT32_MAX) { - /* should not happen */ - return APR_ENOTIMPL; - } N = ceil_power_of_2(nelts); log2n = h2_log2(N); /* Now log2p is the max number of relevant bits, so that - * log2p + log2n == mask_bits. We can uise a lower log2p + * log2p + log2n == mask_bits. We can use a lower log2p * and have a shorter set encoding... */ log2pmax = h2_log2(ceil_power_of_2(maxP)); @@ -895,166 +855,3 @@ return APR_SUCCESS; } -typedef struct { - h2_push_diary *diary; - apr_pool_t *pool; - unsigned char log2p; - const unsigned char *data; - apr_size_t datalen; - apr_size_t offset; - unsigned int bit; - apr_uint64_t last_val; -} gset_decoder; - -static int gset_decode_next_bit(gset_decoder *decoder) -{ - if (++decoder->bit >= 8) { - if (++decoder->offset >= decoder->datalen) { - return -1; - } - decoder->bit = 0; - } - return (decoder->data[decoder->offset] & cbit_mask[decoder->bit])? 1 : 0; -} - -static apr_status_t gset_decode_next(gset_decoder *decoder, apr_uint64_t *phash) -{ - apr_uint64_t flex = 0, fixed = 0, delta; - int i; - - /* read 1 bits until we encounter 0, then read log2n(diary-P) bits. - * On a malformed bit-string, this will not fail, but produce results - * which are pbly too large. Luckily, the diary will modulo the hash. - */ - while (1) { - int bit = gset_decode_next_bit(decoder); - if (bit == -1) { - return APR_EINVAL; - } - if (!bit) { - break; - } - ++flex; - } - - for (i = 0; i < decoder->log2p; ++i) { - int bit = gset_decode_next_bit(decoder); - if (bit == -1) { - return APR_EINVAL; - } - fixed = (fixed << 1) | bit; - } - - delta = (flex << decoder->log2p) | fixed; - *phash = delta + decoder->last_val; - decoder->last_val = *phash; - - /* Intentional no APLOGNO */ - ap_log_perror(APLOG_MARK, GCSLOG_LEVEL, 0, decoder->pool, - "h2_push_diary_digest_dec: val=%"APR_UINT64_T_HEX_FMT", delta=%" - APR_UINT64_T_HEX_FMT", flex=%d, fixed=%"APR_UINT64_T_HEX_FMT, - *phash, delta, (int)flex, fixed); - - return APR_SUCCESS; -} - -/** - * Initialize the push diary by a cache digest as described in - * https://datatracker.ietf.org/doc/draft-kazuho-h2-cache-digest/ - * . - * @param diary the diary to set the digest into - * @param data the binary cache digest - * @param len the length of the cache digest - * @return APR_EINVAL if digest was not successfully parsed - */ -apr_status_t h2_push_diary_digest_set(h2_push_diary *diary, const char *authority, - const char *data, apr_size_t len) -{ - gset_decoder decoder; - unsigned char log2n, log2p; - int N, i; - apr_pool_t *pool = diary->entries->pool; - h2_push_diary_entry e; - apr_status_t status = APR_SUCCESS; - - if (len < 2) { - /* at least this should be there */ - return APR_EINVAL; - } - log2n = data[0]; - log2p = data[1]; - diary->mask_bits = log2n + log2p; - if (diary->mask_bits > 64) { - /* cannot handle */ - return APR_ENOTIMPL; - } - - /* whatever is in the digest, it replaces the diary entries */ - apr_array_clear(diary->entries); - if (!authority || !strcmp("*", authority)) { - diary->authority = NULL; - } - else if (!diary->authority || strcmp(diary->authority, authority)) { - diary->authority = apr_pstrdup(diary->entries->pool, authority); - } - - N = h2_log2inv(log2n + log2p); - - decoder.diary = diary; - decoder.pool = pool; - decoder.log2p = log2p; - decoder.data = (const unsigned char*)data; - decoder.datalen = len; - decoder.offset = 1; - decoder.bit = 8; - decoder.last_val = 0; - - diary->N = N; - /* Determine effective N we use for storage */ - if (!N) { - /* a totally empty cache digest. someone tells us that she has no - * entries in the cache at all. Use our own preferences for N+mask - */ - diary->N = diary->NMax; - return APR_SUCCESS; - } - else if (N > diary->NMax) { - /* Store not more than diary is configured to hold. We open us up - * to DOS attacks otherwise. */ - diary->N = diary->NMax; - } - - /* Intentional no APLOGNO */ - ap_log_perror(APLOG_MARK, GCSLOG_LEVEL, 0, pool, - "h2_push_diary_digest_set: N=%d, log2n=%d, " - "diary->mask_bits=%d, dec.log2p=%d", - (int)diary->N, (int)log2n, diary->mask_bits, - (int)decoder.log2p); - - for (i = 0; i < diary->N; ++i) { - if (gset_decode_next(&decoder, &e.hash) != APR_SUCCESS) { - /* the data may have less than N values */ - break; - } - h2_push_diary_append(diary, &e); - } - - /* Intentional no APLOGNO */ - ap_log_perror(APLOG_MARK, GCSLOG_LEVEL, 0, pool, - "h2_push_diary_digest_set: diary now with %d entries, mask_bits=%d", - (int)diary->entries->nelts, diary->mask_bits); - return status; -} - -apr_status_t h2_push_diary_digest64_set(h2_push_diary *diary, const char *authority, - const char *data64url, apr_pool_t *pool) -{ - const char *data; - apr_size_t len = h2_util_base64url_decode(&data, data64url, pool); - /* Intentional no APLOGNO */ - ap_log_perror(APLOG_MARK, GCSLOG_LEVEL, 0, pool, - "h2_push_diary_digest64_set: digest=%s, dlen=%d", - data64url, (int)len); - return h2_push_diary_digest_set(diary, authority, data, len); -} - --- a/modules/http2/h2_push.h +++ b/modules/http2/h2_push.h @@ -35,6 +35,44 @@ H2_PUSH_DIGEST_SHA256 } h2_push_digest_type; +/******************************************************************************* + * push diary + * + * - The push diary keeps track of resources already PUSHed via HTTP/2 on this + * connection. It records a hash value from the absolute URL of the resource + * pushed. + * - Lacking openssl, + * - with openssl, it uses SHA256 to calculate the hash value, otherwise it + * falls back to apr_hashfunc_default() + * - whatever the method to generate the hash, the diary keeps a maximum of 64 + * bits per hash, limiting the memory consumption to about + * H2PushDiarySize * 8 + * bytes. Entries are sorted by most recently used and oldest entries are + * forgotten first. + * - While useful by itself to avoid duplicated PUSHes on the same connection, + * the original idea was that clients provided a 'Cache-Digest' header with + * the values of *their own* cached resources. This was described in + * + * and some subsequent revisions that tweaked values but kept the overall idea. + * - The draft was abandoned by the IETF http-wg, as support from major clients, + * e.g. browsers, was lacking for various reasons. + * - For these reasons, mod_h2 abandoned its support for client supplied values + * but keeps the diary. It seems to provide value for applications using PUSH, + * is configurable in size and defaults to a very moderate amount of memory + * used. + * - The cache digest header is a Golomb Coded Set of hash values, but it may + * limit the amount of bits per hash value even further. For a good description + * of GCS, read here: + * + ******************************************************************************/ + + +/* + * The push diary is based on the abandoned draft + * + * that describes how to use golomb filters. + */ + typedef struct h2_push_diary h2_push_diary; typedef void h2_push_digest_calc(h2_push_diary *diary, apr_uint64_t *phash, h2_push *push); @@ -101,20 +139,4 @@ int maxP, const char *authority, const char **pdata, apr_size_t *plen); -/** - * Initialize the push diary by a cache digest as described in - * https://datatracker.ietf.org/doc/draft-kazuho-h2-cache-digest/ - * . - * @param diary the diary to set the digest into - * @param authority the authority to set the data for - * @param data the binary cache digest - * @param len the length of the cache digest - * @return APR_EINVAL if digest was not successfully parsed - */ -apr_status_t h2_push_diary_digest_set(h2_push_diary *diary, const char *authority, - const char *data, apr_size_t len); - -apr_status_t h2_push_diary_digest64_set(h2_push_diary *diary, const char *authority, - const char *data64url, apr_pool_t *pool); - #endif /* defined(__mod_h2__h2_push__) */ --- a/modules/http2/h2_request.c +++ b/modules/http2/h2_request.c @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -46,9 +47,9 @@ static int set_h1_header(void *ctx, const char *key, const char *value) { h1_ctx *x = ctx; - x->status = h2_req_add_header(x->headers, x->pool, key, strlen(key), - value, strlen(value)); - return (x->status == APR_SUCCESS)? 1 : 0; + int was_added; + h2_req_add_header(x->headers, x->pool, key, strlen(key), value, strlen(value), 0, &was_added); + return 1; } apr_status_t h2_request_rcreate(h2_request **preq, apr_pool_t *pool, @@ -84,8 +85,7 @@ req->path = path; req->headers = apr_table_make(pool, 10); if (r->server) { - req->serialize = h2_config_geti(h2_config_sget(r->server), - H2_CONF_SER_HEADERS); + req->serialize = h2_config_rgeti(r, H2_CONF_SER_HEADERS); } x.pool = pool; @@ -99,10 +99,12 @@ apr_status_t h2_request_add_header(h2_request *req, apr_pool_t *pool, const char *name, size_t nlen, - const char *value, size_t vlen) + const char *value, size_t vlen, + size_t max_field_len, int *pwas_added) { apr_status_t status = APR_SUCCESS; + *pwas_added = 0; if (nlen <= 0) { return status; } @@ -143,8 +145,9 @@ } } else { - /* non-pseudo header, append to work bucket of stream */ - status = h2_req_add_header(req->headers, pool, name, nlen, value, vlen); + /* non-pseudo header, add to table */ + status = h2_req_add_header(req->headers, pool, name, nlen, value, vlen, + max_field_len, pwas_added); } return status; @@ -156,7 +159,7 @@ /* rfc7540, ch. 8.1.2.3: * - if we have :authority, it overrides any Host header - * - :authority MUST be ommited when converting h1->h2, so we + * - :authority MUST be omitted when converting h1->h2, so we * might get a stream without, but then Host needs to be there */ if (!req->authority) { const char *host = apr_table_get(req->headers, "Host"); @@ -206,13 +209,11 @@ return dst; } -request_rec *h2_request_create_rec(const h2_request *req, conn_rec *c) +#if !AP_MODULE_MAGIC_AT_LEAST(20150222, 13) +static request_rec *my_ap_create_request(conn_rec *c) { - int access_status = HTTP_OK; - const char *rpath; apr_pool_t *p; request_rec *r; - const char *s; apr_pool_create(&p, c->pool); apr_pool_tag(p, "request"); @@ -226,8 +227,8 @@ r->ap_auth_type = NULL; r->allowed_methods = ap_make_method_list(p, 2); - - r->headers_in = apr_table_clone(r->pool, req->headers); + + r->headers_in = apr_table_make(r->pool, 5); r->trailers_in = apr_table_make(r->pool, 5); r->subprocess_env = apr_table_make(r->pool, 25); r->headers_out = apr_table_make(r->pool, 12); @@ -262,6 +263,24 @@ r->useragent_addr = c->client_addr; r->useragent_ip = c->client_ip; + return r; +} +#endif + +request_rec *h2_request_create_rec(const h2_request *req, conn_rec *c) +{ + int access_status = HTTP_OK; + const char *rpath; + const char *s; + +#if AP_MODULE_MAGIC_AT_LEAST(20150222, 13) + request_rec *r = ap_create_request(c); +#else + request_rec *r = my_ap_create_request(c); +#endif + + r->headers_in = apr_table_clone(r->pool, req->headers); + ap_run_pre_read_request(r, c); /* Time to populate r with the data we have. */ @@ -272,6 +291,9 @@ if (r->method_number == M_GET && r->method[0] == 'H') { r->header_only = 1; } + r->the_request = apr_psprintf(r->pool, "%s %s HTTP/2.0", + req->method, req->path ? req->path : ""); + r->headers_in = apr_table_clone(r->pool, req->headers); rpath = (req->path ? req->path : ""); ap_parse_uri(r, rpath); @@ -288,7 +310,9 @@ */ r->hostname = NULL; ap_update_vhost_from_headers(r); - + r->protocol = "HTTP/2.0"; + r->proto_num = HTTP_VERSION(2, 0); + /* we may have switched to another server */ r->per_dir_config = r->server->lookup_defaults; @@ -337,3 +361,4 @@ } + --- a/modules/http2/h2_request.h +++ b/modules/http2/h2_request.h @@ -24,7 +24,8 @@ apr_status_t h2_request_add_header(h2_request *req, apr_pool_t *pool, const char *name, size_t nlen, - const char *value, size_t vlen); + const char *value, size_t vlen, + size_t max_field_len, int *pwas_added); apr_status_t h2_request_add_trailer(h2_request *req, apr_pool_t *pool, const char *name, size_t nlen, --- a/modules/http2/h2_session.c +++ b/modules/http2/h2_session.c @@ -106,7 +106,7 @@ static void cleanup_unprocessed_streams(h2_session *session) { - h2_mplx_stream_do(session->mplx, rst_unprocessed_stream, session); + h2_mplx_m_stream_do(session->mplx, rst_unprocessed_stream, session); } static h2_stream *h2_session_open_stream(h2_session *session, int stream_id, @@ -385,14 +385,19 @@ break; case NGHTTP2_RST_STREAM: ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c, APLOGNO(03067) - "h2_stream(%ld-%d): RST_STREAM by client, errror=%d", + "h2_stream(%ld-%d): RST_STREAM by client, error=%d", session->id, (int)frame->hd.stream_id, (int)frame->rst_stream.error_code); stream = h2_session_stream_get(session, frame->hd.stream_id); if (stream && stream->initiated_on) { + /* A stream reset on a request we sent it. Normal, when the + * client does not want it. */ ++session->pushes_reset; } else { + /* A stream reset on a request it sent us. Could happen in a browser + * when the user navigates away or cancels loading - maybe. */ + h2_mplx_m_client_rst(session->mplx, frame->hd.stream_id); ++session->streams_reset; } break; @@ -462,7 +467,7 @@ } static int h2_session_continue_data(h2_session *session) { - if (h2_mplx_has_master_events(session->mplx)) { + if (h2_mplx_m_has_master_events(session->mplx)) { return 0; } if (h2_conn_io_needs_flush(&session->io)) { @@ -495,9 +500,7 @@ return NGHTTP2_ERR_WOULDBLOCK; } - if (frame->data.padlen > H2_MAX_PADLEN) { - return NGHTTP2_ERR_PROTO; - } + ap_assert(frame->data.padlen <= (H2_MAX_PADLEN+1)); padlen = (unsigned char)frame->data.padlen; stream = h2_session_stream_get(session, stream_id); @@ -513,8 +516,9 @@ H2_STRM_MSG(stream, "send_data_cb for %ld bytes"), (long)length); - status = h2_conn_io_write(&session->io, (const char *)framehd, 9); + status = h2_conn_io_write(&session->io, (const char *)framehd, H2_FRAME_HDR_LEN); if (padlen && status == APR_SUCCESS) { + --padlen; status = h2_conn_io_write(&session->io, (const char *)&padlen, 1); } @@ -622,6 +626,39 @@ } #endif +static ssize_t select_padding_cb(nghttp2_session *ngh2, + const nghttp2_frame *frame, + size_t max_payloadlen, void *user_data) +{ + h2_session *session = user_data; + ssize_t frame_len = frame->hd.length + H2_FRAME_HDR_LEN; /* the total length without padding */ + ssize_t padded_len = frame_len; + + /* Determine # of padding bytes to append to frame. Unless session->padding_always + * the number my be capped by the ui.write_size that currently applies. + */ + if (session->padding_max) { + int n = ap_random_pick(0, session->padding_max); + padded_len = H2MIN(max_payloadlen + H2_FRAME_HDR_LEN, frame_len + n); + } + + if (padded_len != frame_len) { + if (!session->padding_always && session->io.write_size + && (padded_len > session->io.write_size) + && (frame_len <= session->io.write_size)) { + padded_len = session->io.write_size; + } + if (APLOGctrace2(session->c)) { + ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, session->c, + "select padding from [%d, %d]: %d (frame length: 0x%04x, write size: %d)", + (int)frame_len, (int)max_payloadlen+H2_FRAME_HDR_LEN, + (int)(padded_len - frame_len), (int)padded_len, (int)session->io.write_size); + } + return padded_len - H2_FRAME_HDR_LEN; + } + return frame->hd.length; +} + #define NGH2_SET_CALLBACK(callbacks, name, fn)\ nghttp2_session_callbacks_set_##name##_callback(callbacks, fn) @@ -647,6 +684,7 @@ #ifdef H2_NG2_INVALID_HEADER_CB NGH2_SET_CALLBACK(*pcb, on_invalid_header, on_invalid_header_cb); #endif + NGH2_SET_CALLBACK(*pcb, select_padding, select_padding_cb); return APR_SUCCESS; } @@ -691,7 +729,7 @@ * Remove all streams greater than this number without submitting * a RST_STREAM frame, since that should be clear from the GOAWAY * we send. */ - session->local.accepted_max = h2_mplx_shutdown(session->mplx); + session->local.accepted_max = h2_mplx_m_shutdown(session->mplx); session->local.error = error; } else { @@ -741,7 +779,7 @@ } transit(session, trigger, H2_SESSION_ST_CLEANUP); - h2_mplx_release_and_join(session->mplx, session->iowait); + h2_mplx_m_release_and_join(session->mplx, session->iowait); session->mplx = NULL; ap_assert(session->ngh2); @@ -757,13 +795,12 @@ { conn_rec *c = data; h2_session *session; - h2_ctx *ctx = h2_ctx_get(c, 0); - if (ctx && (session = h2_ctx_session_get(ctx))) { + if ((session = h2_ctx_get_session(c))) { /* if the session is still there, now is the last chance * to perform cleanup. Normally, cleanup should have happened * earlier in the connection pre_close. Main reason is that - * any ongoing requests on slave connections might still access + * any ongoing requests on secondary connections might still access * data which has, at this time, already been freed. An example * is mod_ssl that uses request hooks. */ ap_log_cerror(APLOG_MARK, APLOG_WARNING, 0, c, @@ -775,11 +812,8 @@ return APR_SUCCESS; } -static apr_status_t h2_session_create_int(h2_session **psession, - conn_rec *c, - request_rec *r, - h2_ctx *ctx, - h2_workers *workers) +apr_status_t h2_session_create(h2_session **psession, conn_rec *c, request_rec *r, + server_rec *s, h2_workers *workers) { nghttp2_session_callbacks *callbacks = NULL; nghttp2_option *options = NULL; @@ -820,19 +854,16 @@ session->id = c->id; session->c = c; session->r = r; - session->s = h2_ctx_server_get(ctx); + session->s = s; session->pool = pool; - session->config = h2_config_sget(session->s); session->workers = workers; session->state = H2_SESSION_ST_INIT; session->local.accepting = 1; session->remote.accepting = 1; - session->max_stream_count = h2_config_geti(session->config, - H2_CONF_MAX_STREAMS); - session->max_stream_mem = h2_config_geti(session->config, - H2_CONF_STREAM_MAX_MEM); + session->max_stream_count = h2_config_sgeti(s, H2_CONF_MAX_STREAMS); + session->max_stream_mem = h2_config_sgeti(s, H2_CONF_STREAM_MAX_MEM); status = apr_thread_cond_create(&session->iowait, session->pool); if (status != APR_SUCCESS) { @@ -862,14 +893,18 @@ session->monitor->on_state_event = on_stream_state_event; session->monitor->on_event = on_stream_event; - session->mplx = h2_mplx_create(c, session->pool, session->config, - workers); + session->mplx = h2_mplx_m_create(c, s, session->pool, workers); /* connection input filter that feeds the session */ session->cin = h2_filter_cin_create(session); ap_add_input_filter("H2_IN", session->cin, r, c); - h2_conn_io_init(&session->io, c, session->config); + h2_conn_io_init(&session->io, c, s); + session->padding_max = h2_config_sgeti(s, H2_CONF_PADDING_BITS); + if (session->padding_max) { + session->padding_max = (0x01 << session->padding_max) - 1; + } + session->padding_always = h2_config_sgeti(s, H2_CONF_PADDING_ALWAYS); session->bbtmp = apr_brigade_create(session->pool, c->bucket_alloc); status = init_callbacks(c, &callbacks); @@ -888,8 +923,7 @@ apr_pool_destroy(pool); return status; } - nghttp2_option_set_peer_max_concurrent_streams( - options, (uint32_t)session->max_stream_count); + nghttp2_option_set_peer_max_concurrent_streams(options, (uint32_t)session->max_stream_count); /* We need to handle window updates ourself, otherwise we * get flooded by nghttp2. */ nghttp2_option_set_no_auto_window_update(options, 1); @@ -907,7 +941,7 @@ return APR_ENOMEM; } - n = h2_config_geti(session->config, H2_CONF_PUSH_DIARY_SIZE); + n = h2_config_sgeti(s, H2_CONF_PUSH_DIARY_SIZE); session->push_diary = h2_push_diary_create(session->pool, n); if (APLOGcdebug(c)) { @@ -924,22 +958,11 @@ (int)session->push_diary->N); } - apr_pool_pre_cleanup_register(pool, c, session_pool_cleanup); + apr_pool_pre_cleanup_register(pool, c, session_pool_cleanup); + return APR_SUCCESS; } -apr_status_t h2_session_create(h2_session **psession, - conn_rec *c, h2_ctx *ctx, h2_workers *workers) -{ - return h2_session_create_int(psession, c, NULL, ctx, workers); -} - -apr_status_t h2_session_rcreate(h2_session **psession, - request_rec *r, h2_ctx *ctx, h2_workers *workers) -{ - return h2_session_create_int(psession, r->connection, r, ctx, workers); -} - static apr_status_t h2_session_start(h2_session *session, int *rv) { apr_status_t status = APR_SUCCESS; @@ -1004,7 +1027,7 @@ settings[slen].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS; settings[slen].value = (uint32_t)session->max_stream_count; ++slen; - win_size = h2_config_geti(session->config, H2_CONF_WIN_SIZE); + win_size = h2_config_sgeti(session->s, H2_CONF_WIN_SIZE); if (win_size != H2_INITIAL_WINDOW_SIZE) { settings[slen].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; settings[slen].value = win_size; @@ -1156,7 +1179,7 @@ stream = h2_session_open_stream(session, nid, is->id); if (!stream) { ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c, - H2_STRM_LOG(APLOGNO(03077), stream, + H2_STRM_LOG(APLOGNO(03077), is, "failed to create stream obj %d"), nid); /* kill the push_promise */ nghttp2_submit_rst_stream(session->ngh2, NGHTTP2_FLAG_NONE, nid, @@ -1262,7 +1285,7 @@ rv = nghttp2_session_change_stream_priority(session->ngh2, stream->id, &ps); ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c, - ""H2_STRM_LOG(APLOGNO(03203), stream, + H2_STRM_LOG(APLOGNO(03203), stream, "PUSH %s, weight=%d, depends=%d, returned=%d"), ptype, ps.weight, ps.stream_id, rv); status = (rv < 0)? APR_EGENERAL : APR_SUCCESS; @@ -1280,7 +1303,7 @@ { /* iff we can and they can and want */ return (session->remote.accepting /* remote GOAWAY received */ - && h2_config_geti(session->config, H2_CONF_PUSH) + && h2_config_sgeti(session->s, H2_CONF_PUSH) && nghttp2_session_get_remote_settings(session->ngh2, NGHTTP2_SETTINGS_ENABLE_PUSH)); } @@ -1324,6 +1347,7 @@ int eos) { apr_status_t status = APR_SUCCESS; + const char *s; int rv = 0; ap_assert(session); @@ -1391,8 +1415,12 @@ && (headers->status < 400) && (headers->status != 304) && h2_session_push_enabled(session)) { - - h2_stream_submit_pushes(stream, headers); + /* PUSH is possible and enabled on server, unless the request + * denies it, submit resources to push */ + s = apr_table_get(headers->notes, H2_PUSH_MODE_NOTE); + if (!s || strcmp(s, "0")) { + h2_stream_submit_pushes(stream, headers); + } } if (!stream->pref_priority) { @@ -1414,7 +1442,7 @@ } if (headers->status == 103 - && !h2_config_geti(session->config, H2_CONF_EARLY_HINTS)) { + && !h2_config_sgeti(session->s, H2_CONF_EARLY_HINTS)) { /* suppress sending this to the client, it might have triggered * pushes and served its purpose nevertheless */ rv = 0; @@ -1524,7 +1552,7 @@ if (stream) { ap_assert(!stream->scheduled); if (h2_stream_prep_processing(stream) == APR_SUCCESS) { - h2_mplx_process(session->mplx, stream, stream_pri_cmp, session); + h2_mplx_m_process(session->mplx, stream, stream_pri_cmp, session); } else { h2_stream_rst(stream, H2_ERR_INTERNAL_ERROR); @@ -1680,7 +1708,7 @@ * that already served requests - not fair. */ session->idle_sync_until = apr_time_now() + apr_time_from_sec(1); s = "timeout"; - timeout = H2MAX(session->s->timeout, session->s->keep_alive_timeout); + timeout = session->s->timeout; update_child_status(session, SERVER_BUSY_READ, "idle"); ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c, H2_SSSN_LOG("", session, "enter idle, timeout = %d sec"), @@ -1688,8 +1716,8 @@ } else if (session->open_streams) { s = "timeout"; - timeout = session->s->keep_alive_timeout; - update_child_status(session, SERVER_BUSY_KEEPALIVE, "idle"); + timeout = session->s->timeout; + update_child_status(session, SERVER_BUSY_READ, "idle"); } else { /* normal keepalive setup */ @@ -1796,7 +1824,7 @@ session->open_streams); h2_conn_io_flush(&session->io); if (session->open_streams > 0) { - if (h2_mplx_awaits_data(session->mplx)) { + if (h2_mplx_m_awaits_data(session->mplx)) { /* waiting for at least one stream to produce data */ transit(session, "no io", H2_SESSION_ST_WAIT); } @@ -1954,7 +1982,8 @@ ev_stream_closed(session, stream); break; case H2_SS_CLEANUP: - h2_mplx_stream_cleanup(session->mplx, stream); + nghttp2_session_set_stream_user_data(session->ngh2, stream->id, NULL); + h2_mplx_m_stream_cleanup(session->mplx, stream); break; default: break; @@ -2044,7 +2073,7 @@ static apr_status_t dispatch_master(h2_session *session) { apr_status_t status; - status = h2_mplx_dispatch_master_events(session->mplx, + status = h2_mplx_m_dispatch_master_events(session->mplx, on_stream_resume, session); if (status == APR_EAGAIN) { ap_log_cerror(APLOG_MARK, APLOG_TRACE3, status, session->c, @@ -2089,7 +2118,7 @@ switch (session->state) { case H2_SESSION_ST_INIT: ap_update_child_status_from_conn(c->sbh, SERVER_BUSY_READ, c); - if (!h2_is_acceptable_connection(c, 1)) { + if (!h2_is_acceptable_connection(c, session->r, 1)) { update_child_status(session, SERVER_BUSY_READ, "inadequate security"); h2_session_shutdown(session, @@ -2112,7 +2141,7 @@ break; case H2_SESSION_ST_IDLE: - if (session->idle_until && (apr_time_now() + session->idle_delay) > session->idle_until) { + if (session->idle_until && (now + session->idle_delay) > session->idle_until) { ap_log_cerror( APLOG_MARK, APLOG_TRACE1, status, c, H2_SSSN_MSG(session, "idle, timeout reached, closing")); if (session->idle_delay) { @@ -2146,6 +2175,14 @@ session->have_read = 1; } else if (APR_STATUS_IS_EAGAIN(status) || APR_STATUS_IS_TIMEUP(status)) { + status = h2_mplx_m_idle(session->mplx); + if (status == APR_EAGAIN) { + break; + } + else if (status != APR_SUCCESS) { + dispatch_event(session, H2_SESSION_EV_CONN_ERROR, + H2_ERR_ENHANCE_YOUR_CALM, "less is more"); + } status = APR_EAGAIN; goto out; } @@ -2168,7 +2205,7 @@ /* We wait in smaller increments, using a 1 second timeout. * That gives us the chance to check for MPMQ_STOPPING often. */ - status = h2_mplx_idle(session->mplx); + status = h2_mplx_m_idle(session->mplx); if (status == APR_EAGAIN) { break; } @@ -2282,7 +2319,7 @@ "h2_session: wait for data, %ld micros", (long)session->wait_us); } - status = h2_mplx_out_trywait(session->mplx, session->wait_us, + status = h2_mplx_m_out_trywait(session->mplx, session->wait_us, session->iowait); if (status == APR_SUCCESS) { session->wait_us = 0; @@ -2319,7 +2356,7 @@ dispatch_event(session, H2_SESSION_EV_NGH2_DONE, 0, NULL); } if (session->reprioritize) { - h2_mplx_reprioritize(session->mplx, stream_pri_cmp, session); + h2_mplx_m_reprioritize(session->mplx, stream_pri_cmp, session); session->reprioritize = 0; } } --- a/modules/http2/h2_session.h +++ b/modules/http2/h2_session.h @@ -80,12 +80,13 @@ request_rec *r; /* the request that started this in case * of 'h2c', NULL otherwise */ server_rec *s; /* server/vhost we're starting on */ - const struct h2_config *config; /* Relevant config for this session */ apr_pool_t *pool; /* pool to use in session */ struct h2_mplx *mplx; /* multiplexer for stream data */ struct h2_workers *workers; /* for executing stream tasks */ struct h2_filter_cin *cin; /* connection input filter context */ h2_conn_io io; /* io on httpd conn filters */ + int padding_max; /* max number of padding bytes */ + int padding_always; /* padding has precedence over I/O optimizations */ struct nghttp2_session *ngh2; /* the nghttp2 session (internal use) */ h2_session_state state; /* state session is in */ @@ -131,7 +132,7 @@ const char *last_status_msg; /* the one already reported */ struct h2_iqueue *in_pending; /* all streams with input pending */ - struct h2_iqueue *in_process; /* all streams ready for processing on slave */ + struct h2_iqueue *in_process; /* all streams ready for processing on a secondary */ } h2_session; @@ -142,27 +143,15 @@ * The session will apply the configured parameter. * @param psession pointer receiving the created session on success or NULL * @param c the connection to work on + * @param r optional request when protocol was upgraded * @param cfg the module config to apply * @param workers the worker pool to use * @return the created session */ apr_status_t h2_session_create(h2_session **psession, - conn_rec *c, struct h2_ctx *ctx, + conn_rec *c, request_rec *r, server_rec *, struct h2_workers *workers); -/** - * Create a new h2_session for the given request. - * The session will apply the configured parameter. - * @param psession pointer receiving the created session on success or NULL - * @param r the request that was upgraded - * @param cfg the module config to apply - * @param workers the worker pool to use - * @return the created session - */ -apr_status_t h2_session_rcreate(h2_session **psession, - request_rec *r, struct h2_ctx *ctx, - struct h2_workers *workers); - void h2_session_event(h2_session *session, h2_session_event_t ev, int err, const char *msg); --- a/modules/http2/h2_stream.c +++ b/modules/http2/h2_stream.c @@ -365,9 +365,8 @@ static void set_policy_for(h2_stream *stream, h2_request *r) { int enabled = h2_session_push_enabled(stream->session); - stream->push_policy = h2_push_policy_determine(r->headers, stream->pool, - enabled); - r->serialize = h2_config_geti(stream->session->config, H2_CONF_SER_HEADERS); + stream->push_policy = h2_push_policy_determine(r->headers, stream->pool, enabled); + r->serialize = h2_config_sgeti(stream->session->s, H2_CONF_SER_HEADERS); } apr_status_t h2_stream_send_frame(h2_stream *stream, int ftype, int flags, size_t frame_len) @@ -398,13 +397,8 @@ /* start pushed stream */ ap_assert(stream->request == NULL); ap_assert(stream->rtmp != NULL); - status = h2_request_end_headers(stream->rtmp, stream->pool, 1, 0); - if (status != APR_SUCCESS) { - return status; - } - set_policy_for(stream, stream->rtmp); - stream->request = stream->rtmp; - stream->rtmp = NULL; + status = h2_stream_end_headers(stream, 1, 0); + if (status != APR_SUCCESS) goto leave; break; default: @@ -416,6 +410,7 @@ if (status == APR_SUCCESS && eos) { status = transit(stream, on_event(stream, H2_SEV_CLOSED_L)); } +leave: return status; } @@ -451,18 +446,13 @@ ap_assert(stream->request == NULL); if (stream->rtmp == NULL) { /* This can only happen, if the stream has received no header - * name/value pairs at all. The lastest nghttp2 version have become + * name/value pairs at all. The latest nghttp2 version have become * pretty good at detecting this early. In any case, we have * to abort the connection here, since this is clearly a protocol error */ return APR_EINVAL; } - status = h2_request_end_headers(stream->rtmp, stream->pool, eos, frame_len); - if (status != APR_SUCCESS) { - return status; - } - set_policy_for(stream, stream->rtmp); - stream->request = stream->rtmp; - stream->rtmp = NULL; + status = h2_stream_end_headers(stream, eos, frame_len); + if (status != APR_SUCCESS) goto leave; } break; @@ -473,6 +463,7 @@ if (status == APR_SUCCESS && eos) { status = transit(stream, on_event(stream, H2_SEV_CLOSED_R)); } +leave: return status; } @@ -663,11 +654,14 @@ static apr_status_t add_trailer(h2_stream *stream, const char *name, size_t nlen, - const char *value, size_t vlen) + const char *value, size_t vlen, + size_t max_field_len, int *pwas_added) { conn_rec *c = stream->session->c; char *hname, *hvalue; + const char *existing; + *pwas_added = 0; if (nlen == 0 || name[0] == ':') { ap_log_cerror(APLOG_MARK, APLOG_DEBUG, APR_EINVAL, c, H2_STRM_LOG(APLOGNO(03060), stream, @@ -681,9 +675,18 @@ stream->trailers = apr_table_make(stream->pool, 5); } hname = apr_pstrndup(stream->pool, name, nlen); - hvalue = apr_pstrndup(stream->pool, value, vlen); h2_util_camel_case_header(hname, nlen); + existing = apr_table_get(stream->trailers, hname); + if (max_field_len + && ((existing? strlen(existing)+2 : 0) + vlen + nlen + 2 > max_field_len)) { + /* "key: (oldval, )?nval" is too long */ + return APR_EINVAL; + } + if (!existing) *pwas_added = 1; + hvalue = apr_pstrndup(stream->pool, value, vlen); apr_table_mergen(stream->trailers, hname, hvalue); + ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, c, + H2_STRM_MSG(stream, "added trailer '%s: %s'"), hname, hvalue); return APR_SUCCESS; } @@ -693,44 +696,31 @@ const char *value, size_t vlen) { h2_session *session = stream->session; - int error = 0; - apr_status_t status; + int error = 0, was_added = 0; + apr_status_t status = APR_SUCCESS; if (stream->has_response) { return APR_EINVAL; } - ++stream->request_headers_added; + if (name[0] == ':') { if ((vlen) > session->s->limit_req_line) { /* pseudo header: approximation of request line size check */ - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c, - H2_STRM_MSG(stream, "pseudo %s too long"), name); + if (!h2_stream_is_ready(stream)) { + ap_log_cerror(APLOG_MARK, APLOG_INFO, 0, session->c, + H2_STRM_LOG(APLOGNO(10178), stream, + "Request pseudo header exceeds " + "LimitRequestFieldSize: %s"), name); + } error = HTTP_REQUEST_URI_TOO_LARGE; + goto cleanup; } } - else if ((nlen + 2 + vlen) > session->s->limit_req_fieldsize) { - /* header too long */ - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c, - H2_STRM_MSG(stream, "header %s too long"), name); - error = HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE; - } - - if (stream->request_headers_added > session->s->limit_req_fields + 4) { - /* too many header lines, include 4 pseudo headers */ - if (stream->request_headers_added - > session->s->limit_req_fields + 4 + 100) { - /* yeah, right */ - h2_stream_rst(stream, H2_ERR_ENHANCE_YOUR_CALM); - return APR_ECONNRESET; - } - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c, - H2_STRM_MSG(stream, "too many header lines")); - error = HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE; - } - if (error) { - set_error_response(stream, error); - return APR_EINVAL; + if (session->s->limit_req_fields > 0 + && stream->request_headers_added > session->s->limit_req_fields) { + /* already over limit, count this attempt, but do not take it in */ + ++stream->request_headers_added; } else if (H2_SS_IDLE == stream->state) { if (!stream->rtmp) { @@ -738,16 +728,55 @@ NULL, NULL, NULL, NULL, NULL, 0); } status = h2_request_add_header(stream->rtmp, stream->pool, - name, nlen, value, vlen); + name, nlen, value, vlen, + session->s->limit_req_fieldsize, &was_added); + if (was_added) ++stream->request_headers_added; } else if (H2_SS_OPEN == stream->state) { - status = add_trailer(stream, name, nlen, value, vlen); + status = add_trailer(stream, name, nlen, value, vlen, + session->s->limit_req_fieldsize, &was_added); + if (was_added) ++stream->request_headers_added; } else { status = APR_EINVAL; + goto cleanup; + } + + if (APR_EINVAL == status) { + /* header too long */ + if (!h2_stream_is_ready(stream)) { + ap_log_cerror(APLOG_MARK, APLOG_INFO, 0, session->c, + H2_STRM_LOG(APLOGNO(10180), stream,"Request header exceeds " + "LimitRequestFieldSize: %.*s"), + (int)H2MIN(nlen, 80), name); + } + error = HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE; + goto cleanup; + } + + if (session->s->limit_req_fields > 0 + && stream->request_headers_added > session->s->limit_req_fields) { + /* too many header lines */ + if (stream->request_headers_added > session->s->limit_req_fields + 100) { + /* yeah, right, this request is way over the limit, say goodbye */ + h2_stream_rst(stream, H2_ERR_ENHANCE_YOUR_CALM); + return APR_ECONNRESET; + } + if (!h2_stream_is_ready(stream)) { + ap_log_cerror(APLOG_MARK, APLOG_INFO, 0, session->c, + H2_STRM_LOG(APLOGNO(10181), stream, "Number of request headers " + "exceeds LimitRequestFields")); + } + error = HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE; + goto cleanup; } - if (status != APR_SUCCESS) { +cleanup: + if (error) { + set_error_response(stream, error); + return APR_EINVAL; + } + else if (status != APR_SUCCESS) { ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c, H2_STRM_MSG(stream, "header %s not accepted"), name); h2_stream_dispatch(stream, H2_SEV_CANCELLED); @@ -755,6 +784,49 @@ return status; } +typedef struct { + apr_size_t maxlen; + const char *failed_key; +} val_len_check_ctx; + +static int table_check_val_len(void *baton, const char *key, const char *value) +{ + val_len_check_ctx *ctx = baton; + + if (strlen(value) <= ctx->maxlen) return 1; + ctx->failed_key = key; + return 0; +} + +apr_status_t h2_stream_end_headers(h2_stream *stream, int eos, size_t raw_bytes) +{ + apr_status_t status; + val_len_check_ctx ctx; + + status = h2_request_end_headers(stream->rtmp, stream->pool, eos, raw_bytes); + if (APR_SUCCESS == status) { + set_policy_for(stream, stream->rtmp); + stream->request = stream->rtmp; + stream->rtmp = NULL; + + ctx.maxlen = stream->session->s->limit_req_fieldsize; + ctx.failed_key = NULL; + apr_table_do(table_check_val_len, &ctx, stream->request->headers, NULL); + if (ctx.failed_key) { + if (!h2_stream_is_ready(stream)) { + ap_log_cerror(APLOG_MARK, APLOG_INFO, 0, stream->session->c, + H2_STRM_LOG(APLOGNO(10230), stream,"Request header exceeds " + "LimitRequestFieldSize: %.*s"), + (int)H2MIN(strlen(ctx.failed_key), 80), ctx.failed_key); + } + set_error_response(stream, HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE); + /* keep on returning APR_SUCCESS, so that we send a HTTP response and + * do not RST the stream. */ + } + } + return status; +} + static apr_bucket *get_first_headers_bucket(apr_bucket_brigade *bb) { if (bb) { @@ -855,7 +927,7 @@ * is requested. But we can reduce the size in case the master * connection operates in smaller chunks. (TSL warmup) */ if (stream->session->io.write_size > 0) { - max_chunk = stream->session->io.write_size - 9; /* header bits */ + max_chunk = stream->session->io.write_size - H2_FRAME_HDR_LEN; } requested = (*plen > 0)? H2MIN(*plen, max_chunk) : max_chunk; @@ -864,7 +936,7 @@ if (status == APR_EAGAIN) { /* TODO: ugly, someone needs to retrieve the response first */ - h2_mplx_keep_active(stream->session->mplx, stream); + h2_mplx_m_keep_active(stream->session->mplx, stream); ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, H2_STRM_MSG(stream, "prep, response eagain")); return status; @@ -987,7 +1059,7 @@ const char *ctype = apr_table_get(response->headers, "content-type"); if (ctype) { /* FIXME: Not good enough, config needs to come from request->server */ - return h2_config_get_priority(stream->session->config, ctype); + return h2_cconfig_get_priority(stream->session->c, ctype); } } return NULL; --- a/modules/http2/h2_stream.h +++ b/modules/http2/h2_stream.h @@ -198,6 +198,10 @@ apr_status_t h2_stream_add_header(h2_stream *stream, const char *name, size_t nlen, const char *value, size_t vlen); + +/* End the construction of request headers */ +apr_status_t h2_stream_end_headers(h2_stream *stream, int eos, size_t raw_bytes); + apr_status_t h2_stream_send_frame(h2_stream *stream, int frame_type, int flags, size_t frame_len); apr_status_t h2_stream_recv_frame(h2_stream *stream, int frame_type, int flags, size_t frame_len); --- a/modules/http2/h2_switch.c +++ b/modules/http2/h2_switch.c @@ -55,7 +55,6 @@ int is_tls = h2_h2_is_tls(c); const char **protos = is_tls? h2_tls_protos : h2_clear_protos; - (void)s; if (!h2_mpm_supported()) { return DECLINED; } @@ -68,7 +67,7 @@ return DECLINED; } - if (!h2_is_acceptable_connection(c, 0)) { + if (!h2_is_acceptable_connection(c, r, 0)) { ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, c, APLOGNO(03084) "protocol propose: connection requirements not met"); return DECLINED; @@ -81,7 +80,7 @@ */ const char *p; - if (!h2_allows_h2_upgrade(c)) { + if (!h2_allows_h2_upgrade(r)) { return DECLINED; } @@ -150,7 +149,7 @@ ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, "switching protocol to '%s'", protocol); h2_ctx_protocol_set(ctx, protocol); - h2_ctx_server_set(ctx, s); + h2_ctx_server_update(ctx, s); if (r != NULL) { apr_status_t status; @@ -160,12 +159,11 @@ * right away. */ ap_remove_input_filter_byhandle(r->input_filters, "http_in"); - ap_remove_input_filter_byhandle(r->input_filters, "reqtimeout"); ap_remove_output_filter_byhandle(r->output_filters, "HTTP_HEADER"); /* Ok, start an h2_conn on this one. */ - h2_ctx_server_set(ctx, r->server); - status = h2_conn_setup(ctx, r->connection, r); + status = h2_conn_setup(c, r, s); + if (status != APR_SUCCESS) { ap_log_rerror(APLOG_MARK, APLOG_DEBUG, status, r, APLOGNO(03088) "session setup"); @@ -173,7 +171,7 @@ return !OK; } - h2_conn_run(ctx, c); + h2_conn_run(c); } return OK; } --- a/modules/http2/h2_task.c +++ b/modules/http2/h2_task.c @@ -86,7 +86,7 @@ task->request->authority, task->request->path); task->output.opened = 1; - return h2_mplx_out_open(task->mplx, task->stream_id, task->output.beam); + return h2_mplx_t_out_open(task->mplx, task->stream_id, task->output.beam); } static apr_status_t send_out(h2_task *task, apr_bucket_brigade* bb, int block) @@ -97,7 +97,7 @@ apr_brigade_length(bb, 0, &written); H2_TASK_OUT_LOG(APLOG_TRACE2, task, bb, "h2_task send_out"); h2_beam_log(task->output.beam, task->c, APLOG_TRACE2, "send_out(before)"); - /* engines send unblocking */ + status = h2_beam_send(task->output.beam, bb, block? APR_BLOCK_READ : APR_NONBLOCK_READ); h2_beam_log(task->output.beam, task->c, APLOG_TRACE2, "send_out(after)"); @@ -126,33 +126,16 @@ * request_rec out filter chain) into the h2_mplx for further sending * on the master connection. */ -static apr_status_t slave_out(h2_task *task, ap_filter_t* f, - apr_bucket_brigade* bb) +static apr_status_t secondary_out(h2_task *task, ap_filter_t* f, + apr_bucket_brigade* bb) { apr_bucket *b; apr_status_t rv = APR_SUCCESS; int flush = 0, blocking; - if (task->frozen) { - h2_util_bb_log(task->c, task->stream_id, APLOG_TRACE2, - "frozen task output write, ignored", bb); - while (!APR_BRIGADE_EMPTY(bb)) { - b = APR_BRIGADE_FIRST(bb); - if (AP_BUCKET_IS_EOR(b)) { - APR_BUCKET_REMOVE(b); - task->eor = b; - } - else { - apr_bucket_delete(b); - } - } - return APR_SUCCESS; - } - send: - /* we send block once we opened the output, so someone is there - * reading it *and* the task is not assigned to a h2_req_engine */ - blocking = (!task->assigned && task->output.opened); + /* we send block once we opened the output, so someone is there reading it */ + blocking = task->output.opened; for (b = APR_BRIGADE_FIRST(bb); b != APR_BRIGADE_SENTINEL(bb); b = APR_BUCKET_NEXT(b)) { @@ -192,7 +175,7 @@ if (APR_SUCCESS == rv) { /* could not write all, buffer the rest */ ap_log_cerror(APLOG_MARK, APLOG_DEBUG, rv, task->c, APLOGNO(03405) - "h2_slave_out(%s): saving brigade", task->id); + "h2_secondary_out(%s): saving brigade", task->id); ap_assert(NULL); rv = ap_save_brigade(f, &task->output.bb, &bb, task->pool); flush = 1; @@ -206,7 +189,7 @@ } out: ap_log_cerror(APLOG_MARK, APLOG_TRACE2, rv, task->c, - "h2_slave_out(%s): slave_out leave", task->id); + "h2_secondary_out(%s): secondary_out leave", task->id); return rv; } @@ -219,14 +202,14 @@ } /******************************************************************************* - * task slave connection filters + * task secondary connection filters ******************************************************************************/ -static apr_status_t h2_filter_slave_in(ap_filter_t* f, - apr_bucket_brigade* bb, - ap_input_mode_t mode, - apr_read_type_e block, - apr_off_t readbytes) +static apr_status_t h2_filter_secondary_in(ap_filter_t* f, + apr_bucket_brigade* bb, + ap_input_mode_t mode, + apr_read_type_e block, + apr_off_t readbytes) { h2_task *task; apr_status_t status = APR_SUCCESS; @@ -236,12 +219,12 @@ apr_size_t rmax = ((readbytes <= APR_SIZE_MAX)? (apr_size_t)readbytes : APR_SIZE_MAX); - task = h2_ctx_cget_task(f->c); + task = h2_ctx_get_task(f->c); ap_assert(task); if (trace1) { ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, f->c, - "h2_slave_in(%s): read, mode=%d, block=%d, readbytes=%ld", + "h2_secondary_in(%s): read, mode=%d, block=%d, readbytes=%ld", task->id, mode, block, (long)readbytes); } @@ -271,7 +254,7 @@ /* Get more input data for our request. */ if (trace1) { ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, f->c, - "h2_slave_in(%s): get more data from mplx, block=%d, " + "h2_secondary_in(%s): get more data from mplx, block=%d, " "readbytes=%ld", task->id, block, (long)readbytes); } if (task->input.beam) { @@ -284,7 +267,7 @@ if (trace1) { ap_log_cerror(APLOG_MARK, APLOG_TRACE2, status, f->c, - "h2_slave_in(%s): read returned", task->id); + "h2_secondary_in(%s): read returned", task->id); } if (APR_STATUS_IS_EAGAIN(status) && (mode == AP_MODE_GETLINE || block == APR_BLOCK_READ)) { @@ -310,11 +293,9 @@ } } - /* Nothing there, no more data to get. Return APR_EAGAIN on - * speculative reads, this is ap_check_pipeline()'s trick to - * see if the connection needs closing. */ + /* Nothing there, no more data to get. Return. */ if (status == APR_EOF && APR_BRIGADE_EMPTY(task->input.bb)) { - return (mode == AP_MODE_SPECULATIVE)? APR_EAGAIN : APR_EOF; + return status; } if (trace1) { @@ -325,7 +306,7 @@ if (APR_BRIGADE_EMPTY(task->input.bb)) { if (trace1) { ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, f->c, - "h2_slave_in(%s): no data", task->id); + "h2_secondary_in(%s): no data", task->id); } return (block == APR_NONBLOCK_READ)? APR_EAGAIN : APR_EOF; } @@ -353,7 +334,7 @@ buffer[len] = 0; if (trace1) { ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, f->c, - "h2_slave_in(%s): getline: %s", + "h2_secondary_in(%s): getline: %s", task->id, buffer); } } @@ -363,7 +344,7 @@ * to support it. Seems to work. */ ap_log_cerror(APLOG_MARK, APLOG_ERR, APR_ENOTIMPL, f->c, APLOGNO(03472) - "h2_slave_in(%s), unsupported READ mode %d", + "h2_secondary_in(%s), unsupported READ mode %d", task->id, mode); status = APR_ENOTIMPL; } @@ -371,19 +352,19 @@ if (trace1) { apr_brigade_length(bb, 0, &bblen); ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, f->c, - "h2_slave_in(%s): %ld data bytes", task->id, (long)bblen); + "h2_secondary_in(%s): %ld data bytes", task->id, (long)bblen); } return status; } -static apr_status_t h2_filter_slave_output(ap_filter_t* filter, - apr_bucket_brigade* brigade) +static apr_status_t h2_filter_secondary_output(ap_filter_t* filter, + apr_bucket_brigade* brigade) { - h2_task *task = h2_ctx_cget_task(filter->c); + h2_task *task = h2_ctx_get_task(filter->c); apr_status_t status; ap_assert(task); - status = slave_out(task, filter, brigade); + status = secondary_out(task, filter, brigade); if (status != APR_SUCCESS) { h2_task_rst(task, H2_ERR_INTERNAL_ERROR); } @@ -392,14 +373,14 @@ static apr_status_t h2_filter_parse_h1(ap_filter_t* f, apr_bucket_brigade* bb) { - h2_task *task = h2_ctx_cget_task(f->c); + h2_task *task = h2_ctx_get_task(f->c); apr_status_t status; ap_assert(task); /* There are cases where we need to parse a serialized http/1.1 * response. One example is a 100-continue answer in serialized mode * or via a mod_proxy setup */ - while (bb && !task->output.sent_response) { + while (bb && !task->c->aborted && !task->output.sent_response) { status = h2_from_h1_parse_response(task, f, bb); ap_log_cerror(APLOG_MARK, APLOG_TRACE2, status, f->c, "h2_task(%s): parsed response", task->id); @@ -425,8 +406,15 @@ || !strcmp("OPTIONS", task->request->method)); } +int h2_task_has_started(h2_task *task) +{ + return task && task->started_at != 0; +} + void h2_task_redo(h2_task *task) { + task->started_at = 0; + task->worker_done = 0; task->rst_error = 0; } @@ -468,9 +456,9 @@ ap_hook_process_connection(h2_task_process_conn, NULL, NULL, APR_HOOK_FIRST); - ap_register_input_filter("H2_SLAVE_IN", h2_filter_slave_in, + ap_register_input_filter("H2_SECONDARY_IN", h2_filter_secondary_in, NULL, AP_FTYPE_NETWORK); - ap_register_output_filter("H2_SLAVE_OUT", h2_filter_slave_output, + ap_register_output_filter("H2_SECONDARY_OUT", h2_filter_secondary_output, NULL, AP_FTYPE_NETWORK); ap_register_output_filter("H2_PARSE_H1", h2_filter_parse_h1, NULL, AP_FTYPE_NETWORK); @@ -502,17 +490,17 @@ ctx = h2_ctx_get(c, 0); (void)arg; - if (h2_ctx_is_task(ctx)) { + if (ctx->task) { ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, c, - "h2_slave(%s), pre_connection, adding filters", c->log_id); - ap_add_input_filter("H2_SLAVE_IN", NULL, NULL, c); + "h2_secondary(%s), pre_connection, adding filters", c->log_id); + ap_add_input_filter("H2_SECONDARY_IN", NULL, NULL, c); ap_add_output_filter("H2_PARSE_H1", NULL, NULL, c); - ap_add_output_filter("H2_SLAVE_OUT", NULL, NULL, c); + ap_add_output_filter("H2_SECONDARY_OUT", NULL, NULL, c); } return OK; } -h2_task *h2_task_create(conn_rec *slave, int stream_id, +h2_task *h2_task_create(conn_rec *secondary, int stream_id, const h2_request *req, h2_mplx *m, h2_bucket_beam *input, apr_interval_time_t timeout, @@ -521,17 +509,18 @@ apr_pool_t *pool; h2_task *task; - ap_assert(slave); + ap_assert(secondary); ap_assert(req); - apr_pool_create(&pool, slave->pool); + apr_pool_create(&pool, secondary->pool); + apr_pool_tag(pool, "h2_task"); task = apr_pcalloc(pool, sizeof(h2_task)); if (task == NULL) { return NULL; } task->id = "000"; task->stream_id = stream_id; - task->c = slave; + task->c = secondary; task->mplx = m; task->pool = pool; task->request = req; @@ -564,41 +553,40 @@ ap_assert(task); c = task->c; task->worker_started = 1; - task->started_at = apr_time_now(); if (c->master) { - /* Each conn_rec->id is supposed to be unique at a point in time. Since + /* See the discussion at + * + * Each conn_rec->id is supposed to be unique at a point in time. Since * some modules (and maybe external code) uses this id as an identifier - * for the request_rec they handle, it needs to be unique for slave + * for the request_rec they handle, it needs to be unique for secondary * connections also. - * The connection id is generated by the MPM and most MPMs use the formula - * id := (child_num * max_threads) + thread_num - * which means that there is a maximum id of about - * idmax := max_child_count * max_threads - * If we assume 2024 child processes with 2048 threads max, we get - * idmax ~= 2024 * 2048 = 2 ** 22 - * On 32 bit systems, we have not much space left, but on 64 bit systems - * (and higher?) we can use the upper 32 bits without fear of collision. - * 32 bits is just what we need, since a connection can only handle so - * many streams. + * + * The MPM module assigns the connection ids and mod_unique_id is using + * that one to generate identifier for requests. While the implementation + * works for HTTP/1.x, the parallel execution of several requests per + * connection will generate duplicate identifiers on load. + * + * The original implementation for secondary connection identifiers used + * to shift the master connection id up and assign the stream id to the + * lower bits. This was cramped on 32 bit systems, but on 64bit there was + * enough space. + * + * As issue 195 showed, mod_unique_id only uses the lower 32 bit of the + * connection id, even on 64bit systems. Therefore collisions in request ids. + * + * The way master connection ids are generated, there is some space "at the + * top" of the lower 32 bits on allmost all systems. If you have a setup + * with 64k threads per child and 255 child processes, you live on the edge. + * + * The new implementation shifts 8 bits and XORs in the worker + * id. This will experience collisions with > 256 h2 workers and heavy + * load still. There seems to be no way to solve this in all possible + * configurations by mod_h2 alone. */ - int slave_id, free_bits; - + task->c->id = (c->master->id << 8)^worker_id; task->id = apr_psprintf(task->pool, "%ld-%d", c->master->id, task->stream_id); - if (sizeof(unsigned long) >= 8) { - free_bits = 32; - slave_id = task->stream_id; - } - else { - /* Assume we have a more limited number of threads/processes - * and h2 workers on a 32-bit system. Use the worker instead - * of the stream id. */ - free_bits = 8; - slave_id = worker_id; - } - task->c->id = (c->master->id << free_bits)^slave_id; - c->keepalive = AP_CONN_KEEPALIVE; } h2_beam_create(&task->output.beam, c->pool, task->stream_id, "output", @@ -613,7 +601,7 @@ h2_ctx_create_for(c, task); apr_table_setn(c->notes, H2_TASK_ID_NOTE, task->id); - h2_slave_run_pre_connection(c, ap_get_conn_socket(c)); + h2_secondary_run_pre_connection(c, ap_get_conn_socket(c)); task->input.bb = apr_brigade_create(task->pool, c->bucket_alloc); if (task->request->serialize) { @@ -633,18 +621,9 @@ task->c->current_thread = thread; ap_run_process_connection(c); - if (task->frozen) { - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, - "h2_task(%s): process_conn returned frozen task", - task->id); - /* cleanup delayed */ - return APR_EAGAIN; - } - else { - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, - "h2_task(%s): processing done", task->id); - return output_finish(task); - } + ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, + "h2_task(%s): processing done", task->id); + return output_finish(task); } static apr_status_t h2_task_process_request(h2_task *task, conn_rec *c) @@ -682,14 +661,8 @@ ap_process_request(r); - if (task->frozen) { - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, - "h2_task(%s): process_request frozen", task->id); - } - else { - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, - "h2_task(%s): process_request done", task->id); - } + ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, + "h2_task(%s): process_request done", task->id); /* After the call to ap_process_request, the * request pool may have been deleted. We set @@ -724,7 +697,7 @@ } ctx = h2_ctx_get(c, 0); - if (h2_ctx_is_task(ctx)) { + if (ctx->task) { if (!ctx->task->request->serialize) { ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, "h2_h2, processing request directly"); @@ -736,33 +709,8 @@ } else { ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, - "slave_conn(%ld): has no task", c->id); + "secondary_conn(%ld): has no task", c->id); } return DECLINED; } -apr_status_t h2_task_freeze(h2_task *task) -{ - if (!task->frozen) { - task->frozen = 1; - ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, task->c, APLOGNO(03406) - "h2_task(%s), frozen", task->id); - } - return APR_SUCCESS; -} - -apr_status_t h2_task_thaw(h2_task *task) -{ - if (task->frozen) { - task->frozen = 0; - ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, task->c, APLOGNO(03407) - "h2_task(%s), thawed", task->id); - } - task->thawed = 1; - return APR_SUCCESS; -} - -int h2_task_has_thawed(h2_task *task) -{ - return task->thawed; -} --- a/modules/http2/h2_task.h +++ b/modules/http2/h2_task.h @@ -35,14 +35,13 @@ * * Finally, to keep certain connection level filters, such as ourselves and * especially mod_ssl ones, from messing with our data, we need a filter - * of our own to disble those. + * of our own to disable those. */ struct h2_bucket_beam; struct h2_conn; struct h2_mplx; struct h2_task; -struct h2_req_engine; struct h2_request; struct h2_response_parser; struct h2_stream; @@ -80,20 +79,18 @@ struct h2_mplx *mplx; unsigned int filters_set : 1; - unsigned int frozen : 1; - unsigned int thawed : 1; unsigned int worker_started : 1; /* h2_worker started processing */ - unsigned int worker_done : 1; /* h2_worker finished */ + unsigned int redo : 1; /* was throttled, should be restarted later */ + + int worker_done; /* h2_worker finished */ + int done_done; /* task_done has been handled */ apr_time_t started_at; /* when processing started */ apr_time_t done_at; /* when processing was done */ apr_bucket *eor; - - struct h2_req_engine *engine; /* engine hosted by this task */ - struct h2_req_engine *assigned; /* engine that task has been assigned to */ }; -h2_task *h2_task_create(conn_rec *slave, int stream_id, +h2_task *h2_task_create(conn_rec *secondary, int stream_id, const h2_request *req, struct h2_mplx *m, struct h2_bucket_beam *input, apr_interval_time_t timeout, @@ -105,6 +102,7 @@ void h2_task_redo(h2_task *task); int h2_task_can_redo(h2_task *task); +int h2_task_has_started(h2_task *task); /** * Reset the task with the given error code, resets all input/output. @@ -120,8 +118,4 @@ extern APR_OPTIONAL_FN_TYPE(ap_logio_add_bytes_in) *h2_task_logio_add_bytes_in; extern APR_OPTIONAL_FN_TYPE(ap_logio_add_bytes_out) *h2_task_logio_add_bytes_out; -apr_status_t h2_task_freeze(h2_task *task); -apr_status_t h2_task_thaw(h2_task *task); -int h2_task_has_thawed(h2_task *task); - #endif /* defined(__mod_h2__h2_task__) */ --- a/modules/http2/h2_util.c +++ b/modules/http2/h2_util.c @@ -638,15 +638,6 @@ apr_status_t rv; if ((rv = apr_thread_mutex_lock(fifo->lock)) == APR_SUCCESS) { fifo->aborted = 1; - apr_thread_mutex_unlock(fifo->lock); - } - return rv; -} - -apr_status_t h2_fifo_interrupt(h2_fifo *fifo) -{ - apr_status_t rv; - if ((rv = apr_thread_mutex_lock(fifo->lock)) == APR_SUCCESS) { apr_thread_cond_broadcast(fifo->not_empty); apr_thread_cond_broadcast(fifo->not_full); apr_thread_mutex_unlock(fifo->lock); @@ -710,10 +701,6 @@ { apr_status_t rv; - if (fifo->aborted) { - return APR_EOF; - } - if ((rv = apr_thread_mutex_lock(fifo->lock)) == APR_SUCCESS) { rv = fifo_push_int(fifo, elem, block); apr_thread_mutex_unlock(fifo->lock); @@ -754,10 +741,6 @@ { apr_status_t rv; - if (fifo->aborted) { - return APR_EOF; - } - if ((rv = apr_thread_mutex_lock(fifo->lock)) == APR_SUCCESS) { rv = pull_head(fifo, pelem, block); apr_thread_mutex_unlock(fifo->lock); @@ -946,15 +929,6 @@ apr_status_t rv; if ((rv = apr_thread_mutex_lock(fifo->lock)) == APR_SUCCESS) { fifo->aborted = 1; - apr_thread_mutex_unlock(fifo->lock); - } - return rv; -} - -apr_status_t h2_ififo_interrupt(h2_ififo *fifo) -{ - apr_status_t rv; - if ((rv = apr_thread_mutex_lock(fifo->lock)) == APR_SUCCESS) { apr_thread_cond_broadcast(fifo->not_empty); apr_thread_cond_broadcast(fifo->not_full); apr_thread_mutex_unlock(fifo->lock); @@ -1018,10 +992,6 @@ { apr_status_t rv; - if (fifo->aborted) { - return APR_EOF; - } - if ((rv = apr_thread_mutex_lock(fifo->lock)) == APR_SUCCESS) { rv = ififo_push_int(fifo, id, block); apr_thread_mutex_unlock(fifo->lock); @@ -1062,10 +1032,6 @@ { apr_status_t rv; - if (fifo->aborted) { - return APR_EOF; - } - if ((rv = apr_thread_mutex_lock(fifo->lock)) == APR_SUCCESS) { rv = ipull_head(fifo, pi, block); apr_thread_mutex_unlock(fifo->lock); @@ -1088,10 +1054,6 @@ apr_status_t rv; int id; - if (fifo->aborted) { - return APR_EOF; - } - if (APR_SUCCESS == (rv = apr_thread_mutex_lock(fifo->lock))) { if (APR_SUCCESS == (rv = ipull_head(fifo, &id, block))) { switch (fn(id, ctx)) { @@ -1117,39 +1079,40 @@ return ififo_peek(fifo, fn, ctx, 0); } -apr_status_t h2_ififo_remove(h2_ififo *fifo, int id) +static apr_status_t ififo_remove(h2_ififo *fifo, int id) { - apr_status_t rv; + int rc, i; if (fifo->aborted) { return APR_EOF; } - if ((rv = apr_thread_mutex_lock(fifo->lock)) == APR_SUCCESS) { - int i, rc; - int e; - - rc = 0; - for (i = 0; i < fifo->count; ++i) { - e = fifo->elems[inth_index(fifo, i)]; - if (e == id) { - ++rc; - } - else if (rc) { - fifo->elems[inth_index(fifo, i-rc)] = e; - } - } - if (rc) { - fifo->count -= rc; - if (fifo->count + rc == fifo->nelems) { - apr_thread_cond_broadcast(fifo->not_full); - } - rv = APR_SUCCESS; + rc = 0; + for (i = 0; i < fifo->count; ++i) { + int e = fifo->elems[inth_index(fifo, i)]; + if (e == id) { + ++rc; } - else { - rv = APR_EAGAIN; + else if (rc) { + fifo->elems[inth_index(fifo, i-rc)] = e; } - + } + if (!rc) { + return APR_EAGAIN; + } + fifo->count -= rc; + if (fifo->count + rc == fifo->nelems) { + apr_thread_cond_broadcast(fifo->not_full); + } + return APR_SUCCESS; +} + +apr_status_t h2_ififo_remove(h2_ififo *fifo, int id) +{ + apr_status_t rv; + + if ((rv = apr_thread_mutex_lock(fifo->lock)) == APR_SUCCESS) { + rv = ififo_remove(fifo, id); apr_thread_mutex_unlock(fifo->lock); } return rv; @@ -1373,7 +1336,7 @@ return status; } else if (blen == 0) { - /* brigade without data, does it have an EOS bucket somwhere? */ + /* brigade without data, does it have an EOS bucket somewhere? */ *plen = 0; *peos = h2_util_has_eos(bb, -1); } @@ -1840,22 +1803,29 @@ } apr_status_t h2_req_add_header(apr_table_t *headers, apr_pool_t *pool, - const char *name, size_t nlen, - const char *value, size_t vlen) + const char *name, size_t nlen, + const char *value, size_t vlen, + size_t max_field_len, int *pwas_added) { char *hname, *hvalue; + const char *existing; + *pwas_added = 0; if (h2_req_ignore_header(name, nlen)) { return APR_SUCCESS; } else if (H2_HD_MATCH_LIT("cookie", name, nlen)) { - const char *existing = apr_table_get(headers, "cookie"); + existing = apr_table_get(headers, "cookie"); if (existing) { char *nval; /* Cookie header come separately in HTTP/2, but need * to be merged by "; " (instead of default ", ") */ + if (max_field_len && strlen(existing) + vlen + nlen + 4 > max_field_len) { + /* "key: oldval, nval" is too long */ + return APR_EINVAL; + } hvalue = apr_pstrndup(pool, value, vlen); nval = apr_psprintf(pool, "%s; %s", existing, hvalue); apr_table_setn(headers, "Cookie", nval); @@ -1869,8 +1839,16 @@ } hname = apr_pstrndup(pool, name, nlen); - hvalue = apr_pstrndup(pool, value, vlen); h2_util_camel_case_header(hname, nlen); + existing = apr_table_get(headers, hname); + if (max_field_len) { + if ((existing? strlen(existing)+2 : 0) + vlen + nlen + 2 > max_field_len) { + /* "key: (oldval, )?nval" is too long */ + return APR_EINVAL; + } + } + if (!existing) *pwas_added = 1; + hvalue = apr_pstrndup(pool, value, vlen); apr_table_mergen(headers, hname, hvalue); return APR_SUCCESS; @@ -1960,7 +1938,8 @@ case NGHTTP2_GOAWAY: { size_t len = (frame->goaway.opaque_data_len < s_len)? frame->goaway.opaque_data_len : s_len-1; - memcpy(scratch, frame->goaway.opaque_data, len); + if (len) + memcpy(scratch, frame->goaway.opaque_data, len); scratch[len] = '\0'; return apr_snprintf(buffer, maxlen, "GOAWAY[error=%d, reason='%s', " "last_stream=%d]", frame->goaway.error_code, --- a/modules/http2/h2_util.h +++ b/modules/http2/h2_util.h @@ -209,7 +209,6 @@ apr_status_t h2_fifo_set_create(h2_fifo **pfifo, apr_pool_t *pool, int capacity); apr_status_t h2_fifo_term(h2_fifo *fifo); -apr_status_t h2_fifo_interrupt(h2_fifo *fifo); int h2_fifo_count(h2_fifo *fifo); @@ -229,7 +228,7 @@ typedef enum { H2_FIFO_OP_PULL, /* pull the element from the queue, ie discard it */ - H2_FIFO_OP_REPUSH, /* pull and immediatley re-push it */ + H2_FIFO_OP_REPUSH, /* pull and immediately re-push it */ } h2_fifo_op_t; typedef h2_fifo_op_t h2_fifo_peek_fn(void *head, void *ctx); @@ -280,7 +279,6 @@ apr_status_t h2_ififo_set_create(h2_ififo **pfifo, apr_pool_t *pool, int capacity); apr_status_t h2_ififo_term(h2_ififo *fifo); -apr_status_t h2_ififo_interrupt(h2_ififo *fifo); int h2_ififo_count(h2_ififo *fifo); @@ -412,9 +410,14 @@ apr_status_t h2_req_create_ngheader(h2_ngheader **ph, apr_pool_t *p, const struct h2_request *req); +/** + * Add a HTTP/2 header and return the table key if it really was added + * and not ignored. + */ apr_status_t h2_req_add_header(apr_table_t *headers, apr_pool_t *pool, const char *name, size_t nlen, - const char *value, size_t vlen); + const char *value, size_t vlen, + size_t max_field_len, int *pwas_added); /******************************************************************************* * h2_request helpers --- a/modules/http2/h2_version.h +++ b/modules/http2/h2_version.h @@ -27,7 +27,7 @@ * @macro * Version number of the http2 module as c string */ -#define MOD_HTTP2_VERSION "1.11.4" +#define MOD_HTTP2_VERSION "1.15.14" /** * @macro @@ -35,7 +35,6 @@ * release. This is a 24 bit number with 8 bits for major number, 8 bits * for minor and 8 bits for patch. Version 1.2.3 becomes 0x010203. */ -#define MOD_HTTP2_VERSION_NUM 0x010b04 - +#define MOD_HTTP2_VERSION_NUM 0x010f0e #endif /* mod_h2_h2_version_h */ --- a/modules/http2/h2_workers.c +++ b/modules/http2/h2_workers.c @@ -155,7 +155,7 @@ { apr_status_t rv; - rv = h2_mplx_pop_task(m, &slot->task); + rv = h2_mplx_s_pop_task(m, &slot->task); if (slot->task) { /* Ok, we got something to give back to the worker for execution. * If we still have idle workers, we let the worker be sticky, @@ -234,10 +234,10 @@ * mplx the opportunity to give us back a new task right away. */ if (!slot->aborted && (--slot->sticks > 0)) { - h2_mplx_task_done(slot->task->mplx, slot->task, &slot->task); + h2_mplx_s_task_done(slot->task->mplx, slot->task, &slot->task); } else { - h2_mplx_task_done(slot->task->mplx, slot->task, NULL); + h2_mplx_s_task_done(slot->task->mplx, slot->task, NULL); slot->task = NULL; } } @@ -269,7 +269,6 @@ } h2_fifo_term(workers->mplxs); - h2_fifo_interrupt(workers->mplxs); cleanup_zombies(workers); } --- a/modules/http2/mod_http2.c +++ b/modules/http2/mod_http2.c @@ -172,27 +172,6 @@ conn_rec *, request_rec *, char *name); static int http2_is_h2(conn_rec *); -static apr_status_t http2_req_engine_push(const char *ngn_type, - request_rec *r, - http2_req_engine_init *einit) -{ - return h2_mplx_req_engine_push(ngn_type, r, einit); -} - -static apr_status_t http2_req_engine_pull(h2_req_engine *ngn, - apr_read_type_e block, - int capacity, - request_rec **pr) -{ - return h2_mplx_req_engine_pull(ngn, block, capacity, pr); -} - -static void http2_req_engine_done(h2_req_engine *ngn, conn_rec *r_conn, - apr_status_t status) -{ - h2_mplx_req_engine_done(ngn, r_conn, status); -} - static void http2_get_num_workers(server_rec *s, int *minw, int *maxw) { h2_get_num_workers(s, minw, maxw); @@ -220,9 +199,6 @@ APR_REGISTER_OPTIONAL_FN(http2_is_h2); APR_REGISTER_OPTIONAL_FN(http2_var_lookup); - APR_REGISTER_OPTIONAL_FN(http2_req_engine_push); - APR_REGISTER_OPTIONAL_FN(http2_req_engine_pull); - APR_REGISTER_OPTIONAL_FN(http2_req_engine_done); APR_REGISTER_OPTIONAL_FN(http2_get_num_workers); ap_log_perror(APLOG_MARK, APLOG_TRACE1, 0, pool, "installing hooks"); @@ -260,9 +236,8 @@ { if (ctx) { if (r) { - h2_task *task = h2_ctx_get_task(ctx); - if (task) { - h2_stream *stream = h2_mplx_stream_get(task->mplx, task->stream_id); + if (ctx->task) { + h2_stream *stream = h2_mplx_t_stream_get(ctx->task->mplx, ctx->task); if (stream && stream->push_policy != H2_PUSH_NONE) { return "on"; } @@ -273,8 +248,7 @@ } } else if (s) { - const h2_config *cfg = h2_config_sget(s); - if (cfg && h2_config_geti(cfg, H2_CONF_PUSH)) { + if (h2_config_geti(r, s, H2_CONF_PUSH)) { return "on"; } } @@ -285,8 +259,7 @@ conn_rec *c, request_rec *r, h2_ctx *ctx) { if (ctx) { - h2_task *task = h2_ctx_get_task(ctx); - if (task && !H2_STREAM_CLIENT_INITIATED(task->stream_id)) { + if (ctx->task && !H2_STREAM_CLIENT_INITIATED(ctx->task->stream_id)) { return "PUSHED"; } } @@ -297,9 +270,8 @@ conn_rec *c, request_rec *r, h2_ctx *ctx) { if (ctx) { - h2_task *task = h2_ctx_get_task(ctx); - if (task && !H2_STREAM_CLIENT_INITIATED(task->stream_id)) { - h2_stream *stream = h2_mplx_stream_get(task->mplx, task->stream_id); + if (ctx->task && !H2_STREAM_CLIENT_INITIATED(ctx->task->stream_id)) { + h2_stream *stream = h2_mplx_t_stream_get(ctx->task->mplx, ctx->task); if (stream) { return apr_itoa(p, stream->initiated_on); } @@ -312,9 +284,8 @@ conn_rec *c, request_rec *r, h2_ctx *ctx) { if (ctx) { - h2_task *task = h2_ctx_get_task(ctx); - if (task) { - return task->id; + if (ctx->task) { + return ctx->task->id; } } return ""; @@ -366,7 +337,7 @@ for (i = 0; i < H2_ALEN(H2_VARS); ++i) { h2_var_def *vdef = &H2_VARS[i]; if (!strcmp(vdef->name, name)) { - h2_ctx *ctx = (r? h2_ctx_rget(r) : + h2_ctx *ctx = (r? h2_ctx_get(c, 0) : h2_ctx_get(c->master? c->master : c, 0)); return (char *)vdef->lookup(p, s, c, r, ctx); } @@ -377,7 +348,7 @@ static int h2_h2_fixups(request_rec *r) { if (r->connection->master) { - h2_ctx *ctx = h2_ctx_rget(r); + h2_ctx *ctx = h2_ctx_get(r->connection, 0); int i; for (i = 0; ctx && i < H2_ALEN(H2_VARS); ++i) { --- a/modules/http2/mod_http2.dep +++ b/modules/http2/mod_http2.dep @@ -694,7 +694,6 @@ ".\h2_ctx.h"\ ".\h2_h2.h"\ ".\h2_mplx.h"\ - ".\h2_ngn_shed.h"\ ".\h2_private.h"\ ".\h2_request.h"\ ".\h2_stream.h"\ @@ -754,7 +753,6 @@ ".\h2_ctx.h"\ ".\h2_h2.h"\ ".\h2_mplx.h"\ - ".\h2_ngn_shed.h"\ ".\h2_private.h"\ ".\h2_request.h"\ ".\h2_task.h"\ --- a/modules/http2/mod_http2.dsp +++ b/modules/http2/mod_http2.dsp @@ -145,10 +145,6 @@ # End Source File # Begin Source File -SOURCE=./h2_ngn_shed.c -# End Source File -# Begin Source File - SOURCE=./h2_push.c # End Source File # Begin Source File --- a/modules/http2/mod_http2.h +++ b/modules/http2/mod_http2.h @@ -30,22 +30,20 @@ /******************************************************************************* - * HTTP/2 request engines + * START HTTP/2 request engines (DEPRECATED) ******************************************************************************/ + +/* The following functions were introduced for the experimental mod_proxy_http2 + * support, but have been abandoned since. + * They are still declared here for backward compatibility, in case someone + * tries to build an old mod_proxy_http2 against it, but will disappear + * completely sometime in the future. + */ struct apr_thread_cond_t; - typedef struct h2_req_engine h2_req_engine; - typedef void http2_output_consumed(void *ctx, conn_rec *c, apr_off_t consumed); -/** - * Initialize a h2_req_engine. The structure will be passed in but - * only the name and master are set. The function should initialize - * all fields. - * @param engine the allocated, partially filled structure - * @param r the first request to process, or NULL - */ typedef apr_status_t http2_req_engine_init(h2_req_engine *engine, const char *id, const char *type, @@ -55,35 +53,11 @@ http2_output_consumed **pconsumed, void **pbaton); -/** - * Push a request to an engine with the specified name for further processing. - * If no such engine is available, einit is not NULL, einit is called - * with a new engine record and the caller is responsible for running the - * new engine instance. - * @param engine_type the type of the engine to add the request to - * @param r the request to push to an engine for processing - * @param einit an optional initialization callback for a new engine - * of the requested type, should no instance be available. - * By passing a non-NULL callback, the caller is willing - * to init and run a new engine itself. - * @return APR_SUCCESS iff slave was successfully added to an engine - */ APR_DECLARE_OPTIONAL_FN(apr_status_t, http2_req_engine_push, (const char *engine_type, request_rec *r, http2_req_engine_init *einit)); -/** - * Get a new request for processing in this engine. - * @param engine the engine which is done processing the slave - * @param block if call should block waiting for request to come - * @param capacity how many parallel requests are acceptable - * @param pr the request that needs processing or NULL - * @return APR_SUCCESS if new request was assigned - * APR_EAGAIN if no new request is available - * APR_EOF if engine may shut down, as no more request will be scheduled - * APR_ECONNABORTED if the engine needs to shut down immediately - */ APR_DECLARE_OPTIONAL_FN(apr_status_t, http2_req_engine_pull, (h2_req_engine *engine, apr_read_type_e block, @@ -98,4 +72,8 @@ http2_get_num_workers, (server_rec *s, int *minw, int *max)); +/******************************************************************************* + * END HTTP/2 request engines (DEPRECATED) + ******************************************************************************/ + #endif --- a/modules/http2/mod_http2.mak +++ b/modules/http2/mod_http2.mak @@ -61,7 +61,6 @@ -@erase "$(INTDIR)\h2_h2.obj" -@erase "$(INTDIR)\h2_headers.obj" -@erase "$(INTDIR)\h2_mplx.obj" - -@erase "$(INTDIR)\h2_ngn_shed.obj" -@erase "$(INTDIR)\h2_push.obj" -@erase "$(INTDIR)\h2_request.obj" -@erase "$(INTDIR)\h2_session.obj" @@ -138,7 +137,6 @@ "$(INTDIR)\h2_h2.obj" \ "$(INTDIR)\h2_headers.obj" \ "$(INTDIR)\h2_mplx.obj" \ - "$(INTDIR)\h2_ngn_shed.obj" \ "$(INTDIR)\h2_push.obj" \ "$(INTDIR)\h2_request.obj" \ "$(INTDIR)\h2_session.obj" \ @@ -207,7 +205,6 @@ -@erase "$(INTDIR)\h2_h2.obj" -@erase "$(INTDIR)\h2_headers.obj" -@erase "$(INTDIR)\h2_mplx.obj" - -@erase "$(INTDIR)\h2_ngn_shed.obj" -@erase "$(INTDIR)\h2_push.obj" -@erase "$(INTDIR)\h2_request.obj" -@erase "$(INTDIR)\h2_session.obj" @@ -284,7 +281,6 @@ "$(INTDIR)\h2_h2.obj" \ "$(INTDIR)\h2_headers.obj" \ "$(INTDIR)\h2_mplx.obj" \ - "$(INTDIR)\h2_ngn_shed.obj" \ "$(INTDIR)\h2_push.obj" \ "$(INTDIR)\h2_request.obj" \ "$(INTDIR)\h2_session.obj" \ @@ -469,11 +465,6 @@ "$(INTDIR)\h2_mplx.obj" : $(SOURCE) "$(INTDIR)" -SOURCE=./h2_ngn_shed.c - -"$(INTDIR)\h2_ngn_shed.obj" : $(SOURCE) "$(INTDIR)" - - SOURCE=./h2_push.c "$(INTDIR)\h2_push.obj" : $(SOURCE) "$(INTDIR)" --- a/modules/http2/mod_proxy_http2.c +++ b/modules/http2/mod_proxy_http2.c @@ -16,13 +16,14 @@ #include +#include #include #include #include "mod_http2.h" #include "mod_proxy_http2.h" -#include "h2_request.h" +#include "h2.h" #include "h2_proxy_util.h" #include "h2_version.h" #include "h2_proxy_session.h" @@ -46,19 +47,12 @@ /* Optional functions from mod_http2 */ static int (*is_h2)(conn_rec *c); -static apr_status_t (*req_engine_push)(const char *name, request_rec *r, - http2_req_engine_init *einit); -static apr_status_t (*req_engine_pull)(h2_req_engine *engine, - apr_read_type_e block, - int capacity, - request_rec **pr); -static void (*req_engine_done)(h2_req_engine *engine, conn_rec *r_conn, - apr_status_t status); - + typedef struct h2_proxy_ctx { + const char *id; + conn_rec *master; conn_rec *owner; apr_pool_t *pool; - request_rec *rbase; server_rec *server; const char *proxy_func; char server_portstr[32]; @@ -66,19 +60,15 @@ proxy_worker *worker; proxy_server_conf *conf; - h2_req_engine *engine; - const char *engine_id; - const char *engine_type; - apr_pool_t *engine_pool; apr_size_t req_buffer_size; - h2_proxy_fifo *requests; int capacity; - unsigned standalone : 1; unsigned is_ssl : 1; - unsigned flushall : 1; - apr_status_t r_status; /* status of our first request work */ + request_rec *r; /* the request processed in this ctx */ + apr_status_t r_status; /* status of request work */ + int r_done; /* request was processed, not necessarily successfully */ + int r_may_retry; /* request may be retried */ h2_proxy_session *session; /* current http2 session against backend */ } h2_proxy_ctx; @@ -104,16 +94,6 @@ MOD_HTTP2_VERSION, ngh2? ngh2->version_str : "unknown"); is_h2 = APR_RETRIEVE_OPTIONAL_FN(http2_is_h2); - req_engine_push = APR_RETRIEVE_OPTIONAL_FN(http2_req_engine_push); - req_engine_pull = APR_RETRIEVE_OPTIONAL_FN(http2_req_engine_pull); - req_engine_done = APR_RETRIEVE_OPTIONAL_FN(http2_req_engine_done); - - /* we need all of them */ - if (!req_engine_push || !req_engine_pull || !req_engine_done) { - req_engine_push = NULL; - req_engine_pull = NULL; - req_engine_done = NULL; - } return status; } @@ -204,45 +184,6 @@ return OK; } -static void out_consumed(void *baton, conn_rec *c, apr_off_t bytes) -{ - h2_proxy_ctx *ctx = baton; - - if (ctx->session) { - h2_proxy_session_update_window(ctx->session, c, bytes); - } -} - -static apr_status_t proxy_engine_init(h2_req_engine *engine, - const char *id, - const char *type, - apr_pool_t *pool, - apr_size_t req_buffer_size, - request_rec *r, - http2_output_consumed **pconsumed, - void **pctx) -{ - h2_proxy_ctx *ctx = ap_get_module_config(r->connection->conn_config, - &proxy_http2_module); - if (!ctx) { - ap_log_rerror(APLOG_MARK, APLOG_WARNING, 0, r, APLOGNO(03368) - "h2_proxy_session, engine init, no ctx found"); - return APR_ENOTIMPL; - } - - ctx->pool = pool; - ctx->engine = engine; - ctx->engine_id = id; - ctx->engine_type = type; - ctx->engine_pool = pool; - ctx->req_buffer_size = req_buffer_size; - ctx->capacity = H2MIN(100, h2_proxy_fifo_capacity(ctx->requests)); - - *pconsumed = out_consumed; - *pctx = ctx; - return APR_SUCCESS; -} - static apr_status_t add_request(h2_proxy_session *session, request_rec *r) { h2_proxy_ctx *ctx = session->user_data; @@ -252,7 +193,7 @@ url = apr_table_get(r->notes, H2_PROXY_REQ_URL_NOTE); apr_table_setn(r->notes, "proxy-source-port", apr_psprintf(r->pool, "%hu", ctx->p_conn->connection->local_addr->port)); - status = h2_proxy_session_submit(session, url, r, ctx->standalone); + status = h2_proxy_session_submit(session, url, r, 1); if (status != APR_SUCCESS) { ap_log_cerror(APLOG_MARK, APLOG_ERR, status, r->connection, APLOGNO(03351) "pass request body failed to %pI (%s) from %s (%s)", @@ -266,43 +207,15 @@ static void request_done(h2_proxy_ctx *ctx, request_rec *r, apr_status_t status, int touched) { - const char *task_id = apr_table_get(r->connection->notes, H2_TASK_ID_NOTE); - - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, r->connection, - "h2_proxy_session(%s): request done %s, touched=%d", - ctx->engine_id, task_id, touched); - if (status != APR_SUCCESS) { - if (!touched) { - /* untouched request, need rescheduling */ - status = h2_proxy_fifo_push(ctx->requests, r); - ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, r->connection, - APLOGNO(03369) - "h2_proxy_session(%s): rescheduled request %s", - ctx->engine_id, task_id); - return; - } - else { - const char *uri; - uri = apr_uri_unparse(r->pool, &r->parsed_uri, 0); - ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, r->connection, - APLOGNO(03471) "h2_proxy_session(%s): request %s -> %s " - "not complete, cannot repeat", - ctx->engine_id, task_id, uri); - } - } - - if (r == ctx->rbase) { + if (r == ctx->r) { + ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, r->connection, + "h2_proxy_session(%s): request done, touched=%d", + ctx->id, touched); + ctx->r_done = 1; + if (touched) ctx->r_may_retry = 0; ctx->r_status = ((status == APR_SUCCESS)? APR_SUCCESS : HTTP_SERVICE_UNAVAILABLE); } - - if (req_engine_done && ctx->engine) { - ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, r->connection, - APLOGNO(03370) - "h2_proxy_session(%s): finished request %s", - ctx->engine_id, task_id); - req_engine_done(ctx->engine, r->connection, status); - } } static void session_req_done(h2_proxy_session *session, request_rec *r, @@ -311,43 +224,15 @@ request_done(session->user_data, r, status, touched); } -static apr_status_t next_request(h2_proxy_ctx *ctx, int before_leave) -{ - if (h2_proxy_fifo_count(ctx->requests) > 0) { - return APR_SUCCESS; - } - else if (req_engine_pull && ctx->engine) { - apr_status_t status; - request_rec *r = NULL; - - status = req_engine_pull(ctx->engine, before_leave? - APR_BLOCK_READ: APR_NONBLOCK_READ, - ctx->capacity, &r); - if (status == APR_SUCCESS && r) { - ap_log_cerror(APLOG_MARK, APLOG_TRACE3, status, ctx->owner, - "h2_proxy_engine(%s): pulled request (%s) %s", - ctx->engine_id, - before_leave? "before leave" : "regular", - r->the_request); - h2_proxy_fifo_push(ctx->requests, r); - } - return APR_STATUS_IS_EAGAIN(status)? APR_SUCCESS : status; - } - return APR_EOF; -} - -static apr_status_t proxy_engine_run(h2_proxy_ctx *ctx) { +static apr_status_t ctx_run(h2_proxy_ctx *ctx) { apr_status_t status = OK; int h2_front; - request_rec *r; /* Step Four: Send the Request in a new HTTP/2 stream and * loop until we got the response or encounter errors. */ - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, ctx->owner, - "eng(%s): setup session", ctx->engine_id); h2_front = is_h2? is_h2(ctx->owner) : 0; - ctx->session = h2_proxy_session_setup(ctx->engine_id, ctx->p_conn, ctx->conf, + ctx->session = h2_proxy_session_setup(ctx->id, ctx->p_conn, ctx->conf, h2_front, 30, h2_proxy_log2((int)ctx->req_buffer_size), session_req_done); @@ -358,105 +243,45 @@ } ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, ctx->owner, APLOGNO(03373) - "eng(%s): run session %s", ctx->engine_id, ctx->session->id); + "eng(%s): run session %s", ctx->id, ctx->session->id); ctx->session->user_data = ctx; - while (!ctx->owner->aborted) { - if (APR_SUCCESS == h2_proxy_fifo_try_pull(ctx->requests, (void**)&r)) { - add_request(ctx->session, r); - } - + ctx->r_done = 0; + add_request(ctx->session, ctx->r); + + while (!ctx->master->aborted && !ctx->r_done) { + status = h2_proxy_session_process(ctx->session); - - if (status == APR_SUCCESS) { - apr_status_t s2; - /* ongoing processing, call again */ - if (ctx->session->remote_max_concurrent > 0 - && ctx->session->remote_max_concurrent != ctx->capacity) { - ctx->capacity = H2MIN((int)ctx->session->remote_max_concurrent, - h2_proxy_fifo_capacity(ctx->requests)); - } - s2 = next_request(ctx, 0); - if (s2 == APR_ECONNABORTED) { - /* master connection gone */ - ap_log_cerror(APLOG_MARK, APLOG_DEBUG, s2, ctx->owner, - APLOGNO(03374) "eng(%s): pull request", - ctx->engine_id); - /* give notice that we're leaving and cancel all ongoing - * streams. */ - next_request(ctx, 1); - h2_proxy_session_cancel_all(ctx->session); - h2_proxy_session_process(ctx->session); - status = ctx->r_status = APR_SUCCESS; - break; - } - if ((h2_proxy_fifo_count(ctx->requests) == 0) - && h2_proxy_ihash_empty(ctx->session->streams)) { - break; - } - } - else { - /* end of processing, maybe error */ + if (status != APR_SUCCESS) { + /* Encountered an error during session processing */ ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, ctx->owner, APLOGNO(03375) "eng(%s): end of session %s", - ctx->engine_id, ctx->session->id); - /* - * Any open stream of that session needs to + ctx->id, ctx->session->id); + /* Any open stream of that session needs to * a) be reopened on the new session iff safe to do so * b) reported as done (failed) otherwise */ h2_proxy_session_cleanup(ctx->session, session_req_done); - break; + goto out; } } - ctx->session->user_data = NULL; - ctx->session = NULL; - - return status; -} - -static apr_status_t push_request_somewhere(h2_proxy_ctx *ctx, request_rec *r) -{ - conn_rec *c = ctx->owner; - const char *engine_type, *hostname; - - hostname = (ctx->p_conn->ssl_hostname? - ctx->p_conn->ssl_hostname : ctx->p_conn->hostname); - engine_type = apr_psprintf(ctx->pool, "proxy_http2 %s%s", hostname, - ctx->server_portstr); - - if (c->master && req_engine_push && r && is_h2 && is_h2(c)) { - /* If we are have req_engine capabilities, push the handling of this - * request (e.g. slave connection) to a proxy_http2 engine which - * uses the same backend. We may be called to create an engine - * ourself. */ - if (req_engine_push(engine_type, r, proxy_engine_init) == APR_SUCCESS) { - if (ctx->engine == NULL) { - /* request has been assigned to an engine in another thread */ - return SUSPENDED; - } +out: + if (ctx->master->aborted) { + /* master connection gone */ + ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, ctx->owner, + APLOGNO(03374) "eng(%s): master connection gone", ctx->id); + /* cancel all ongoing requests */ + h2_proxy_session_cancel_all(ctx->session); + h2_proxy_session_process(ctx->session); + if (!ctx->master->aborted) { + status = ctx->r_status = APR_SUCCESS; } } - if (!ctx->engine) { - /* No engine was available or has been initialized, handle this - * request just by ourself. */ - ctx->engine_id = apr_psprintf(ctx->pool, "eng-proxy-%ld", c->id); - ctx->engine_type = engine_type; - ctx->engine_pool = ctx->pool; - ctx->req_buffer_size = (32*1024); - ctx->standalone = 1; - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, - "h2_proxy_http2(%ld): setup standalone engine for type %s", - c->id, engine_type); - } - else { - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, - "H2: hosting engine %s", ctx->engine_id); - } - - return h2_proxy_fifo_push(ctx->requests, r); + ctx->session->user_data = NULL; + ctx->session = NULL; + return status; } static int proxy_http2_handler(request_rec *r, @@ -466,7 +291,7 @@ const char *proxyname, apr_port_t proxyport) { - const char *proxy_func; + const char *proxy_func, *task_id; char *locurl = url, *u; apr_size_t slen; int is_ssl = 0; @@ -498,29 +323,35 @@ default: return DECLINED; } + + task_id = apr_table_get(r->connection->notes, H2_TASK_ID_NOTE); ctx = apr_pcalloc(r->pool, sizeof(*ctx)); - ctx->owner = r->connection; - ctx->pool = r->pool; - ctx->rbase = r; - ctx->server = r->server; + ctx->master = r->connection->master? r->connection->master : r->connection; + ctx->id = task_id? task_id : apr_psprintf(r->pool, "%ld", (long)ctx->master->id); + ctx->owner = r->connection; + ctx->pool = r->pool; + ctx->server = r->server; ctx->proxy_func = proxy_func; - ctx->is_ssl = is_ssl; - ctx->worker = worker; - ctx->conf = conf; - ctx->flushall = apr_table_get(r->subprocess_env, "proxy-flushall")? 1 : 0; - ctx->r_status = HTTP_SERVICE_UNAVAILABLE; - - h2_proxy_fifo_set_create(&ctx->requests, ctx->pool, 100); + ctx->is_ssl = is_ssl; + ctx->worker = worker; + ctx->conf = conf; + ctx->req_buffer_size = (32*1024); + ctx->r = r; + ctx->r_status = status = HTTP_SERVICE_UNAVAILABLE; + ctx->r_done = 0; + ctx->r_may_retry = 1; ap_set_module_config(ctx->owner->conn_config, &proxy_http2_module, ctx); /* scheme says, this is for us. */ - apr_table_setn(ctx->rbase->notes, H2_PROXY_REQ_URL_NOTE, url); - ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, ctx->rbase, + apr_table_setn(ctx->r->notes, H2_PROXY_REQ_URL_NOTE, url); + ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, ctx->r, "H2: serving URL %s", url); run_connect: + if (ctx->master->aborted) goto cleanup; + /* Get a proxy_conn_rec from the worker, might be a new one, might * be one still open from another request, or it might fail if the * worker is stopped or in error. */ @@ -530,25 +361,11 @@ } ctx->p_conn->is_ssl = ctx->is_ssl; - if (ctx->is_ssl && ctx->p_conn->connection) { - /* If there are some metadata on the connection (e.g. TLS alert), - * let mod_ssl detect them, and create a new connection below. - */ - apr_bucket_brigade *tmp_bb; - tmp_bb = apr_brigade_create(ctx->rbase->pool, - ctx->rbase->connection->bucket_alloc); - status = ap_get_brigade(ctx->p_conn->connection->input_filters, tmp_bb, - AP_MODE_SPECULATIVE, APR_NONBLOCK_READ, 1); - if (status != APR_SUCCESS && !APR_STATUS_IS_EAGAIN(status)) { - ctx->p_conn->close = 1; - } - apr_brigade_cleanup(tmp_bb); - } /* Step One: Determine the URL to connect to (might be a proxy), * initialize the backend accordingly and determine the server * port string we can expect in responses. */ - if ((status = ap_proxy_determine_connection(ctx->pool, ctx->rbase, conf, worker, + if ((status = ap_proxy_determine_connection(ctx->pool, ctx->r, conf, worker, ctx->p_conn, &uri, &locurl, proxyname, proxyport, ctx->server_portstr, @@ -556,17 +373,6 @@ goto cleanup; } - /* If we are not already hosting an engine, try to push the request - * to an already existing engine or host a new engine here. */ - if (r && !ctx->engine) { - ctx->r_status = push_request_somewhere(ctx, r); - r = NULL; - if (ctx->r_status == SUSPENDED) { - /* request was pushed to another thread, leave processing here */ - goto cleanup; - } - } - /* Step Two: Make the Connection (or check that an already existing * socket is still usable). On success, we have a socket connected to * backend->hostname. */ @@ -575,70 +381,56 @@ ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, ctx->owner, APLOGNO(03352) "H2: failed to make connection to backend: %s", ctx->p_conn->hostname); - goto reconnect; + goto cleanup; } /* Step Three: Create conn_rec for the socket we have open now. */ - if (!ctx->p_conn->connection) { - status = ap_proxy_connection_create_ex(ctx->proxy_func, - ctx->p_conn, ctx->rbase); - if (status != OK) { - ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, ctx->owner, APLOGNO(03353) - "setup new connection: is_ssl=%d %s %s %s", - ctx->p_conn->is_ssl, ctx->p_conn->ssl_hostname, - locurl, ctx->p_conn->hostname); - goto reconnect; - } - - if (!ctx->p_conn->data) { - /* New conection: set a note on the connection what CN is - * requested and what protocol we want */ - if (ctx->p_conn->ssl_hostname) { - ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, ctx->owner, - "set SNI to %s for (%s)", - ctx->p_conn->ssl_hostname, - ctx->p_conn->hostname); - apr_table_setn(ctx->p_conn->connection->notes, - "proxy-request-hostname", ctx->p_conn->ssl_hostname); - } - if (ctx->is_ssl) { - apr_table_setn(ctx->p_conn->connection->notes, - "proxy-request-alpn-protos", "h2"); - } - } + status = ap_proxy_connection_create_ex(ctx->proxy_func, ctx->p_conn, ctx->r); + if (status != OK) { + ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, ctx->owner, APLOGNO(03353) + "setup new connection: is_ssl=%d %s %s %s", + ctx->p_conn->is_ssl, ctx->p_conn->ssl_hostname, + locurl, ctx->p_conn->hostname); + ctx->r_status = status; + goto cleanup; } - -run_session: - status = proxy_engine_run(ctx); - if (status == APR_SUCCESS) { - /* session and connection still ok */ - if (next_request(ctx, 1) == APR_SUCCESS) { - /* more requests, run again */ - ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, ctx->owner, APLOGNO(03376) - "run_session, again"); - goto run_session; + + if (!ctx->p_conn->data && ctx->is_ssl) { + /* New SSL connection: set a note on the connection about what + * protocol we want. + */ + apr_table_setn(ctx->p_conn->connection->notes, + "proxy-request-alpn-protos", "h2"); + if (ctx->p_conn->ssl_hostname) { + ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, ctx->owner, + "set SNI to %s for (%s)", + ctx->p_conn->ssl_hostname, + ctx->p_conn->hostname); + apr_table_setn(ctx->p_conn->connection->notes, + "proxy-request-hostname", ctx->p_conn->ssl_hostname); } - /* done */ - ctx->engine = NULL; } -reconnect: - if (next_request(ctx, 1) == APR_SUCCESS) { - /* Still more to do, tear down old conn and start over */ + if (ctx->master->aborted) goto cleanup; + status = ctx_run(ctx); + + if (ctx->r_status != APR_SUCCESS && ctx->r_may_retry && !ctx->master->aborted) { + /* Not successfully processed, but may retry, tear down old conn and start over */ if (ctx->p_conn) { ctx->p_conn->close = 1; - /*only in trunk so far */ - /*proxy_run_detach_backend(r, ctx->p_conn);*/ +#if AP_MODULE_MAGIC_AT_LEAST(20140207, 2) + proxy_run_detach_backend(r, ctx->p_conn); +#endif ap_proxy_release_connection(ctx->proxy_func, ctx->p_conn, ctx->server); ctx->p_conn = NULL; } ++reconnects; - if (reconnects < 5 && !ctx->owner->aborted) { + if (reconnects < 5) { goto run_connect; } ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, ctx->owner, APLOGNO(10023) - "giving up after %d reconnects, %d requests todo", - reconnects, h2_proxy_fifo_count(ctx->requests)); + "giving up after %d reconnects, request-done=%d", + reconnects, ctx->r_done); } cleanup: @@ -647,17 +439,13 @@ /* close socket when errors happened or session shut down (EOF) */ ctx->p_conn->close = 1; } - /*only in trunk so far */ - /*proxy_run_detach_backend(ctx->rbase, ctx->p_conn);*/ +#if AP_MODULE_MAGIC_AT_LEAST(20140207, 2) + proxy_run_detach_backend(ctx->r, ctx->p_conn); +#endif ap_proxy_release_connection(ctx->proxy_func, ctx->p_conn, ctx->server); ctx->p_conn = NULL; } - /* Any requests will still have need to fail */ - while (APR_SUCCESS == h2_proxy_fifo_try_pull(ctx->requests, (void**)&r)) { - request_done(ctx, r, HTTP_SERVICE_UNAVAILABLE, 1); - } - ap_set_module_config(ctx->owner->conn_config, &proxy_http2_module, NULL); ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, ctx->owner, APLOGNO(03377) "leaving handler");