From cff6d757e3ba609c08ef2aaa00f07e53551e5bf6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Mon, 3 Jun 2024 07:11:10 +0200 Subject: Adding upstream version 3.0.0. Signed-off-by: Daniel Baumann --- src/h3.c | 587 +++++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 343 insertions(+), 244 deletions(-) (limited to 'src/h3.c') diff --git a/src/h3.c b/src/h3.c index 4aa1a52..321d883 100644 --- a/src/h3.c +++ b/src/h3.c @@ -36,8 +36,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -58,19 +58,21 @@ static const struct trace_event h3_trace_events[] = { { .mask = H3_EV_RX_HDR, .name = "rx_hdr", .desc = "receipt of H3 HEADERS frame" }, #define H3_EV_RX_SETTINGS (1ULL << 3) { .mask = H3_EV_RX_SETTINGS, .name = "rx_settings", .desc = "receipt of H3 SETTINGS frame" }, -#define H3_EV_TX_DATA (1ULL << 4) +#define H3_EV_TX_FRAME (1ULL << 4) + { .mask = H3_EV_TX_FRAME, .name = "tx_frame", .desc = "transmission of any H3 frame" }, +#define H3_EV_TX_DATA (1ULL << 5) { .mask = H3_EV_TX_DATA, .name = "tx_data", .desc = "transmission of H3 DATA frame" }, -#define H3_EV_TX_HDR (1ULL << 5) +#define H3_EV_TX_HDR (1ULL << 6) { .mask = H3_EV_TX_HDR, .name = "tx_hdr", .desc = "transmission of H3 HEADERS frame" }, -#define H3_EV_TX_SETTINGS (1ULL << 6) +#define H3_EV_TX_SETTINGS (1ULL << 7) { .mask = H3_EV_TX_SETTINGS, .name = "tx_settings", .desc = "transmission of H3 SETTINGS frame" }, -#define H3_EV_H3S_NEW (1ULL << 7) +#define H3_EV_H3S_NEW (1ULL << 8) { .mask = H3_EV_H3S_NEW, .name = "h3s_new", .desc = "new H3 stream" }, -#define H3_EV_H3S_END (1ULL << 8) +#define H3_EV_H3S_END (1ULL << 9) { .mask = H3_EV_H3S_END, .name = "h3s_end", .desc = "H3 stream terminated" }, -#define H3_EV_H3C_NEW (1ULL << 9) +#define H3_EV_H3C_NEW (1ULL << 10) { .mask = H3_EV_H3C_NEW, .name = "h3c_new", .desc = "new H3 connection" }, -#define H3_EV_H3C_END (1ULL << 10) +#define H3_EV_H3C_END (1ULL << 11) { .mask = H3_EV_H3C_END, .name = "h3c_end", .desc = "H3 connection terminated" }, #define H3_EV_STRM_SEND (1ULL << 12) { .mask = H3_EV_STRM_SEND, .name = "strm_send", .desc = "sending data for stream" }, @@ -129,7 +131,7 @@ static uint64_t h3_settings_max_field_section_size = QUIC_VARINT_8_BYTE_MAX; /* struct h3c { struct qcc *qcc; struct qcs *ctrl_strm; /* Control stream */ - enum h3_err err; + int err; uint32_t flags; /* Settings */ @@ -195,7 +197,8 @@ static ssize_t h3_init_uni_stream(struct h3c *h3c, struct qcs *qcs, case H3_UNI_S_T_CTRL: if (h3c->flags & H3_CF_UNI_CTRL_SET) { TRACE_ERROR("duplicated control stream", H3_EV_H3S_NEW, qcs->qcc->conn, qcs); - qcc_set_error(qcs->qcc, H3_STREAM_CREATION_ERROR, 1); + qcc_set_error(qcs->qcc, H3_ERR_STREAM_CREATION_ERROR, 1); + qcc_report_glitch(qcs->qcc, 1); goto err; } h3c->flags |= H3_CF_UNI_CTRL_SET; @@ -210,7 +213,8 @@ static ssize_t h3_init_uni_stream(struct h3c *h3c, struct qcs *qcs, case H3_UNI_S_T_QPACK_DEC: if (h3c->flags & H3_CF_UNI_QPACK_DEC_SET) { TRACE_ERROR("duplicated qpack decoder stream", H3_EV_H3S_NEW, qcs->qcc->conn, qcs); - qcc_set_error(qcs->qcc, H3_STREAM_CREATION_ERROR, 1); + qcc_set_error(qcs->qcc, H3_ERR_STREAM_CREATION_ERROR, 1); + qcc_report_glitch(qcs->qcc, 1); goto err; } h3c->flags |= H3_CF_UNI_QPACK_DEC_SET; @@ -221,7 +225,8 @@ static ssize_t h3_init_uni_stream(struct h3c *h3c, struct qcs *qcs, case H3_UNI_S_T_QPACK_ENC: if (h3c->flags & H3_CF_UNI_QPACK_ENC_SET) { TRACE_ERROR("duplicated qpack encoder stream", H3_EV_H3S_NEW, qcs->qcc->conn, qcs); - qcc_set_error(qcs->qcc, H3_STREAM_CREATION_ERROR, 1); + qcc_set_error(qcs->qcc, H3_ERR_STREAM_CREATION_ERROR, 1); + qcc_report_glitch(qcs->qcc, 1); goto err; } h3c->flags |= H3_CF_UNI_QPACK_ENC_SET; @@ -320,7 +325,7 @@ static int h3_check_frame_valid(struct h3c *h3c, struct qcs *qcs, uint64_t ftype /* cf H3_FT_HEADERS case. */ if (h3s->type == H3S_T_CTRL || (h3s->st_req != H3S_ST_REQ_HEADERS && h3s->st_req != H3S_ST_REQ_DATA)) { - ret = H3_FRAME_UNEXPECTED; + ret = H3_ERR_FRAME_UNEXPECTED; } break; @@ -347,7 +352,7 @@ static int h3_check_frame_valid(struct h3c *h3c, struct qcs *qcs, uint64_t ftype * own rules; see Section 9. */ if (h3s->type == H3S_T_CTRL || h3s->st_req == H3S_ST_REQ_TRAILERS) - ret = H3_FRAME_UNEXPECTED; + ret = H3_ERR_FRAME_UNEXPECTED; break; case H3_FT_CANCEL_PUSH: @@ -374,9 +379,9 @@ static int h3_check_frame_valid(struct h3c *h3c, struct qcs *qcs, uint64_t ftype */ if (h3s->type != H3S_T_CTRL) - ret = H3_FRAME_UNEXPECTED; + ret = H3_ERR_FRAME_UNEXPECTED; else if (!(h3c->flags & H3_CF_SETTINGS_RECV)) - ret = H3_MISSING_SETTINGS; + ret = H3_ERR_MISSING_SETTINGS; break; case H3_FT_SETTINGS: @@ -394,7 +399,7 @@ static int h3_check_frame_valid(struct h3c *h3c, struct qcs *qcs, uint64_t ftype * H3_FRAME_UNEXPECTED. */ if (h3s->type != H3S_T_CTRL || h3c->flags & H3_CF_SETTINGS_RECV) - ret = H3_FRAME_UNEXPECTED; + ret = H3_ERR_FRAME_UNEXPECTED; break; case H3_FT_PUSH_PROMISE: @@ -406,7 +411,7 @@ static int h3_check_frame_valid(struct h3c *h3c, struct qcs *qcs, uint64_t ftype */ /* TODO server-side only. */ - ret = H3_FRAME_UNEXPECTED; + ret = H3_ERR_FRAME_UNEXPECTED; break; default: @@ -420,7 +425,7 @@ static int h3_check_frame_valid(struct h3c *h3c, struct qcs *qcs, uint64_t ftype * not satisfy that requirement and SHOULD be treated as an error. */ if (h3s->type == H3S_T_CTRL && !(h3c->flags & H3_CF_SETTINGS_RECV)) - ret = H3_MISSING_SETTINGS; + ret = H3_ERR_MISSING_SETTINGS; break; } @@ -461,7 +466,8 @@ static int h3_check_body_size(struct qcs *qcs, int fin) if (h3s->data_len > h3s->body_len || (fin && h3s->data_len < h3s->body_len)) { TRACE_ERROR("Content-length does not match DATA frame size", H3_EV_RX_FRAME|H3_EV_RX_DATA, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(qcs->qcc, 1); ret = -1; } @@ -527,6 +533,7 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, int cookie = -1, last_cookie = -1, i; const char *ctl; int relaxed = !!(h3c->qcc->proxy->options2 & PR_O2_REQBUG_OK); + int qpack_err; /* RFC 9114 4.1.2. Malformed Requests and Responses * @@ -558,14 +565,16 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, list, sizeof(list) / sizeof(list[0])); if (ret < 0) { TRACE_ERROR("QPACK decoding error", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3c->err = -ret; + if ((qpack_err = qpack_err_decode(ret)) >= 0) { + h3c->err = qpack_err; + qcc_report_glitch(qcs->qcc, 1); + } len = -1; goto out; } - if (!qcs_get_buf(qcs, &htx_buf)) { + if (!b_alloc(&htx_buf, DB_SE_RX)) { TRACE_ERROR("HTX buffer alloc failure", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3c->err = H3_INTERNAL_ERROR; len = -1; goto out; } @@ -604,7 +613,8 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, ctl = ist_find_ctl(list[hdr_idx].v); if (unlikely(ctl) && http_header_has_forbidden_char(list[hdr_idx].v, ctl)) { TRACE_ERROR("control character present in pseudo-header value", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -615,7 +625,8 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, if (isteq(list[hdr_idx].n, ist(":method"))) { if (isttest(meth)) { TRACE_ERROR("duplicated method pseudo-header", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -624,7 +635,8 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, else if (isteq(list[hdr_idx].n, ist(":path"))) { if (isttest(path)) { TRACE_ERROR("duplicated path pseudo-header", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -636,7 +648,8 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, ctl = ist_find_range(list[hdr_idx].v, 0, '#'); if (unlikely(ctl) && http_path_has_forbidden_char(list[hdr_idx].v, ctl)) { TRACE_ERROR("forbidden character in ':path' pseudo-header", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -648,7 +661,8 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, if (isttest(scheme)) { /* duplicated pseudo-header */ TRACE_ERROR("duplicated scheme pseudo-header", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -657,20 +671,23 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, else if (isteq(list[hdr_idx].n, ist(":authority"))) { if (isttest(authority)) { TRACE_ERROR("duplicated authority pseudo-header", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } if (h3_set_authority(qcs, &authority, list[hdr_idx].v)) { - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } } else { TRACE_ERROR("unknown pseudo-header", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -687,7 +704,8 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, */ if (!isttest(meth) || !isttest(scheme) || !isttest(path)) { TRACE_ERROR("missing mandatory pseudo-header", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -698,7 +716,6 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, sl = htx_add_stline(htx, HTX_BLK_REQ_SL, flags, meth, path, ist("HTTP/3.0")); if (!sl) { - h3c->err = H3_INTERNAL_ERROR; len = -1; goto out; } @@ -710,7 +727,6 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, if (isttest(authority)) { if (!htx_add_header(htx, ist("host"), authority)) { - h3c->err = H3_INTERNAL_ERROR; len = -1; goto out; } @@ -723,7 +739,8 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, if (istmatch(list[hdr_idx].n, ist(":"))) { TRACE_ERROR("pseudo-header field after fields", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -732,7 +749,8 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, const char c = list[hdr_idx].n.ptr[i]; if ((uint8_t)(c - 'A') < 'Z' - 'A' || !HTTP_IS_TOKEN(c)) { TRACE_ERROR("invalid characters in field name", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -753,14 +771,16 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, ctl = ist_find_ctl(list[hdr_idx].v); if (unlikely(ctl) && http_header_has_forbidden_char(list[hdr_idx].v, ctl)) { TRACE_ERROR("control character present in header value", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } if (isteq(list[hdr_idx].n, ist("host"))) { if (h3_set_authority(qcs, &authority, list[hdr_idx].v)) { - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -776,7 +796,8 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, h3s->flags & H3_SF_HAVE_CLEN); if (ret < 0) { TRACE_ERROR("invalid content-length", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -810,7 +831,8 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, * connection-specific fields MUST be treated as malformed. */ TRACE_ERROR("invalid connection header", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -823,13 +845,13 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, * NOT contain any value other than "trailers". */ TRACE_ERROR("invalid te header", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } if (!htx_add_header(htx, list[hdr_idx].n, list[hdr_idx].v)) { - h3c->err = H3_INTERNAL_ERROR; len = -1; goto out; } @@ -845,21 +867,20 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, */ if (!isttest(authority)) { TRACE_ERROR("missing mandatory pseudo-header", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } if (cookie >= 0) { if (http_cookie_merge(htx, list, cookie)) { - h3c->err = H3_INTERNAL_ERROR; len = -1; goto out; } } if (!htx_add_endof(htx, HTX_BLK_EOH)) { - h3c->err = H3_INTERNAL_ERROR; len = -1; goto out; } @@ -871,7 +892,6 @@ static ssize_t h3_headers_to_htx(struct qcs *qcs, const struct buffer *buf, htx = NULL; if (!qcs_attach_sc(qcs, &htx_buf, fin)) { - h3c->err = H3_INTERNAL_ERROR; len = -1; goto out; } @@ -927,6 +947,7 @@ static ssize_t h3_trailers_to_htx(struct qcs *qcs, const struct buffer *buf, struct http_hdr list[global.tune.max_http_hdr]; int hdr_idx, ret; const char *ctl; + int qpack_err; int i; TRACE_ENTER(H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); @@ -937,14 +958,16 @@ static ssize_t h3_trailers_to_htx(struct qcs *qcs, const struct buffer *buf, list, sizeof(list) / sizeof(list[0])); if (ret < 0) { TRACE_ERROR("QPACK decoding error", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3c->err = -ret; + if ((qpack_err = qpack_err_decode(ret)) >= 0) { + h3c->err = qpack_err; + qcc_report_glitch(qcs->qcc, 1); + } len = -1; goto out; } - if (!(appbuf = qcs_get_buf(qcs, &qcs->rx.app_buf))) { + if (!(appbuf = qcc_get_stream_rxbuf(qcs))) { TRACE_ERROR("HTX buffer alloc failure", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3c->err = H3_INTERNAL_ERROR; len = -1; goto out; } @@ -978,7 +1001,8 @@ static ssize_t h3_trailers_to_htx(struct qcs *qcs, const struct buffer *buf, */ if (istmatch(list[hdr_idx].n, ist(":"))) { TRACE_ERROR("pseudo-header field in trailers", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -987,7 +1011,8 @@ static ssize_t h3_trailers_to_htx(struct qcs *qcs, const struct buffer *buf, const char c = list[hdr_idx].n.ptr[i]; if ((uint8_t)(c - 'A') < 'Z' - 'A' || !HTTP_IS_TOKEN(c)) { TRACE_ERROR("invalid characters in field name", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -1002,7 +1027,8 @@ static ssize_t h3_trailers_to_htx(struct qcs *qcs, const struct buffer *buf, isteq(list[hdr_idx].n, ist("te")) || isteq(list[hdr_idx].n, ist("transfer-encoding"))) { TRACE_ERROR("forbidden HTTP/3 headers", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } @@ -1021,14 +1047,14 @@ static ssize_t h3_trailers_to_htx(struct qcs *qcs, const struct buffer *buf, ctl = ist_find_ctl(list[hdr_idx].v); if (unlikely(ctl) && http_header_has_forbidden_char(list[hdr_idx].v, ctl)) { TRACE_ERROR("control character present in trailer value", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3s->err = H3_MESSAGE_ERROR; + h3s->err = H3_ERR_MESSAGE_ERROR; + qcc_report_glitch(h3c->qcc, 1); len = -1; goto out; } if (!htx_add_trailer(htx, list[hdr_idx].n, list[hdr_idx].v)) { TRACE_ERROR("cannot add trailer", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3c->err = H3_INTERNAL_ERROR; len = -1; goto out; } @@ -1038,7 +1064,6 @@ static ssize_t h3_trailers_to_htx(struct qcs *qcs, const struct buffer *buf, if (!htx_add_endof(htx, HTX_BLK_EOT)) { TRACE_ERROR("cannot add trailer", H3_EV_RX_FRAME|H3_EV_RX_HDR, qcs->qcc->conn, qcs); - h3c->err = H3_INTERNAL_ERROR; len = -1; goto out; } @@ -1064,8 +1089,6 @@ static ssize_t h3_trailers_to_htx(struct qcs *qcs, const struct buffer *buf, static ssize_t h3_data_to_htx(struct qcs *qcs, const struct buffer *buf, uint64_t len, char fin) { - struct h3s *h3s = qcs->ctx; - struct h3c *h3c = h3s->h3c; struct buffer *appbuf; struct htx *htx = NULL; size_t htx_sent = 0; @@ -1074,11 +1097,9 @@ static ssize_t h3_data_to_htx(struct qcs *qcs, const struct buffer *buf, TRACE_ENTER(H3_EV_RX_FRAME|H3_EV_RX_DATA, qcs->qcc->conn, qcs); - if (!(appbuf = qcs_get_buf(qcs, &qcs->rx.app_buf))) { + if (!(appbuf = qcc_get_stream_rxbuf(qcs))) { TRACE_ERROR("data buffer alloc failure", H3_EV_RX_FRAME|H3_EV_RX_DATA, qcs->qcc->conn, qcs); - h3c->err = H3_INTERNAL_ERROR; - len = -1; - goto out; + goto err; } htx = htx_from_buf(appbuf); @@ -1129,6 +1150,10 @@ static ssize_t h3_data_to_htx(struct qcs *qcs, const struct buffer *buf, TRACE_LEAVE(H3_EV_RX_FRAME|H3_EV_RX_DATA, qcs->qcc->conn, qcs); return htx_sent; + + err: + TRACE_DEVEL("leaving on error", H3_EV_RX_FRAME|H3_EV_RX_DATA, qcs->qcc->conn, qcs); + return -1; } /* Parse a SETTINGS frame of length of payload . @@ -1150,7 +1175,8 @@ static ssize_t h3_parse_settings_frm(struct h3c *h3c, const struct buffer *buf, while (b_data(&b)) { if (!b_quic_dec_int(&id, &b, &ret) || !b_quic_dec_int(&value, &b, &ret)) { - h3c->err = H3_FRAME_ERROR; + h3c->err = H3_ERR_FRAME_ERROR; + qcc_report_glitch(h3c->qcc, 1); return -1; } @@ -1167,7 +1193,8 @@ static ssize_t h3_parse_settings_frm(struct h3c *h3c, const struct buffer *buf, /* Ignore duplicate check for ID too big used for GREASE. */ if (id < sizeof(mask)) { if (ha_bit_test(id, &mask)) { - h3c->err = H3_SETTINGS_ERROR; + h3c->err = H3_ERR_SETTINGS_ERROR; + qcc_report_glitch(h3c->qcc, 1); return -1; } ha_bit_set(id, &mask); @@ -1197,7 +1224,8 @@ static ssize_t h3_parse_settings_frm(struct h3c *h3c, const struct buffer *buf, * their receipt MUST be treated as a connection error of type * H3_SETTINGS_ERROR. */ - h3c->err = H3_SETTINGS_ERROR; + h3c->err = H3_ERR_SETTINGS_ERROR; + qcc_report_glitch(h3c->qcc, 1); return -1; default: /* MUST be ignored */ @@ -1209,16 +1237,16 @@ static ssize_t h3_parse_settings_frm(struct h3c *h3c, const struct buffer *buf, return ret; } -/* Decode remotely initiated bidi-stream. must be set to indicate - * that we received the last data of the stream. +/* Transcode HTTP/3 payload received in buffer to HTX data for stream + * . If is set, it indicates that no more data will arrive after. * * Returns 0 on success else non-zero. */ -static ssize_t h3_decode_qcs(struct qcs *qcs, struct buffer *b, int fin) +static ssize_t h3_rcv_buf(struct qcs *qcs, struct buffer *b, int fin) { struct h3s *h3s = qcs->ctx; struct h3c *h3c = h3s->h3c; - ssize_t total = 0, ret; + ssize_t total = 0, ret = 0; TRACE_ENTER(H3_EV_RX_FRAME, qcs->qcc->conn, qcs); @@ -1256,31 +1284,36 @@ static ssize_t h3_decode_qcs(struct qcs *qcs, struct buffer *b, int fin) */ if (h3s->type == H3S_T_CTRL && fin) { TRACE_ERROR("control stream closed by remote peer", H3_EV_RX_FRAME, qcs->qcc->conn, qcs); - qcc_set_error(qcs->qcc, H3_CLOSED_CRITICAL_STREAM, 1); + qcc_set_error(qcs->qcc, H3_ERR_CLOSED_CRITICAL_STREAM, 1); + qcc_report_glitch(qcs->qcc, 1); goto err; } if (!b_data(b) && fin && quic_stream_is_bidi(qcs->id)) { struct buffer *appbuf; struct htx *htx; + int eom; TRACE_PROTO("received FIN without data", H3_EV_RX_FRAME, qcs->qcc->conn, qcs); - if (!(appbuf = qcs_get_buf(qcs, &qcs->rx.app_buf))) { + if (!(appbuf = qcc_get_stream_rxbuf(qcs))) { TRACE_ERROR("data buffer alloc failure", H3_EV_RX_FRAME, qcs->qcc->conn, qcs); - h3c->err = H3_INTERNAL_ERROR; + qcc_set_error(qcs->qcc, H3_ERR_INTERNAL_ERROR, 1); goto err; } htx = htx_from_buf(appbuf); - if (!htx_set_eom(htx)) { + eom = htx_set_eom(htx); + htx_to_buf(htx, appbuf); + if (!eom) { TRACE_ERROR("cannot set EOM", H3_EV_RX_FRAME, qcs->qcc->conn, qcs); - h3c->err = H3_INTERNAL_ERROR; + qcc_set_error(qcs->qcc, H3_ERR_INTERNAL_ERROR, 1); + goto err; } - htx_to_buf(htx, appbuf); + goto done; } - while (b_data(b) && !(qcs->flags & QC_SF_DEM_FULL) && !h3c->err && !h3s->err) { + while (b_data(b) && !(qcs->flags & QC_SF_DEM_FULL) && ret >= 0) { uint64_t ftype, flen; char last_stream_frame = 0; @@ -1307,6 +1340,7 @@ static ssize_t h3_decode_qcs(struct qcs *qcs, struct buffer *b, int fin) if ((ret = h3_check_frame_valid(h3c, qcs, ftype))) { TRACE_ERROR("received an invalid frame", H3_EV_RX_FRAME, qcs->qcc->conn, qcs); qcc_set_error(qcs->qcc, ret, 1); + qcc_report_glitch(qcs->qcc, 1); goto err; } @@ -1329,7 +1363,8 @@ static ssize_t h3_decode_qcs(struct qcs *qcs, struct buffer *b, int fin) */ if (flen > QC_S_RX_BUF_SZ) { TRACE_ERROR("received a too big frame", H3_EV_RX_FRAME, qcs->qcc->conn, qcs); - qcc_set_error(qcs->qcc, H3_EXCESSIVE_LOAD, 1); + qcc_set_error(qcs->qcc, H3_ERR_EXCESSIVE_LOAD, 1); + qcc_report_glitch(qcs->qcc, 1); goto err; } break; @@ -1405,6 +1440,10 @@ static ssize_t h3_decode_qcs(struct qcs *qcs, struct buffer *b, int fin) qcc_set_error(qcs->qcc, h3c->err, 1); return b_data(b); } + else if (unlikely(ret < 0)) { + qcc_set_error(qcs->qcc, H3_ERR_INTERNAL_ERROR, 1); + goto err; + } /* TODO may be useful to wakeup the MUX if blocked due to full buffer. * However, currently, io-cb of MUX does not handle Rx. @@ -1419,17 +1458,6 @@ static ssize_t h3_decode_qcs(struct qcs *qcs, struct buffer *b, int fin) return -1; } -/* Returns buffer for data sending. - * May be NULL if the allocation failed. - */ -static struct buffer *mux_get_buf(struct qcs *qcs) -{ - if (!b_size(&qcs->tx.buf)) - b_alloc(&qcs->tx.buf); - - return &qcs->tx.buf; -} - /* Function used to emit stream data from control uni-stream. * * On success return the number of sent bytes. A negative code is used on @@ -1437,13 +1465,14 @@ static struct buffer *mux_get_buf(struct qcs *qcs) */ static int h3_control_send(struct qcs *qcs, void *ctx) { + int err; int ret; struct h3c *h3c = ctx; unsigned char data[(2 + 3) * 2 * QUIC_VARINT_MAX_SIZE]; /* enough for 3 settings */ struct buffer pos, *res; size_t frm_len; - TRACE_ENTER(H3_EV_TX_SETTINGS, qcs->qcc->conn, qcs); + TRACE_ENTER(H3_EV_TX_FRAME|H3_EV_TX_SETTINGS, qcs->qcc->conn, qcs); BUG_ON_HOT(h3c->flags & H3_CF_SETTINGS_SENT); @@ -1472,9 +1501,14 @@ static int h3_control_send(struct qcs *qcs, void *ctx) b_quic_enc_int(&pos, h3_settings_max_field_section_size, 0); } - res = mux_get_buf(qcs); - if (b_is_null(res)) { - TRACE_ERROR("cannot allocate Tx buffer", H3_EV_TX_SETTINGS, qcs->qcc->conn, qcs); + if (qfctl_sblocked(&qcs->tx.fc) || qfctl_sblocked(&qcs->qcc->tx.fc)) { + TRACE_ERROR("not enough initial credit for control stream", H3_EV_TX_FRAME|H3_EV_TX_SETTINGS, qcs->qcc->conn, qcs); + goto err; + } + + if (!(res = qcc_get_stream_txbuf(qcs, &err))) { + /* Consider alloc failure fatal for control stream even on conn buf limit. */ + TRACE_ERROR("cannot allocate Tx buffer", H3_EV_TX_FRAME|H3_EV_TX_SETTINGS, qcs->qcc->conn, qcs); goto err; } @@ -1487,22 +1521,21 @@ static int h3_control_send(struct qcs *qcs, void *ctx) ret = b_force_xfer(res, &pos, b_data(&pos)); if (ret > 0) { /* Register qcs for sending before other streams. */ - qcc_send_stream(qcs, 1); + qcc_send_stream(qcs, 1, ret); h3c->flags |= H3_CF_SETTINGS_SENT; } - TRACE_LEAVE(H3_EV_TX_SETTINGS, qcs->qcc->conn, qcs); + TRACE_LEAVE(H3_EV_TX_FRAME|H3_EV_TX_SETTINGS, qcs->qcc->conn, qcs); return ret; err: - TRACE_DEVEL("leaving on error", H3_EV_TX_SETTINGS, qcs->qcc->conn, qcs); + TRACE_DEVEL("leaving on error", H3_EV_TX_FRAME|H3_EV_TX_SETTINGS, qcs->qcc->conn, qcs); return -1; } static int h3_resp_headers_send(struct qcs *qcs, struct htx *htx) { - struct h3s *h3s = qcs->ctx; - struct h3c *h3c = h3s->h3c; + int err; struct buffer outbuf; struct buffer headers_buf = BUF_NULL; struct buffer *res; @@ -1515,7 +1548,7 @@ static int h3_resp_headers_send(struct qcs *qcs, struct htx *htx) int hdr; int status = 0; - TRACE_ENTER(H3_EV_TX_HDR, qcs->qcc->conn, qcs); + TRACE_ENTER(H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); sl = NULL; hdr = 0; @@ -1537,8 +1570,7 @@ static int h3_resp_headers_send(struct qcs *qcs, struct htx *htx) } else if (type == HTX_BLK_HDR) { if (unlikely(hdr >= sizeof(list) / sizeof(list[0]) - 1)) { - TRACE_ERROR("too many headers", H3_EV_TX_HDR, qcs->qcc->conn, qcs); - h3c->err = H3_INTERNAL_ERROR; + TRACE_ERROR("too many headers", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); goto err; } list[hdr].n = htx_get_blk_name(htx, blk); @@ -1555,11 +1587,14 @@ static int h3_resp_headers_send(struct qcs *qcs, struct htx *htx) list[hdr].n = ist(""); - res = mux_get_buf(qcs); - if (b_is_null(res)) { - TRACE_ERROR("cannot allocate Tx buffer", H3_EV_TX_HDR, qcs->qcc->conn, qcs); - h3c->err = H3_INTERNAL_ERROR; - goto err; + if (!(res = qcc_get_stream_txbuf(qcs, &err))) { + if (err) { + TRACE_ERROR("cannot allocate Tx buffer", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); + goto err; + } + + TRACE_STATE("conn buf limit reached", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); + goto end; } /* At least 5 bytes to store frame type + length as a varint max size */ @@ -1571,11 +1606,12 @@ static int h3_resp_headers_send(struct qcs *qcs, struct htx *htx) /* Start the headers after frame type + length */ headers_buf = b_make(b_head(res) + 5, b_size(res) - 5, 0, 0); + TRACE_DATA("encoding HEADERS frame", H3_EV_TX_FRAME|H3_EV_TX_HDR, + qcs->qcc->conn, qcs); if (qpack_encode_field_section_line(&headers_buf)) ABORT_NOW(); if (qpack_encode_int_status(&headers_buf, status)) { - TRACE_ERROR("invalid status code", H3_EV_TX_HDR, qcs->qcc->conn, qcs); - h3c->err = H3_INTERNAL_ERROR; + TRACE_ERROR("invalid status code", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); goto err; } @@ -1630,11 +1666,12 @@ static int h3_resp_headers_send(struct qcs *qcs, struct htx *htx) break; } - TRACE_LEAVE(H3_EV_TX_HDR, qcs->qcc->conn, qcs); + end: + TRACE_LEAVE(H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); return ret; err: - TRACE_DEVEL("leaving on error", H3_EV_TX_HDR, qcs->qcc->conn, qcs); + TRACE_DEVEL("leaving on error", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); return -1; } @@ -1646,12 +1683,12 @@ static int h3_resp_headers_send(struct qcs *qcs, struct htx *htx) * Caller is responsible to emit an empty QUIC STREAM frame to signal the end * of the stream. * - * Returns the size of HTX blocks removed. + * Returns the size of HTX blocks removed. A negative error code is returned in + * case of a fatal error which should caused a connection closure. */ static int h3_resp_trailers_send(struct qcs *qcs, struct htx *htx) { - struct h3s *h3s = qcs->ctx; - struct h3c *h3c = h3s->h3c; + int err; struct buffer headers_buf = BUF_NULL; struct buffer *res; struct http_hdr list[global.tune.max_http_hdr]; @@ -1661,7 +1698,7 @@ static int h3_resp_trailers_send(struct qcs *qcs, struct htx *htx) int ret = 0; int hdr; - TRACE_ENTER(H3_EV_TX_HDR, qcs->qcc->conn, qcs); + TRACE_ENTER(H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); hdr = 0; for (blk = htx_get_head_blk(htx); blk; blk = htx_get_next_blk(htx, blk)) { @@ -1675,8 +1712,7 @@ static int h3_resp_trailers_send(struct qcs *qcs, struct htx *htx) if (type == HTX_BLK_TLR) { if (unlikely(hdr >= sizeof(list) / sizeof(list[0]) - 1)) { - TRACE_ERROR("too many headers", H3_EV_TX_HDR, qcs->qcc->conn, qcs); - h3c->err = H3_INTERNAL_ERROR; + TRACE_ERROR("too many headers", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); goto err; } list[hdr].n = htx_get_blk_name(htx, blk); @@ -1684,8 +1720,7 @@ static int h3_resp_trailers_send(struct qcs *qcs, struct htx *htx) hdr++; } else { - TRACE_ERROR("unexpected HTX block", H3_EV_TX_HDR, qcs->qcc->conn, qcs); - h3c->err = H3_INTERNAL_ERROR; + TRACE_ERROR("unexpected HTX block", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); goto err; } } @@ -1694,22 +1729,41 @@ static int h3_resp_trailers_send(struct qcs *qcs, struct htx *htx) /* No headers encoded here so no need to generate a H3 HEADERS * frame. Mux will send an empty QUIC STREAM frame with FIN. */ - TRACE_DATA("skipping trailer", H3_EV_TX_HDR, qcs->qcc->conn, qcs); + TRACE_DATA("skipping trailer", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); + + /* Truncate UNUSED / EOT HTX blocks. */ + blk = htx_get_head_blk(htx); + while (blk) { + type = htx_get_blk_type(blk); + ret += htx_get_blksz(blk); + blk = htx_remove_blk(htx, blk); + if (type == HTX_BLK_EOT) + break; + } goto end; } + list[hdr].n = ist(""); - res = mux_get_buf(qcs); - if (b_is_null(res)) { - TRACE_ERROR("cannot allocate Tx buffer", H3_EV_TX_HDR, qcs->qcc->conn, qcs); - h3c->err = H3_INTERNAL_ERROR; - goto err; + start: + if (!(res = qcc_get_stream_txbuf(qcs, &err))) { + if (err) { + TRACE_ERROR("cannot allocate Tx buffer", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); + goto err; + } + + TRACE_STATE("conn buf limit reached", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); + goto end; } /* At least 9 bytes to store frame type + length as a varint max size */ if (b_room(res) < 9) { - qcs->flags |= QC_SF_BLK_MROOM; - goto err; + TRACE_STATE("not enough room for trailers frame", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); + if (qcc_release_stream_txbuf(qcs)) + goto end; + + /* Buffer released, restart processing. */ + goto start; } /* Force buffer realignment as size required to encode headers is unknown. */ @@ -1719,8 +1773,12 @@ static int h3_resp_trailers_send(struct qcs *qcs, struct htx *htx) headers_buf = b_make(b_peek(res, b_data(res) + 9), b_contig_space(res) - 9, 0, 0); if (qpack_encode_field_section_line(&headers_buf)) { - qcs->flags |= QC_SF_BLK_MROOM; - goto err; + TRACE_STATE("not enough room for trailers section line", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); + if (qcc_release_stream_txbuf(qcs)) + goto end; + + /* Buffer released, restart processing. */ + goto start; } tail = b_tail(&headers_buf); @@ -1740,8 +1798,12 @@ static int h3_resp_trailers_send(struct qcs *qcs, struct htx *htx) } if (qpack_encode_header(&headers_buf, list[hdr].n, list[hdr].v)) { - qcs->flags |= QC_SF_BLK_MROOM; - goto err; + TRACE_STATE("not enough room for all trailers", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); + if (qcc_release_stream_txbuf(qcs)) + goto end; + + /* Buffer released, restart processing. */ + goto start; } } @@ -1750,20 +1812,21 @@ static int h3_resp_trailers_send(struct qcs *qcs, struct htx *htx) /* No headers encoded here so no need to generate a H3 HEADERS * frame. Mux will send an empty QUIC STREAM frame with FIN. */ - TRACE_DATA("skipping trailer", H3_EV_TX_HDR, qcs->qcc->conn, qcs); - goto end; + TRACE_DATA("skipping trailer", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); + } + else { + /* Now that all headers are encoded, we are certain that res + * buffer is big enough. + */ + TRACE_DATA("encoding TRAILERS frame", H3_EV_TX_FRAME|H3_EV_TX_HDR, + qcs->qcc->conn, qcs); + b_putchr(res, 0x01); /* h3 HEADERS frame type */ + if (!b_quic_enc_int(res, b_data(&headers_buf), 8)) + ABORT_NOW(); + b_add(res, b_data(&headers_buf)); } - /* Now that all headers are encoded, we are certain that res buffer is - * big enough. - */ - b_putchr(res, 0x01); /* h3 HEADERS frame type */ - if (!b_quic_enc_int(res, b_data(&headers_buf), 8)) - ABORT_NOW(); - b_add(res, b_data(&headers_buf)); - - end: - ret = 0; + /* Encoding success, truncate HTX blocks until EOT. */ blk = htx_get_head_blk(htx); while (blk) { type = htx_get_blk_type(blk); @@ -1773,23 +1836,28 @@ static int h3_resp_trailers_send(struct qcs *qcs, struct htx *htx) break; } - TRACE_LEAVE(H3_EV_TX_HDR, qcs->qcc->conn, qcs); + end: + TRACE_LEAVE(H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); return ret; err: - TRACE_DEVEL("leaving on error", H3_EV_TX_HDR, qcs->qcc->conn, qcs); + TRACE_DEVEL("leaving on error", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); return -1; } -/* Returns the total of bytes sent. This corresponds to the +/* Convert a series of HTX data blocks from buffer of size into + * HTTP/3 frames encoded into Tx buffer. The caller must also specify the + * underlying HTX area via as this will be used if zero-copy can be + * performed. + * + * Returns the total bytes of encoded HTTP/3 payload. This corresponds to the * total bytes of HTX block removed. A negative error code is returned in case * of a fatal error which should caused a connection closure. */ -static int h3_resp_data_send(struct qcs *qcs, struct buffer *buf, size_t count) +static int h3_resp_data_send(struct qcs *qcs, struct htx *htx, + struct buffer *buf, size_t count) { - struct htx *htx; - struct h3s *h3s = qcs->ctx; - struct h3c *h3c = h3s->h3c; + int err; struct buffer outbuf; struct buffer *res; size_t total = 0; @@ -1797,9 +1865,7 @@ static int h3_resp_data_send(struct qcs *qcs, struct buffer *buf, size_t count) struct htx_blk *blk; enum htx_blk_type type; - TRACE_ENTER(H3_EV_TX_DATA, qcs->qcc->conn, qcs); - - htx = htx_from_buf(buf); + TRACE_ENTER(H3_EV_TX_FRAME|H3_EV_TX_DATA, qcs->qcc->conn, qcs); new_frame: if (!count || htx_is_empty(htx)) @@ -1815,31 +1881,43 @@ static int h3_resp_data_send(struct qcs *qcs, struct buffer *buf, size_t count) if (type != HTX_BLK_DATA) goto end; - res = mux_get_buf(qcs); - if (b_is_null(res)) { - TRACE_ERROR("cannot allocate Tx buffer", H3_EV_TX_DATA, qcs->qcc->conn, qcs); - h3c->err = H3_INTERNAL_ERROR; - goto err; + if (!(res = qcc_get_stream_txbuf(qcs, &err))) { + if (err) { + TRACE_ERROR("cannot allocate Tx buffer", H3_EV_TX_FRAME|H3_EV_TX_DATA, qcs->qcc->conn, qcs); + goto err; + } + + /* Connection buf limit reached, stconn will subscribe on SEND. */ + TRACE_STATE("conn buf limit reached", H3_EV_TX_FRAME|H3_EV_TX_HDR, qcs->qcc->conn, qcs); + goto end; } + /* If HTX contains only one DATA block, try to exchange it with MUX + * buffer to perform zero-copy. This is only achievable if MUX buffer + * is currently empty. + */ if (unlikely(fsize == count && - !b_data(res) && - htx_nbblks(htx) == 1 && type == HTX_BLK_DATA)) { + !b_data(res) && + htx_nbblks(htx) == 1 && type == HTX_BLK_DATA)) { void *old_area = res->area; - /* map an H2 frame to the HTX block so that we can put the - * frame header there. - */ - *res = b_make(buf->area, buf->size, sizeof(struct htx) + blk->addr - hsize, fsize + hsize); - outbuf = b_make(b_head(res), hsize, 0, 0); - b_putchr(&outbuf, 0x00); /* h3 frame type = DATA */ - b_quic_enc_int(&outbuf, fsize, QUIC_VARINT_MAX_SIZE); /* h3 frame length */ + TRACE_DATA("perform zero-copy DATA transfer", + H3_EV_TX_FRAME|H3_EV_TX_DATA, qcs->qcc->conn, qcs); + + /* remap MUX buffer to HTX area, keep an offset for H3 header. */ + *res = b_make(buf->area, buf->size, + sizeof(struct htx) + blk->addr - hsize, 0); + + /* write H3 header frame before old HTX block. */ + b_putchr(res, 0x00); /* h3 frame type = DATA */ + b_quic_enc_int(res, fsize, QUIC_VARINT_MAX_SIZE); /* h3 frame length */ + b_add(res, fsize); - /* and exchange with our old area */ + /* assign old MUX area to HTX buffer. */ buf->area = old_area; buf->data = buf->head = 0; total += fsize; - fsize = 0; + goto end; } @@ -1851,23 +1929,29 @@ static int h3_resp_data_send(struct qcs *qcs, struct buffer *buf, size_t count) outbuf = b_make(b_tail(res), b_contig_space(res), 0, 0); if (b_size(&outbuf) > hsize || !b_space_wraps(res)) break; - b_slow_realign(res, trash.area, b_data(res)); + if (qcc_realign_stream_txbuf(qcs, res)) + break; } - /* Not enough room for headers and at least one data byte, block the - * stream. It is expected that the stream connector layer will subscribe - * on SEND. + /* Not enough room for headers and at least one data byte, try to + * release the current buffer and allocate a new one. If not possible, + * stconn layer will subscribe on SEND. */ if (b_size(&outbuf) <= hsize) { - TRACE_STATE("not enough room for data frame", H3_EV_TX_DATA, qcs->qcc->conn, qcs); - qcs->flags |= QC_SF_BLK_MROOM; - goto end; + TRACE_STATE("not enough room for data frame", H3_EV_TX_FRAME|H3_EV_TX_DATA, qcs->qcc->conn, qcs); + if (qcc_release_stream_txbuf(qcs)) + goto end; + + /* Buffer released, restart processing. */ + goto new_frame; } if (b_size(&outbuf) < hsize + fsize) fsize = b_size(&outbuf) - hsize; BUG_ON(fsize <= 0); + TRACE_DATA("encoding DATA frame", H3_EV_TX_FRAME|H3_EV_TX_DATA, + qcs->qcc->conn, qcs); b_putchr(&outbuf, 0x00); /* h3 frame type = DATA */ b_quic_enc_int(&outbuf, fsize, 0); /* h3 frame length */ @@ -1885,19 +1969,17 @@ static int h3_resp_data_send(struct qcs *qcs, struct buffer *buf, size_t count) goto new_frame; end: - TRACE_LEAVE(H3_EV_TX_DATA, qcs->qcc->conn, qcs); + TRACE_LEAVE(H3_EV_TX_FRAME|H3_EV_TX_DATA, qcs->qcc->conn, qcs); return total; err: BUG_ON(total); /* Must return HTX removed size if at least on frame encoded. */ - TRACE_DEVEL("leaving on error", H3_EV_TX_DATA, qcs->qcc->conn, qcs); + TRACE_DEVEL("leaving on error", H3_EV_TX_FRAME|H3_EV_TX_DATA, qcs->qcc->conn, qcs); return -1; } static size_t h3_snd_buf(struct qcs *qcs, struct buffer *buf, size_t count) { - struct h3s *h3s = qcs->ctx; - struct h3c *h3c = h3s->h3c; size_t total = 0; enum htx_blk_type btype; struct htx *htx; @@ -1910,12 +1992,7 @@ static size_t h3_snd_buf(struct qcs *qcs, struct buffer *buf, size_t count) htx = htx_from_buf(buf); - if (htx->extra && htx->extra == HTX_UNKOWN_PAYLOAD_LENGTH) - qcs->flags |= QC_SF_UNKNOWN_PL_LENGTH; - - while (count && !htx_is_empty(htx) && - !(qcs->flags & QC_SF_BLK_MROOM) && !h3c->err) { - + while (count && !htx_is_empty(htx) && qcc_stream_can_send(qcs) && ret >= 0) { idx = htx_get_head(htx); blk = htx_get_blk(htx, idx); btype = htx_get_blk_type(blk); @@ -1937,9 +2014,11 @@ static size_t h3_snd_buf(struct qcs *qcs, struct buffer *buf, size_t count) break; case HTX_BLK_DATA: - ret = h3_resp_data_send(qcs, buf, count); + ret = h3_resp_data_send(qcs, htx, buf, count); if (ret > 0) { + /* Reload HTX. This is necessary if 0-copy was performed. */ htx = htx_from_buf(buf); + total += ret; count -= ret; if (ret < bsize) @@ -1964,16 +2043,11 @@ static size_t h3_snd_buf(struct qcs *qcs, struct buffer *buf, size_t count) count -= bsize; break; } - - /* If an error occured, either buffer space or connection error - * must be set to break current loop. - */ - BUG_ON(ret < 0 && !(qcs->flags & QC_SF_BLK_MROOM) && !h3c->err); } - /* Interrupt sending on connection error. */ - if (unlikely(h3c->err)) { - qcc_set_error(qcs->qcc, h3c->err, 1); + /* Interrupt sending on fatal error. */ + if (unlikely(ret < 0)) { + qcc_set_error(qcs->qcc, H3_ERR_INTERNAL_ERROR, 1); goto out; } @@ -1998,7 +2072,7 @@ static size_t h3_snd_buf(struct qcs *qcs, struct buffer *buf, size_t count) /* Generate a STOP_SENDING if full response transferred before * receiving the full request. */ - qcs->err = H3_NO_ERROR; + qcs->err = H3_ERR_NO_ERROR; qcc_abort_stream_read(qcs); } @@ -2011,15 +2085,21 @@ static size_t h3_snd_buf(struct qcs *qcs, struct buffer *buf, size_t count) static size_t h3_nego_ff(struct qcs *qcs, size_t count) { + int err; struct buffer *res; int hsize; size_t sz, ret = 0; TRACE_ENTER(H3_EV_STRM_SEND, qcs->qcc->conn, qcs); - res = mux_get_buf(qcs); - if (b_is_null(res)) { - qcs->sd->iobuf.flags |= IOBUF_FL_NO_FF; + start: + if (!(res = qcc_get_stream_txbuf(qcs, &err))) { + if (err) { + qcs->sd->iobuf.flags |= IOBUF_FL_NO_FF; + goto end; + } + + qcs->sd->iobuf.flags |= IOBUF_FL_FF_BLOCKED; goto end; } @@ -2028,7 +2108,8 @@ static size_t h3_nego_ff(struct qcs *qcs, size_t count) while (1) { if (b_contig_space(res) >= hsize || !b_space_wraps(res)) break; - b_slow_realign(res, trash.area, b_data(res)); + if (qcc_realign_stream_txbuf(qcs, res)) + break; } /* Not enough room for headers and at least one data byte, block the @@ -2036,9 +2117,13 @@ static size_t h3_nego_ff(struct qcs *qcs, size_t count) * on SEND. */ if (b_contig_space(res) <= hsize) { - qcs->flags |= QC_SF_BLK_MROOM; - qcs->sd->iobuf.flags |= IOBUF_FL_FF_BLOCKED; - goto end; + if (qcc_release_stream_txbuf(qcs)) { + qcs->sd->iobuf.flags |= IOBUF_FL_FF_BLOCKED; + goto end; + } + + /* Buffer released, restart processing. */ + goto start; } /* Cannot forward more than available room in output buffer */ @@ -2064,6 +2149,8 @@ static size_t h3_done_ff(struct qcs *qcs) h3_debug_printf(stderr, "%s\n", __func__); if (qcs->sd->iobuf.data) { + TRACE_DATA("encoding DATA frame (fast forward)", + H3_EV_TX_FRAME|H3_EV_TX_DATA, qcs->qcc->conn, qcs); b_sub(qcs->sd->iobuf.buf, qcs->sd->iobuf.data); b_putchr(qcs->sd->iobuf.buf, 0x00); /* h3 frame type = DATA */ b_quic_enc_int(qcs->sd->iobuf.buf, qcs->sd->iobuf.data, QUIC_VARINT_MAX_SIZE); /* h3 frame length */ @@ -2105,7 +2192,8 @@ static int h3_close(struct qcs *qcs, enum qcc_app_ops_close_side side) */ if (qcs == h3c->ctrl_strm || h3s->type == H3S_T_CTRL) { TRACE_ERROR("closure detected on control stream", H3_EV_H3S_END, qcs->qcc->conn, qcs); - qcc_set_error(qcs->qcc, H3_CLOSED_CRITICAL_STREAM, 1); + qcc_set_error(qcs->qcc, H3_ERR_CLOSED_CRITICAL_STREAM, 1); + qcc_report_glitch(qcs->qcc, 1); return 1; } @@ -2136,7 +2224,7 @@ static int h3_attach(struct qcs *qcs, void *conn_ctx) */ TRACE_STATE("reject stream higher than goaway", H3_EV_H3S_NEW, qcs->qcc->conn, qcs); qcc_abort_stream_read(qcs); - qcc_reset_stream(qcs, H3_REQUEST_REJECTED); + qcc_reset_stream(qcs, H3_ERR_REQUEST_REJECTED); goto done; } @@ -2187,47 +2275,18 @@ static void h3_detach(struct qcs *qcs) TRACE_LEAVE(H3_EV_H3S_END, qcs->qcc->conn, qcs); } -/* Initialize H3 control stream and prepare SETTINGS emission. - * - * Returns 0 on success else non-zero. - */ -static int h3_finalize(void *ctx) -{ - struct h3c *h3c = ctx; - struct qcc *qcc = h3c->qcc; - struct qcs *qcs; - - TRACE_ENTER(H3_EV_H3C_NEW, qcc->conn); - - qcs = qcc_init_stream_local(h3c->qcc, 0); - if (!qcs) { - TRACE_ERROR("cannot init control stream", H3_EV_H3C_NEW, qcc->conn); - goto err; - } - - h3c->ctrl_strm = qcs; - - if (h3_control_send(qcs, h3c) < 0) - goto err; - - TRACE_LEAVE(H3_EV_H3C_NEW, qcc->conn); - return 0; - - err: - TRACE_DEVEL("leaving on error", H3_EV_H3C_NEW, qcc->conn); - return 1; -} - /* Generate a GOAWAY frame for connection on the control stream. * * Returns 0 on success else non-zero. */ static int h3_send_goaway(struct h3c *h3c) { + int err; struct qcs *qcs = h3c->ctrl_strm; struct buffer pos, *res; unsigned char data[3 * QUIC_VARINT_MAX_SIZE]; size_t frm_len = quic_int_getsize(h3c->id_goaway); + size_t xfer; TRACE_ENTER(H3_EV_H3C_END, h3c->qcc->conn); @@ -2242,15 +2301,16 @@ static int h3_send_goaway(struct h3c *h3c) b_quic_enc_int(&pos, frm_len, 0); b_quic_enc_int(&pos, h3c->id_goaway, 0); - res = mux_get_buf(qcs); - if (b_is_null(res) || b_room(res) < b_data(&pos)) { - /* Do not try forcefully to emit GOAWAY if no space left. */ + res = qcc_get_stream_txbuf(qcs, &err); + if (!res || b_room(res) < b_data(&pos) || + qfctl_sblocked(&qcs->tx.fc) || qfctl_sblocked(&h3c->qcc->tx.fc)) { + /* Do not try forcefully to emit GOAWAY if no buffer available or not enough space left. */ TRACE_ERROR("cannot send GOAWAY", H3_EV_H3C_END, h3c->qcc->conn, qcs); goto err; } - b_force_xfer(res, &pos, b_data(&pos)); - qcc_send_stream(qcs, 1); + xfer = b_force_xfer(res, &pos, b_data(&pos)); + qcc_send_stream(qcs, 1, xfer); h3c->flags |= H3_CF_GOAWAY_SENT; TRACE_LEAVE(H3_EV_H3C_END, h3c->qcc->conn); @@ -2271,7 +2331,7 @@ static int h3_send_goaway(struct h3c *h3c) static int h3_init(struct qcc *qcc) { struct h3c *h3c; - struct quic_conn *qc = qcc->conn->handle.qc; + const struct listener *li = __objt_listener(qcc->conn->target); TRACE_ENTER(H3_EV_H3C_NEW, qcc->conn); @@ -2288,9 +2348,8 @@ static int h3_init(struct qcc *qcc) h3c->id_goaway = 0; qcc->ctx = h3c; - /* TODO cleanup only ref to quic_conn */ h3c->prx_counters = - EXTRA_COUNTERS_GET(qc->li->bind_conf->frontend->extra_counters_fe, + EXTRA_COUNTERS_GET(li->bind_conf->frontend->extra_counters_fe, &h3_stats_module); LIST_INIT(&h3c->buf_wait.list); @@ -2298,10 +2357,43 @@ static int h3_init(struct qcc *qcc) return 1; fail_no_h3: + qcc_set_error(qcc, H3_ERR_INTERNAL_ERROR, 1); TRACE_DEVEL("leaving on error", H3_EV_H3C_NEW, qcc->conn); return 0; } +/* Initialize H3 control stream and prepare SETTINGS emission. + * + * Returns 0 on success else non-zero. + */ +static int h3_finalize(void *ctx) +{ + struct h3c *h3c = ctx; + struct qcc *qcc = h3c->qcc; + struct qcs *qcs; + + TRACE_ENTER(H3_EV_H3C_NEW, qcc->conn); + + qcs = qcc_init_stream_local(qcc, 0); + if (!qcs) { + TRACE_ERROR("cannot init control stream", H3_EV_H3C_NEW, qcc->conn); + goto err; + } + + h3c->ctrl_strm = qcs; + + if (h3_control_send(qcs, h3c) < 0) + goto err; + + TRACE_LEAVE(H3_EV_H3C_NEW, qcc->conn); + return 0; + + err: + qcc_set_error(qcc, H3_ERR_INTERNAL_ERROR, 1); + TRACE_DEVEL("leaving on error", H3_EV_H3C_NEW, qcc->conn); + return 1; +} + /* Send a HTTP/3 GOAWAY followed by a CONNECTION_CLOSE_APP. */ static void h3_shutdown(void *ctx) { @@ -2324,7 +2416,7 @@ static void h3_shutdown(void *ctx) * graceful shutdown SHOULD use the H3_NO_ERROR error code when closing * the connection. */ - h3c->qcc->err = quic_err_app(H3_NO_ERROR); + h3c->qcc->err = quic_err_app(H3_ERR_NO_ERROR); TRACE_LEAVE(H3_EV_H3C_END, h3c->qcc->conn); } @@ -2343,6 +2435,12 @@ static void h3_stats_inc_err_cnt(void *ctx, int err_code) h3_inc_err_cnt(h3c->prx_counters, err_code); } +static void h3_report_susp(void *ctx) +{ + struct h3c *h3c = ctx; + h3c->qcc->err = quic_err_app(H3_ERR_EXCESSIVE_LOAD); +} + static inline const char *h3_ft_str(uint64_t type) { switch (type) { @@ -2389,15 +2487,16 @@ static void h3_trace(enum trace_level level, uint64_t mask, /* HTTP/3 application layer operations */ const struct qcc_app_ops h3_ops = { .init = h3_init, + .finalize = h3_finalize, .attach = h3_attach, - .decode_qcs = h3_decode_qcs, + .rcv_buf = h3_rcv_buf, .snd_buf = h3_snd_buf, .nego_ff = h3_nego_ff, .done_ff = h3_done_ff, .close = h3_close, .detach = h3_detach, - .finalize = h3_finalize, .shutdown = h3_shutdown, .inc_err_cnt = h3_stats_inc_err_cnt, + .report_susp = h3_report_susp, .release = h3_release, }; -- cgit v1.2.3