From b46aad6df449445a9fc4aa7b32bd40005438e3f7 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 13 Apr 2024 14:18:05 +0200 Subject: Adding upstream version 2.9.5. Signed-off-by: Daniel Baumann --- dev/base64/base64rev-gen.c | 70 +++ dev/coccinelle/bug_on.cocci | 7 + dev/coccinelle/cs_endp_flags.cocci | 76 +++ dev/coccinelle/endp_flags.cocci | 76 +++ dev/coccinelle/ha_free.cocci | 6 + dev/coccinelle/ist.cocci | 86 +++ dev/coccinelle/realloc_leak.cocci | 6 + dev/coccinelle/strcmp.cocci | 309 +++++++++++ dev/coccinelle/xalloc_cast.cocci | 11 + dev/coccinelle/xalloc_size.cocci | 41 ++ dev/flags/README | 12 + dev/flags/flags.c | 157 ++++++ dev/flags/show-fd-to-flags.sh | 2 + dev/flags/show-sess-to-flags.sh | 209 +++++++ dev/h2/mkhdr.sh | 151 ++++++ dev/haring/README | 9 + dev/haring/haring.c | 266 +++++++++ dev/hpack/README | 4 + dev/hpack/decode.c | 215 ++++++++ dev/hpack/gen-enc.c | 205 +++++++ dev/hpack/gen-rht.c | 369 +++++++++++++ dev/plug_qdisc/README | 59 ++ dev/plug_qdisc/plug_qdisc.c | 86 +++ dev/poll/Makefile | 13 + dev/poll/poll.c | 445 +++++++++++++++ dev/qpack/decode.c | 171 ++++++ dev/sslkeylogger/sslkeylogger.lua | 47 ++ dev/tcploop/Makefile | 13 + dev/tcploop/tcploop.c | 1055 ++++++++++++++++++++++++++++++++++++ dev/trace/trace.awk | 78 +++ dev/udp/udp-perturb.c | 527 ++++++++++++++++++ 31 files changed, 4781 insertions(+) create mode 100644 dev/base64/base64rev-gen.c create mode 100644 dev/coccinelle/bug_on.cocci create mode 100644 dev/coccinelle/cs_endp_flags.cocci create mode 100644 dev/coccinelle/endp_flags.cocci create mode 100644 dev/coccinelle/ha_free.cocci create mode 100644 dev/coccinelle/ist.cocci create mode 100644 dev/coccinelle/realloc_leak.cocci create mode 100644 dev/coccinelle/strcmp.cocci create mode 100644 dev/coccinelle/xalloc_cast.cocci create mode 100644 dev/coccinelle/xalloc_size.cocci create mode 100644 dev/flags/README create mode 100644 dev/flags/flags.c create mode 100755 dev/flags/show-fd-to-flags.sh create mode 100755 dev/flags/show-sess-to-flags.sh create mode 100755 dev/h2/mkhdr.sh create mode 100644 dev/haring/README create mode 100644 dev/haring/haring.c create mode 100644 dev/hpack/README create mode 100644 dev/hpack/decode.c create mode 100644 dev/hpack/gen-enc.c create mode 100644 dev/hpack/gen-rht.c create mode 100644 dev/plug_qdisc/README create mode 100644 dev/plug_qdisc/plug_qdisc.c create mode 100644 dev/poll/Makefile create mode 100644 dev/poll/poll.c create mode 100644 dev/qpack/decode.c create mode 100644 dev/sslkeylogger/sslkeylogger.lua create mode 100644 dev/tcploop/Makefile create mode 100644 dev/tcploop/tcploop.c create mode 100755 dev/trace/trace.awk create mode 100644 dev/udp/udp-perturb.c (limited to 'dev') diff --git a/dev/base64/base64rev-gen.c b/dev/base64/base64rev-gen.c new file mode 100644 index 0000000..faffc87 --- /dev/null +++ b/dev/base64/base64rev-gen.c @@ -0,0 +1,70 @@ +/* + * base64rev generator + * + * Copyright 2009-2010 Krzysztof Piotr Oledzki + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + * + */ + +#include + +const char base64tab[65]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +char base64rev[128]; + +#define base '#' /* arbitrary chosen base value */ +#define B64MAX 64 +#define B64PADV B64MAX + +int main() { + char *p, c; + int i, min = 255, max = 0; + + for (i = 0; i < sizeof(base64rev); i++) + base64rev[i] = base; + + for (i = 0; i < B64MAX; i++) { + c = base64tab[i]; + + if (min > c) + min = c; + + if (max < c) + max = c; + } + + for (i = 0; i < B64MAX; i++) { + c = base64tab[i]; + + if (base+i+1 > 127) { + printf("Wrong base value @%d\n", i); + return 1; + } + + base64rev[c - min] = base+i+1; + } + + base64rev['=' - min] = base + B64PADV; + + base64rev[max - min + 1] = '\0'; + + printf("#define B64BASE '%c'\n", base); + printf("#define B64CMIN '%c'\n", min); + printf("#define B64CMAX '%c'\n", max); + printf("#define B64PADV %u\n", B64PADV); + + p = base64rev; + printf("const char base64rev[]=\""); + for (p = base64rev; *p; p++) { + if (*p == '\\') + printf("\\%c", *p); + else + printf("%c", *p); + } + printf("\"\n"); + + return 0; +} diff --git a/dev/coccinelle/bug_on.cocci b/dev/coccinelle/bug_on.cocci new file mode 100644 index 0000000..3837879 --- /dev/null +++ b/dev/coccinelle/bug_on.cocci @@ -0,0 +1,7 @@ +@@ +expression E; +@@ + +- if (E) +- ABORT_NOW(); ++ BUG_ON(E); diff --git a/dev/coccinelle/cs_endp_flags.cocci b/dev/coccinelle/cs_endp_flags.cocci new file mode 100644 index 0000000..639d321 --- /dev/null +++ b/dev/coccinelle/cs_endp_flags.cocci @@ -0,0 +1,76 @@ +@@ +struct conn_stream *cs; +expression e; +@@ +( +- (cs->endp->flags & (e)) ++ sc_ep_test(cs, e) +| +- (cs->endp->flags & e) ++ sc_ep_test(cs, e) +| +- cs->endp->flags & (e) ++ sc_ep_test(cs, e) +| +- cs->endp->flags & e ++ sc_ep_test(cs, e) +) + +@@ +struct conn_stream *cs; +expression e; +@@ +( +- cs->endp->flags |= (e) ++ sc_ep_set(cs, e) +| +- cs->endp->flags |= e ++ sc_ep_set(cs, e) +) + +@@ +struct conn_stream *cs; +expression e; +@@ +( +- cs->endp->flags &= ~(e) ++ sc_ep_clr(cs, e) +| +- cs->endp->flags &= (e) ++ sc_ep_clr(cs, ~e) +| +- cs->endp->flags &= ~e ++ sc_ep_clr(cs, e) +| +- cs->endp->flags &= e ++ sc_ep_clr(cs, ~e) +) + +@@ +struct conn_stream *cs; +@@ +- cs->endp->flags = 0 ++ sc_ep_zero(cs) + +@@ +struct conn_stream *cs; +expression e; +@@ +( +- cs->endp->flags = (e) ++ sc_ep_setall(cs, e) +| +- cs->endp->flags = e ++ sc_ep_setall(cs, e) +) + +@@ +struct conn_stream *cs; +@@ +( +- (cs->endp->flags) ++ sc_ep_get(cs) +| +- cs->endp->flags ++ sc_ep_get(cs) +) diff --git a/dev/coccinelle/endp_flags.cocci b/dev/coccinelle/endp_flags.cocci new file mode 100644 index 0000000..fceda27 --- /dev/null +++ b/dev/coccinelle/endp_flags.cocci @@ -0,0 +1,76 @@ +@@ +struct cs_endpoint *endp; +expression e; +@@ +( +- (endp->flags & (e)) ++ se_fl_test(endp, e) +| +- (endp->flags & e) ++ se_fl_test(endp, e) +| +- endp->flags & (e) ++ se_fl_test(endp, e) +| +- endp->flags & e ++ se_fl_test(endp, e) +) + +@@ +struct cs_endpoint *endp; +expression e; +@@ +( +- endp->flags |= (e) ++ se_fl_set(endp, e) +| +- endp->flags |= e ++ se_fl_set(endp, e) +) + +@@ +struct cs_endpoint *endp; +expression e; +@@ +( +- endp->flags &= ~(e) ++ se_fl_clr(endp, e) +| +- endp->flags &= (e) ++ se_fl_clr(endp, ~e) +| +- endp->flags &= ~e ++ se_fl_clr(endp, e) +| +- endp->flags &= e ++ se_fl_clr(endp, ~e) +) + +@@ +struct cs_endpoint *endp; +@@ +- endp->flags = 0 ++ se_fl_zero(endp) + +@@ +struct cs_endpoint *endp; +expression e; +@@ +( +- endp->flags = (e) ++ se_fl_setall(endp, e) +| +- endp->flags = e ++ se_fl_setall(endp, e) +) + +@@ +struct cs_endpoint *endp; +@@ +( +- (endp->flags) ++ se_fl_get(endp) +| +- endp->flags ++ se_fl_get(endp) +) diff --git a/dev/coccinelle/ha_free.cocci b/dev/coccinelle/ha_free.cocci new file mode 100644 index 0000000..0019039 --- /dev/null +++ b/dev/coccinelle/ha_free.cocci @@ -0,0 +1,6 @@ +@ rule @ +expression E; +@@ +- free(E); +- E = NULL; ++ ha_free(&E); diff --git a/dev/coccinelle/ist.cocci b/dev/coccinelle/ist.cocci new file mode 100644 index 0000000..acde626 --- /dev/null +++ b/dev/coccinelle/ist.cocci @@ -0,0 +1,86 @@ +@@ +struct ist i; +expression p, l; +@@ + +( +- i.ptr = p; +- i.len = strlen(i.ptr); ++ i = ist(p); +| +- i.ptr = p; +- i.len = l; ++ i = ist2(p, l); +) + +@@ +@@ + +- ist2(NULL, 0) ++ IST_NULL + +@@ +struct ist i; +expression e; +@@ + +- i.ptr += e; +- i.len -= e; ++ i = istadv(i, e); + +@@ +struct ist i; +@@ + +- i = istadv(i, 1); ++ i = istnext(i); + +@@ +struct ist i; +@@ + +- i.ptr++; +- i.len--; ++ i = istnext(i); + +@@ +struct ist i; +@@ + +- (\(i.ptr\|istptr(i)\) + \(i.len\|istlen(i)\)) ++ istend(i) + +@@ +struct ist i; +expression e; +@@ + +- if (\(i.len\|istlen(i)\) > e) { i.len = e; } ++ i = isttrim(i, e); + +@@ +struct ist i; +struct buffer *b; +@@ + +- chunk_memcat(b, \(i.ptr\|istptr(i)\) , \(i.len\|istlen(i)\)); ++ chunk_istcat(b, i); + +@@ +struct ist i; +@@ + +- i.ptr != NULL ++ isttest(i) + +@@ +char *s; +@@ + +( +- ist2(s, strlen(s)) ++ ist(s) +| +- ist2(strdup(s), strlen(s)) ++ ist(strdup(s)) +) diff --git a/dev/coccinelle/realloc_leak.cocci b/dev/coccinelle/realloc_leak.cocci new file mode 100644 index 0000000..c201b80 --- /dev/null +++ b/dev/coccinelle/realloc_leak.cocci @@ -0,0 +1,6 @@ +@@ +expression E; +expression F; +@@ + +* E = realloc(E, F); diff --git a/dev/coccinelle/strcmp.cocci b/dev/coccinelle/strcmp.cocci new file mode 100644 index 0000000..f6064bf --- /dev/null +++ b/dev/coccinelle/strcmp.cocci @@ -0,0 +1,309 @@ +@@ +statement S; +expression E; +expression F; +@@ + + if ( +( +dns_hostname_cmp +| +eb_memcmp +| +memcmp +| +strcasecmp +| +strcmp +| +strncasecmp +| +strncmp +) +- (E, F) ++ (E, F) != 0 + ) +( + S +| + { ... } +) + +@@ +statement S; +expression E; +expression F; +@@ + + if ( +- ! +( +dns_hostname_cmp +| +eb_memcmp +| +memcmp +| +strcasecmp +| +strcmp +| +strncasecmp +| +strncmp +) +- (E, F) ++ (E, F) == 0 + ) +( + S +| + { ... } +) + +@@ +expression E; +expression F; +expression G; +@@ + +( +G && +( +dns_hostname_cmp +| +eb_memcmp +| +memcmp +| +strcasecmp +| +strcmp +| +strncasecmp +| +strncmp +) +- (E, F) ++ (E, F) != 0 +) + +@@ +expression E; +expression F; +expression G; +@@ + +( +G || +( +dns_hostname_cmp +| +eb_memcmp +| +memcmp +| +strcasecmp +| +strcmp +| +strncasecmp +| +strncmp +) +- (E, F) ++ (E, F) != 0 +) + +@@ +expression E; +expression F; +expression G; +@@ + +( +( +dns_hostname_cmp +| +eb_memcmp +| +memcmp +| +strcasecmp +| +strcmp +| +strncasecmp +| +strncmp +) +- (E, F) ++ (E, F) != 0 +&& G +) + +@@ +expression E; +expression F; +expression G; +@@ + +( +( +dns_hostname_cmp +| +eb_memcmp +| +memcmp +| +strcasecmp +| +strcmp +| +strncasecmp +| +strncmp +) +- (E, F) ++ (E, F) != 0 +|| G +) + +@@ +expression E; +expression F; +expression G; +@@ + +( +G && +- ! +( +dns_hostname_cmp +| +eb_memcmp +| +memcmp +| +strcasecmp +| +strcmp +| +strncasecmp +| +strncmp +) +- (E, F) ++ (E, F) == 0 +) + +@@ +expression E; +expression F; +expression G; +@@ + +( +G || +- ! +( +dns_hostname_cmp +| +eb_memcmp +| +memcmp +| +strcasecmp +| +strcmp +| +strncasecmp +| +strncmp +) +- (E, F) ++ (E, F) == 0 +) + +@@ +expression E; +expression F; +expression G; +@@ + +( +- ! +( +dns_hostname_cmp +| +eb_memcmp +| +memcmp +| +strcasecmp +| +strcmp +| +strncasecmp +| +strncmp +) +- (E, F) ++ (E, F) == 0 +&& G +) + +@@ +expression E; +expression F; +expression G; +@@ + +( +- ! +( +dns_hostname_cmp +| +eb_memcmp +| +memcmp +| +strcasecmp +| +strcmp +| +strncasecmp +| +strncmp +) +- (E, F) ++ (E, F) == 0 +|| G +) + +@@ +expression E; +expression F; +expression G; +@@ + +( +- ! +( +dns_hostname_cmp +| +eb_memcmp +| +memcmp +| +strcasecmp +| +strcmp +| +strncasecmp +| +strncmp +) +- (E, F) ++ (E, F) == 0 +) diff --git a/dev/coccinelle/xalloc_cast.cocci b/dev/coccinelle/xalloc_cast.cocci new file mode 100644 index 0000000..75baa00 --- /dev/null +++ b/dev/coccinelle/xalloc_cast.cocci @@ -0,0 +1,11 @@ +@@ +type T; +@@ + +- (T*) +( +malloc +| +calloc +) + (...) diff --git a/dev/coccinelle/xalloc_size.cocci b/dev/coccinelle/xalloc_size.cocci new file mode 100644 index 0000000..80808e3 --- /dev/null +++ b/dev/coccinelle/xalloc_size.cocci @@ -0,0 +1,41 @@ +@@ +type T; +expression E; +expression t; +@@ + +( + t = calloc(E, sizeof(*t)) +| +- t = calloc(E, sizeof(T)) ++ t = calloc(E, sizeof(*t)) +) + +@@ +type T; +T *x; +@@ + + x = malloc( +- sizeof(T) ++ sizeof(*x) + ) + +@@ +type T; +T *x; +@@ + + x = calloc(1, +- sizeof(T) ++ sizeof(*x) + ) + +@@ +@@ + + calloc( ++ 1, + ... +- ,1 + ) diff --git a/dev/flags/README b/dev/flags/README new file mode 100644 index 0000000..f3730c7 --- /dev/null +++ b/dev/flags/README @@ -0,0 +1,12 @@ +This needs to be built from the top makefile, for example : + + make dev/flags/flags + +Then the executable is usable either one value at a time from the +command line, either with values coming from stdin with "-" passed +alone instead of the value. + +It is possible to restrict the decoding to certain fields only by +specifying one of "ana", "chn", "conn", "sc", "si", "sierr", "strm", +"task", or "txn" before the value. + diff --git a/dev/flags/flags.c b/dev/flags/flags.c new file mode 100644 index 0000000..65af237 --- /dev/null +++ b/dev/flags/flags.c @@ -0,0 +1,157 @@ +#include +#include + +/* make the include files below expose their flags */ +#define HA_EXPOSE_FLAGS + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// 1 bit per flag, no hole permitted here +#define SHOW_AS_ANA 0x00000001 +#define SHOW_AS_CHN 0x00000002 +#define SHOW_AS_CONN 0x00000004 +#define SHOW_AS_SC 0x00000008 +#define SHOW_AS_SET 0x00000010 +#define SHOW_AS_STRM 0x00000020 +#define SHOW_AS_TASK 0x00000040 +#define SHOW_AS_TXN 0x00000080 +#define SHOW_AS_SD 0x00000100 +#define SHOW_AS_HSL 0x00000200 +#define SHOW_AS_HTX 0x00000400 +#define SHOW_AS_HMSG 0x00000800 +#define SHOW_AS_FD 0x00001000 +#define SHOW_AS_H2C 0x00002000 +#define SHOW_AS_H2S 0x00004000 +#define SHOW_AS_H1C 0x00008000 +#define SHOW_AS_H1S 0x00010000 +#define SHOW_AS_FCONN 0x00020000 +#define SHOW_AS_FSTRM 0x00040000 + +// command line names, must be in exact same order as the SHOW_AS_* flags above +// so that show_as_words[i] matches flag 1U< 0) { + argv++; argc--; + if (argc < 1) + usage_exit(name); + + f = get_show_as(argv[0]); + if (!f) + break; + show_as |= f; + } + + if (!show_as) + show_as = ~0U; + + if (argc > 1) + multi = 1; + + if (strcmp(argv[0], "-") == 0) + use_stdin = 1; + + while (argc > 0) { + if (use_stdin) { + value = fgets(line, sizeof(line), stdin); + if (!value) + break; + + /* skip common leading delimiters that slip from copy-paste */ + while (*value == ' ' || *value == '\t' || *value == ':' || *value == '=') + value++; + + /* stop at the end of the number and trim any C suffix like "UL" */ + err = value; + while (*err == '-' || *err == '+' || + (isalnum((unsigned char)*err) && toupper((unsigned char)*err) != 'U' && toupper((unsigned char)*err) != 'L')) + err++; + *err = 0; + } else { + value = argv[0]; + argv++; argc--; + } + + flags = strtoul(value, &err, 0); + if (!*value || *err) { + fprintf(stderr, "Unparsable value: <%s>\n", value); + usage_exit(name); + } + + if (multi || use_stdin) + printf("### 0x%08x:\n", flags); + + if (show_as & SHOW_AS_ANA) printf("chn->ana = %s\n", (chn_show_analysers(buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_CHN) printf("chn->flags = %s\n", (chn_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_CONN) printf("conn->flags = %s\n", (conn_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_SC) printf("sc->flags = %s\n", (sc_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_SD) printf("sd->flags = %s\n", (se_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_SET) printf("strm->et = %s\n", (strm_et_show_flags(buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_STRM) printf("strm->flags = %s\n", (strm_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_TASK) printf("task->state = %s\n", (task_show_state (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_TXN) printf("txn->flags = %s\n", (txn_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_HSL) printf("sl->flags = %s\n", (hsl_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_HTX) printf("htx->flags = %s\n", (htx_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_HMSG) printf("hmsg->flags = %s\n", (hmsg_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_FD) printf("fd->flags = %s\n", (fd_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_H2C) printf("h2c->flags = %s\n", (h2c_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_H2S) printf("h2s->flags = %s\n", (h2s_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_H1C) printf("h1c->flags = %s\n", (h1c_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_H1S) printf("h1s->flags = %s\n", (h1s_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_FCONN) printf("fconn->flags = %s\n",(fconn_show_flags (buf, bsz, " | ", flags), buf)); + if (show_as & SHOW_AS_FSTRM) printf("fstrm->flags = %s\n",(fstrm_show_flags (buf, bsz, " | ", flags), buf)); + } + return 0; +} diff --git a/dev/flags/show-fd-to-flags.sh b/dev/flags/show-fd-to-flags.sh new file mode 100755 index 0000000..29757c3 --- /dev/null +++ b/dev/flags/show-fd-to-flags.sh @@ -0,0 +1,2 @@ +#!/bin/sh +awk '{print $12}' | grep cflg= | sort | uniq -c | sort -nr | while read a b; do c=${b##*=}; d=$(${0%/*}/flags conn $c);d=${d##*= }; printf "%6d %s %s\n" $a "$b" "$d";done diff --git a/dev/flags/show-sess-to-flags.sh b/dev/flags/show-sess-to-flags.sh new file mode 100755 index 0000000..79003a4 --- /dev/null +++ b/dev/flags/show-sess-to-flags.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash + +# This script is used to resolve various flags that appear on "show sess all". +# All identified ones will be appended at the end, with a short name and their +# value, followed by either the value resolved by "flags" when it's found, or +# by the copy-pastable command to use to resolve them. The path to FLAGS is +# searched in this order: 1) $FLAGS, 2) in the path, 3) dev/flags/flags, 4) +# in the same directory as the script. +# +# This script is horrendous, but it's not a reason for making it even more +# disgusting. The big regex flag mapping mess at the end is readable on a +# large screen and it's easier to spot mistakes using this aligned format, +# so please preserve this as much as possible and avoid multi-line formats. +# +# The append_* functions provide different variants that are still commented +# out. It's mostly a matter of taste, they're equivalent. +# +# Usage: socat /path/to/socket - <<< "show sess all" | ./$0 > output +# +# options: +# --color=never, --no-color: never colorize output +# --color=always: always colorize output (default: only on terminal) + +# look for "flags in path then in dev/flags/flags then next to the script" +FLAGS="${FLAGS:-$(command -v flags)}" +if [ -z "$FLAGS" ]; then + if [ -e dev/flags/flags ]; then + FLAGS=dev/flags/flags; + elif [ -e "${0%/*}/flags" ]; then + FLAGS="${0%/*}/flags" + else + # OK still not found,let's write a copy-pastable command + FLAGS="echo ./flags" + fi +fi + +HTTP_METH=( "OPTIONS" "GET" "HEAD" "POST" "PUT" "DELETE" "TRACE" "CONNECT" "OTHER" ) +out=( ) +decode=( ) + +# returns str $2 and $3 concatenated with enough spaces in between so that the +# total size doesn't exceed $1 chars, but always inserts at least one space. +justify() { + local pad=" " + local str + + while str="${2}${pad}${3}" && [ ${#str} -le $1 ]; do + pad="${pad} " + done + echo -n "$str" +} + +# remove spaces at the beginning and end in "$1" +trim() { + while [ -n "$1" -a -z "${1## *}" ]; do + set -- "${1# }" + done + while [ -n "$1" -a -z "${1%%* }" ]; do + set -- "${1% }" + done + echo -n "$1" +} + +# pass $1=ctx name, $2=argname, $3=value, append the decoded line to decode[] +append_flag() { + set -- "$1" "$2" "$(printf "%#x" $3)" + #decode[${#decode[@]}]="$1=$3 [ $(set -- $($FLAGS $2 $3 | cut -f2- -d=); echo $*) ]" + #decode[${#decode[@]}]="$(printf "%-14s %10s %s" $1 $3 "$(set -- $($FLAGS $2 $3 | cut -f2- -d=); echo $*)")" + #decode[${#decode[@]}]="$(justify 22 "$1" "$3") $(set -- $($FLAGS $2 $3 | cut -f2- -d=); echo $*)" + decode[${#decode[@]}]="$(justify 22 "$1" "$3") $(set -- $($FLAGS $2 $3 | cut -f2- -d= | tr -d '|'); echo "$*")" + #decode[${#decode[@]}]="$(justify 22 "$1" "$3") $(set -- $($FLAGS $2 $(printf "%#x" $3) | cut -f2- -d= | tr -d '|'); echo "$*")" + #decode[${#decode[@]}]="$(justify 22 "$1" "$3") $(trim "$($FLAGS $2 $3 | cut -f2- -d= | tr -d '|')")" + #decode[${#decode[@]}]="$(justify 22 "$1" "$3") $(trim "$($FLAGS $2 $3 | cut -f2- -d= | tr -d ' ')")" +} + +# pass $1=ctx name, $2=value, $3=decoded value +append_str() { + #decode[${#decode[@]}]="$1=$2 [ $3 ]" + #decode[${#decode[@]}]="$(printf "%-14s %10s %s" $1 $2 $3)" + decode[${#decode[@]}]="$(justify 22 "$1" "$2") $(trim $3)" +} + +# dump and reset the buffers +dump_and_reset() { + local line + + line=0 + while [ $line -lt ${#out[@]} ]; do + if [ -n "$COLOR" ]; then + # highlight name=value for values made of upper case letters + echo "${out[$line]}" | \ + sed -e 's,\(^0x.*\),\x1b[1;37m\1\x1b[0m,g' \ + -e 's,\([^ ,=]*\)=\([A-Z][^:, ]*\),\x1b[1;36m\1\x1b[0m=\x1b[1;33m\2\x1b[0m,g' + + else + echo "${out[$line]}" + fi + ((line++)) + done + + [ ${#decode[@]} -eq 0 ] || echo " -----------------------------------" + + line=0 + while [ $line -lt ${#decode[@]} ]; do + echo " ${decode[$line]}" + ((line++, total++)) + done + + [ ${#decode[@]} -eq 0 ] || echo " -----------------------------------" + + decode=( ) + out=( ) +} + +### main entry point + +if [ -t 1 ]; then + # terminal on stdout, enable color by default + COLOR=1 +else + COLOR= +fi + +if [ "$1" == "--no-color" -o "$1" == "--color=never" ]; then + shift + COLOR= +elif [ "$1" == "--color=always" ]; then + shift + COLOR=1 +fi + +ctx=strm +while read -r; do + [ "$REPLY" != "EOF" ] || break # for debugging + + if [[ "$REPLY" =~ ^[[:blank:]]*task= ]]; then + ctx=task; + elif [[ "$REPLY" =~ ^[[:blank:]]*txn= ]]; then + ctx=txn; + elif [[ "$REPLY" =~ ^[[:blank:]]*scf= ]]; then + ctx=scf; + elif [[ "$REPLY" =~ ^[[:blank:]]*co0= ]]; then + ctx=cof; + elif [[ "$REPLY" =~ ^[[:blank:]]*app0= ]]; then + ctx=appf; + elif [[ "$REPLY" =~ ^[[:blank:]]*req= ]]; then + ctx=req; + elif [[ "$REPLY" =~ ^[[:blank:]]*scb= ]]; then + ctx=scb; + elif [[ "$REPLY" =~ ^[[:blank:]]*co1= ]]; then + ctx=cob; + elif [[ "$REPLY" =~ ^[[:blank:]]*app1= ]]; then + ctx=appb; + elif [[ "$REPLY" =~ ^[[:blank:]]*res= ]]; then + ctx=res; + elif [[ "$REPLY" =~ ^0x ]]; then + # here we dump what we have and we reset + dump_and_reset + ctx=strm; + fi + + if [ $ctx = strm ]; then + ! [[ "$REPLY" =~ [[:blank:]]flags=([0-9a-fx]*) ]] || append_flag strm.flg strm "${BASH_REMATCH[1]}" + elif [ $ctx = task ]; then + ! [[ "$REPLY" =~ \(state=([0-9a-fx]*) ]] || append_flag task.state task "${BASH_REMATCH[1]}" + elif [ $ctx = txn ]; then + ! [[ "$REPLY" =~ [[:blank:]]meth=([^[:blank:]]*) ]] || append_str txn.meth "${BASH_REMATCH[1]}" "${HTTP_METH[$((${BASH_REMATCH[1]}))]}" + ! [[ "$REPLY" =~ [[:blank:]]flags=([0-9a-fx]*) ]] || append_flag txn.flg txn "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]req\.f=([0-9a-fx]*) ]] || append_flag txn.req.flg hmsg "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]rsp\.f=([0-9a-fx]*) ]] || append_flag txn.rsp.flg hmsg "${BASH_REMATCH[1]}" + elif [ $ctx = scf ]; then + ! [[ "$REPLY" =~ [[:blank:]]flags=([0-9a-fx]*) ]] || append_flag f.sc.flg sc "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]endp=[[:alnum:]]*,[[:alnum:]]*,([0-9a-fx]*) ]] || append_flag f.sc.sd.flg sd "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]h1s.*\.sd\.flg=([0-9a-fx]*) ]] || append_flag f.h1s.sd.flg sd "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]h1s\.flg=([0-9a-fx]*) ]] || append_flag f.h1s.flg h1s "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]h1c\.flg=([0-9a-fx]*) ]] || append_flag f.h1c.flg h1c "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ ^[[:blank:]]*\.sc=.*\.flg=.*\.app=.*\.sd=[^=]*\.flg=([0-9a-fx]*) ]] || append_flag f.h2s.sd.flg sd "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]h2s.*\.flg=([0-9a-fx]*) ]] || append_flag f.h2s.flg h2s "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]h2c.*\.flg=([0-9a-fx]*) ]] || append_flag f.h2c.flg h2c "${BASH_REMATCH[1]}" + elif [ $ctx = cof ]; then + ! [[ "$REPLY" =~ [[:blank:]]flags=([0-9a-fx]*) ]] || append_flag f.co.flg conn "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]fd.state=([0-9a-fx]*) ]] || append_flag f.co.fd.st fd 0x"${BASH_REMATCH[1]#0x}" + elif [ $ctx = req ]; then + ! [[ "$REPLY" =~ [[:blank:]]\(f=([0-9a-fx]*) ]] || append_flag req.flg chn "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]an=([0-9a-fx]*) ]] || append_flag req.ana ana "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]htx.*flags=([0-9a-fx]*) ]] || append_flag req.htx.flg htx "${BASH_REMATCH[1]}" + elif [ $ctx = scb ]; then + ! [[ "$REPLY" =~ [[:blank:]]flags=([0-9a-fx]*) ]] || append_flag b.sc.flg sc "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]endp=[[:alnum:]]*,[[:alnum:]]*,([0-9a-fx]*) ]] || append_flag b.sc.sd.flg sd "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]h1s.*\.sd\.flg=([0-9a-fx]*) ]] || append_flag b.h1s.sd.flg sd "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]h1s\.flg=([0-9a-fx]*) ]] || append_flag b.h1s.flg h1s "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]h1c\.flg=([0-9a-fx]*) ]] || append_flag b.h1c.flg h1c "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ ^[[:blank:]]*\.sc=.*\.flg=.*\.app=.*\.sd=[^=]*\.flg=([0-9a-fx]*) ]] || append_flag b.h2s.sd.flg sd "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]h2s.*\.flg=([0-9a-fx]*) ]] || append_flag b.h2s.flg h2s "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]h2c.*\.flg=([0-9a-fx]*) ]] || append_flag b.h2c.flg h2c "${BASH_REMATCH[1]}" + elif [ $ctx = cob ]; then + ! [[ "$REPLY" =~ [[:blank:]]flags=([0-9a-fx]*) ]] || append_flag b.co.flg conn "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]fd.state=([0-9a-fx]*) ]] || append_flag b.co.fd.st fd "${BASH_REMATCH[1]}" + elif [ $ctx = res ]; then + ! [[ "$REPLY" =~ [[:blank:]]\(f=([0-9a-fx]*) ]] || append_flag res.flg chn "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]an=([0-9a-fx]*) ]] || append_flag res.ana ana "${BASH_REMATCH[1]}" + ! [[ "$REPLY" =~ [[:blank:]]htx.*flags=([0-9a-fx]*) ]] || append_flag res.htx.flg htx "${BASH_REMATCH[1]}" + fi + + out[${#out[@]}]="$REPLY" +done + +# dump the last stream +dump_and_reset diff --git a/dev/h2/mkhdr.sh b/dev/h2/mkhdr.sh new file mode 100755 index 0000000..4d129fa --- /dev/null +++ b/dev/h2/mkhdr.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +# Usage: mkhdr -l -t -f -sid > hdr.bin +# All fields are optional. 0 assumed when absent. + +USAGE=\ +"Usage: %s [-l ] [-t ] [-f ] [-i ] [ -d ] > hdr.bin + Numbers are decimal or 0xhex. Not set=0. If is passed, it points + to a file that is read and chunked into frames of bytes. + +Supported symbolic types (case insensitive prefix match): + DATA (0x00) PUSH_PROMISE (0x05) + HEADERS (0x01) PING (0x06) + PRIORITY (0x02) GOAWAY (0x07) + RST_STREAM (0x03) WINDOW_UPDATE (0x08) + SETTINGS (0x04) CONTINUATION (0x09) + +Supported symbolic flags (case insensitive prefix match): + ES (0x01) PAD (0x08) + EH (0x04) PRIO (0x20) + +" + +LEN= +TYPE= +FLAGS= +ID= + +die() { + [ "$#" -eq 0 ] || echo "$*" >&2 + exit 1 +} + +quit() { + [ "$#" -eq 0 ] || echo "$*" + exit 0 +} + +# print usage with $1 as the cmd name +usage() { + printf "$USAGE" "$1"; +} + +# Send frame made of $1 $2 $3 $4 to stdout. +# Usage: mkframe +mkframe() { + local L="${1:-0}" + local T="${2:-0}" + local F="${3:-0}" + local I="${4:-0}" + local t f + + # get the first match in this order + for t in DATA:0x00 HEADERS:0x01 RST_STREAM:0x03 SETTINGS:0x04 PING:0x06 \ + GOAWAY:0x07 WINDOW_UPDATE:0x08 CONTINUATION:0x09 PRIORITY:0x02 \ + PUSH_PROMISE:0x05; do + if [ -z "${t##${T^^*}*}" ]; then + T="${t##*:}" + break + fi + done + + if [ -n "${T##[0-9]*}" ]; then + echo "Unknown type '$T'" >&2 + usage "${0##*}" + die + fi + + # get the first match in this order + for f in ES:0x01 EH:0x04 PAD:0x08 PRIO:0x20; do + if [ -z "${f##${F^^*}*}" ]; then + F="${f##*:}" + fi + done + + if [ -n "${F##[0-9]*}" ]; then + echo "Unknown type '$T'" >&2 + usage "${0##*}" + die + fi + + L=$(( L )); T=$(( T )); F=$(( F )); I=$(( I )) + + L0=$(( (L >> 16) & 255 )); L0=$(printf "%02x" $L0) + L1=$(( (L >> 8) & 255 )); L1=$(printf "%02x" $L1) + L2=$(( (L >> 0) & 255 )); L2=$(printf "%02x" $L2) + + T0=$(( (T >> 0) & 255 )); T0=$(printf "%02x" $T0) + F0=$(( (F >> 0) & 255 )); F0=$(printf "%02x" $F0) + + I0=$(( (I >> 24) & 127 )); I0=$(printf "%02x" $I0) + I1=$(( (I >> 16) & 255 )); I1=$(printf "%02x" $I1) + I2=$(( (I >> 8) & 255 )); I2=$(printf "%02x" $I2) + I3=$(( (I >> 0) & 255 )); I3=$(printf "%02x" $I3) + + printf "\x$L0\x$L1\x$L2\x$T0\x$F0\x$I0\x$I1\x$I2\x$I3" +} + +## main + +if [ $# -le 1 ]; then + usage "${0##*}" + die +fi + +while [ -n "$1" -a -z "${1##-*}" ]; do + case "$1" in + -l) LEN="$2" ; shift 2 ;; + -t) TYPE="$2" ; shift 2 ;; + -f) FLAGS="$2" ; shift 2 ;; + -i) ID="$2" ; shift 2 ;; + -d) DATA="$2" ; shift 2 ;; + -h|--help) usage "${0##*}"; quit;; + *) usage "${0##*}"; die ;; + esac +done + +if [ $# -gt 0 ]; then + usage "${0##*}" + die +fi + +# default values for LEN and ID +LEN=${LEN:-0}; +if [ -n "${LEN##[0-9]*}" ]; then + echo "Unparsable length '$LEN'" >&2 + usage "${0##*}" + die +fi + +ID=${ID:-0}; +if [ -n "${ID##[0-9]*}" ]; then + echo "Unparsable stream ID '$ID'" >&2 + usage "${0##*}" + die +fi + +if [ -z "$DATA" ]; then + mkframe "$LEN" "$TYPE" "$FLAGS" "$ID" +else + # read file $DATA in chunks and send it in multiple frames + # advertising their respective lengths. + [ $LEN -gt 0 ] || LEN=16384 + + while read -rN "$LEN" payload || [ ${#payload} -gt 0 ]; do + mkframe "${#payload}" "$TYPE" "$FLAGS" "$ID" + echo -n "$payload" + done < "$DATA" +fi + +exit 0 diff --git a/dev/haring/README b/dev/haring/README new file mode 100644 index 0000000..5205cf2 --- /dev/null +++ b/dev/haring/README @@ -0,0 +1,9 @@ +This needs to be built from the top makefile, for example : + + make dev/haring/haring + +If HAProxy is built with special options such -DDEBUG_THREAD or with +multi-threading support enabled (which changes the ring's header size), +it can be worth reusing the same build options for haring, usually they +will remain compatible, and will simplify the handling of different file +layouts, at the expense of dragging more dependencies into the executable. diff --git a/dev/haring/haring.c b/dev/haring/haring.c new file mode 100644 index 0000000..ee7e1aa --- /dev/null +++ b/dev/haring/haring.c @@ -0,0 +1,266 @@ +/* + * post-mortem ring reader for haproxy + * + * Copyright (C) 2022 Willy Tarreau + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +/* we do not implement BUG_ON() */ +#undef DEBUG_STRICT + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +int force = 0; // force access to a different layout +int lfremap = 0; // remap LF in traces +int repair = 0; // repair file + + +/* display the message and exit with the code */ +__attribute__((noreturn)) void die(int code, const char *format, ...) +{ + va_list args; + + if (format) { + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + } + exit(code); +} + +/* display the usage message and exit with the code */ +__attribute__((noreturn)) void usage(int code, const char *arg0) +{ + die(code, + "Usage: %s [options]* \n" + "\n" + "options :\n" + " -f : force accessing a non-matching layout for 'ring struct'\n" + " -l : replace LF in contents with CR VT\n" + " -r : \"repair\" corrupted file (actively search for message boundaries)\n" + "\n" + "", arg0); +} + +/* This function dumps all events from the ring whose pointer is in into + * the appctx's output buffer, and takes from the seek offset into the + * buffer's history (0 for oldest known event). It looks at for boolean + * options: bit0 means it must wait for new data or any key to be pressed. Bit1 + * means it must seek directly to the end to wait for new contents. It returns + * 0 if the output buffer or events are missing is full and it needs to be + * called again, otherwise non-zero. It is meant to be used with + * cli_release_show_ring() to clean up. + */ +int dump_ring(struct ring *ring, size_t ofs, int flags) +{ + struct buffer buf; + uint64_t msg_len = 0; + size_t len, cnt; + const char *blk1 = NULL, *blk2 = NULL, *p; + size_t len1 = 0, len2 = 0, bl; + + /* Explanation: the storage area in the writing process starts after + * the end of the structure. Since the whole area is mmapped(), we know + * it starts at 0 mod 4096, hence the buf->area pointer's 12 LSB point + * to the relative offset of the storage area. As there will always be + * users using the wrong version of the tool with a dump, we need to + * run a few checks first. After that we'll create our own buffer + * descriptor matching that area. + */ + if ((((long)ring->buf.area) & 4095) != sizeof(*ring)) { + if (!force) { + fprintf(stderr, "FATAL: header in file is %ld bytes long vs %ld expected!\n", + (((long)ring->buf.area) & 4095), + (long)sizeof(*ring)); + exit(1); + } + else { + fprintf(stderr, "WARNING: header in file is %ld bytes long vs %ld expected!\n", + (((long)ring->buf.area) & 4095), + (long)sizeof(*ring)); + } + /* maybe we could emit a warning at least ? */ + } + + /* Now make our own buffer pointing to that area */ + buf = b_make(((void *)ring + (((long)ring->buf.area) & 4095)), + ring->buf.size, ring->buf.head, ring->buf.data); + + /* explanation for the initialization below: it would be better to do + * this in the parsing function but this would occasionally result in + * dropped events because we'd take a reference on the oldest message + * and keep it while being scheduled. Thus instead let's take it the + * first time we enter here so that we have a chance to pass many + * existing messages before grabbing a reference to a location. This + * value cannot be produced after initialization. + */ + if (unlikely(ofs == ~0)) { + ofs = 0; + + /* going to the end means looking at tail-1 */ + ofs = (flags & RING_WF_SEEK_NEW) ? buf.data - 1 : 0; + + //HA_ATOMIC_INC(b_peek(&buf, ofs)); + } + + while (1) { + //HA_RWLOCK_RDLOCK(RING_LOCK, &ring->lock); + + if (ofs >= buf.size) { + fprintf(stderr, "FATAL error at %d\n", __LINE__); + return 1; + } + //HA_ATOMIC_DEC(b_peek(&buf, ofs)); + + /* in this loop, ofs always points to the counter byte that precedes + * the message so that we can take our reference there if we have to + * stop before the end. + */ + while (ofs + 1 < b_data(&buf)) { + if (unlikely(repair && *b_peek(&buf, ofs))) { + /* in repair mode we consider that we could have landed + * in the middle of a message so we skip all bytes till + * the next zero. + */ + ofs++; + continue; + } + cnt = 1; + len = b_peek_varint(&buf, ofs + cnt, &msg_len); + if (!len) + break; + cnt += len; + + if (msg_len + ofs + cnt + 1 > buf.data) { + fprintf(stderr, "FATAL error at %d\n", __LINE__); + return 1; + } + + len = b_getblk_nc(&buf, &blk1, &len1, &blk2, &len2, ofs + cnt, msg_len); + if (!lfremap) { + if (len > 0 && len1) + fwrite(blk1, len1, 1, stdout); + if (len > 1 && len2) + fwrite(blk2, len2, 1, stdout); + } else { + while (len > 0) { + for (; len1; p++) { + p = memchr(blk1, '\n', len1); + if (!p || p > blk1) { + bl = p ? p - blk1 : len1; + fwrite(blk1, bl, 1, stdout); + blk1 += bl; + len1 -= bl; + } + + if (p) { + putchar('\r'); + putchar('\v'); + blk1++; + len1--; + } + } + len--; + blk1 = blk2; + len1 = len2; + } + } + + putchar('\n'); + + ofs += cnt + msg_len; + } + + //HA_ATOMIC_INC(b_peek(&buf, ofs)); + //HA_RWLOCK_RDUNLOCK(RING_LOCK, &ring->lock); + + if (!(flags & RING_WF_WAIT_MODE)) + break; + + /* pause 10ms before checking for new stuff */ + usleep(10000); + } + return 0; +} + +int main(int argc, char **argv) +{ + struct ring *ring; + struct stat statbuf; + const char *arg0; + int fd; + + arg0 = argv[0]; + while (argc > 1 && argv[1][0] == '-') { + argc--; argv++; + if (strcmp(argv[0], "-f") == 0) + force = 1; + else if (strcmp(argv[0], "-l") == 0) + lfremap = 1; + else if (strcmp(argv[0], "-r") == 0) + repair = 1; + else if (strcmp(argv[0], "--") == 0) + break; + else + usage(1, arg0); + } + + if (argc < 2) + usage(1, arg0); + + fd = open(argv[1], O_RDONLY); + if (fd < 0) { + perror("open()"); + return 1; + } + + if (fstat(fd, &statbuf) < 0) { + perror("fstat()"); + return 1; + } + + ring = mmap(NULL, statbuf.st_size, PROT_READ, MAP_SHARED, fd, 0); + close(fd); + + if (ring == MAP_FAILED) { + perror("mmap()"); + return 1; + } + + return dump_ring(ring, ~0, 0); +} + + +/* + * Local variables: + * c-indent-level: 8 + * c-basic-offset: 8 + * End: + */ diff --git a/dev/hpack/README b/dev/hpack/README new file mode 100644 index 0000000..d7258b5 --- /dev/null +++ b/dev/hpack/README @@ -0,0 +1,4 @@ +This needs to be built from the top makefile, for example : + + make dev/hpack/{decode,gen-enc,gen-rht} + diff --git a/dev/hpack/decode.c b/dev/hpack/decode.c new file mode 100644 index 0000000..13c95c7 --- /dev/null +++ b/dev/hpack/decode.c @@ -0,0 +1,215 @@ +/* + * HPACK stream decoder. Takes a series of hex codes on stdin using one line + * per HEADERS frame. Spaces, tabs, CR, '-' and ',' are silently skipped. + * e.g. : + * echo 82864188f439ce75c875fa5784 | dev/hpack/decode + * + * The DHT size may optionally be changed in argv[1]. + * + * Build like this : + * gcc -I../../include -O0 -g -fno-strict-aliasing -fwrapv \ + * -o decode decode.c + */ + +#define HPACK_STANDALONE + +#include +#include +#include +#include +#include +#include +#include + +#define MAX_RQ_SIZE 65536 +#define MAX_HDR_NUM 1000 + +char hex[MAX_RQ_SIZE*3+3]; // enough for "[ XX]* \0" +uint8_t buf[MAX_RQ_SIZE]; + +char trash_buf[MAX_RQ_SIZE]; +char tmp_buf[MAX_RQ_SIZE]; + +THREAD_LOCAL struct buffer trash = { .area = trash_buf, .data = 0, .size = sizeof(trash_buf) }; +struct buffer tmp = { .area = tmp_buf, .data = 0, .size = sizeof(tmp_buf) }; + +/* displays a long memory block at , assuming first byte of + * has address . String may be placed as a prefix in front of + * each line. It may be NULL if unused. The output is emitted to file . + */ +void debug_hexdump(FILE *out, const char *pfx, const char *buf, + unsigned int baseaddr, int len) +{ + unsigned int i; + int b, j; + + for (i = 0; i < (len + (baseaddr & 15)); i += 16) { + b = i - (baseaddr & 15); + fprintf(out, "%s%08x: ", pfx ? pfx : "", i + (baseaddr & ~15)); + for (j = 0; j < 8; j++) { + if (b + j >= 0 && b + j < len) + fprintf(out, "%02x ", (unsigned char)buf[b + j]); + else + fprintf(out, " "); + } + + if (b + j >= 0 && b + j < len) + fputc('-', out); + else + fputc(' ', out); + + for (j = 8; j < 16; j++) { + if (b + j >= 0 && b + j < len) + fprintf(out, " %02x", (unsigned char)buf[b + j]); + else + fprintf(out, " "); + } + + fprintf(out, " "); + for (j = 0; j < 16; j++) { + if (b + j >= 0 && b + j < len) { + if (isprint((unsigned char)buf[b + j])) + fputc((unsigned char)buf[b + j], out); + else + fputc('.', out); + } + else + fputc(' ', out); + } + fputc('\n', out); + } +} + +/* enable DEBUG_HPACK to show each individual hpack code */ +#define DEBUG_HPACK +#include "../src/hpack-huff.c" +#include "../src/hpack-tbl.c" +#include "../src/hpack-dec.c" + +/* display the message and exit with the code */ +__attribute__((noreturn)) void die(int code, const char *format, ...) +{ + va_list args; + + if (format) { + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + } + exit(code); +} + +/* reads and stops at the first LF, '#' or \0. Converts from hex to + * binary, ignoring spaces, tabs, CR, "-" and ','. The output is sent into + * for no more than bytes. The number of bytes placed there is + * returned, or a negative value in case of parsing error. + */ +int hex2bin(const char *hex, uint8_t *bin, int size) +{ + int a, b, c; + uint8_t code; + int len = 0; + + a = b = -1; + + for (; *hex; hex++) { + c = *hex; + if (c == ' ' || c == '\t' || c == '\r' || + c == '-' || c == ',') + continue; + + if (c == '\n' || c == '#') + break; + + if (c >= '0' && c <= '9') + c -= '0'; + else if (c >= 'a' && c <= 'f') + c -= 'a' - 10; + else if (c >= 'A' && c <= 'F') + c -= 'A' - 10; + else + return -1; + + if (a == -1) + a = c; + else + b = c; + + if (b == -1) + continue; + + code = (a << 4) | b; + a = b = -1; + if (len >= size) + return -2; + + bin[len] = code; + len++; + } + if (a >= 0 || b >= 0) + return -3; + return len; +} + +int main(int argc, char **argv) +{ + struct hpack_dht *dht; + struct http_hdr list[MAX_HDR_NUM]; + struct pool_head pool; + int outlen; + int dht_size = 4096; + int len, idx; + int line; + + /* first arg: dht size */ + if (argc > 1) { + dht_size = atoi(argv[1]); + argv++; argc--; + } + + pool.size = dht_size; + pool_head_hpack_tbl = &pool; + dht = hpack_dht_alloc(); + if (!dht) { + die(1, "cannot initialize dht\n"); + return 1; + } + + for (line = 1; fgets(hex, sizeof(hex), stdin); line++) { + len = hex2bin(hex, buf, sizeof(buf)); + if (len <= 0) + continue; + printf("###### line %d : frame len=%d #######\n", line, len); + debug_hexdump(stdout, " ", (const char *)buf, 0, len); + + outlen = hpack_decode_frame(dht, buf, len, list, + sizeof(list)/sizeof(list[0]), &tmp); + if (outlen <= 0) { + printf(" HPACK decoding failed: %d\n", outlen); + continue; + } + + printf("<<< Found %d headers :\n", outlen); + for (idx = 0; idx < outlen - 1; idx++) { + //printf(" \e[1;34m%s\e[0m: ", + // list[idx].n.ptr ? istpad(trash.str, list[idx].n).ptr : h2_phdr_to_str(list[idx].n.len)); + + //printf("\e[1;35m%s\e[0m\n", istpad(trash.str, list[idx].v).ptr); + + printf(" %s: ", list[idx].n.ptr ? + istpad(trash.area, list[idx].n).ptr : + h2_phdr_to_str(list[idx].n.len)); + + printf("%s [n=(%p,%d) v=(%p,%d)]\n", + istpad(trash.area, list[idx].v).ptr, + list[idx].n.ptr, (int)list[idx].n.len, list[idx].v.ptr, (int)list[idx].v.len); + } + puts(">>>"); +#ifdef DEBUG_HPACK + printf("<<=== DHT dump [ptr=%p]:\n", dht); + hpack_dht_dump(stdout, dht); + puts("===>>"); +#endif + } + return 0; +} diff --git a/dev/hpack/gen-enc.c b/dev/hpack/gen-enc.c new file mode 100644 index 0000000..3fc5ef9 --- /dev/null +++ b/dev/hpack/gen-enc.c @@ -0,0 +1,205 @@ +/* + * HPACK encoding table generator. It produces a stream of + * and a table pointing to the first of each series. + * The end of the stream is marked by =0. In parallel, a length-indexed + * table is built to access the first entry of each length. + * + * Build like this : + * gcc -I../../include -o gen-enc gen-enc.c + */ +#define HPACK_STANDALONE + +#include +#include +#include +#include +#include +#include +#include +#include "../../src/hpack-tbl.c" + +struct idxhdr { + const char *ptr; + int len; + int idx; +}; + +struct idxhdr idxhdr[HPACK_SHT_SIZE]; +static int positions[32]; +static char known_hdr[1024]; + +/* preferred ordering of headers of similar size. Those not mentioned will be + * less prioritized. + */ +const struct { + const char *name; + const int rank; +} ranks[] = { + { .name = "age", .rank = 1 }, + { .name = "via", .rank = 2 }, + + { .name = "date", .rank = 1 }, + { .name = "host", .rank = 2 }, + + { .name = "accept", .rank = 1 }, + { .name = "server", .rank = 2 }, + { .name = "cookie", .rank = 3 }, + + { .name = "referer", .rank = 1 }, + { .name = "expires", .rank = 2 }, + + { .name = "location", .rank = 1 }, + + { .name = "user-agent", .rank = 1 }, + { .name = "set-cookie", .rank = 2 }, + + { .name = "content-type", .rank = 1 }, + + { .name = "cache-control", .rank = 1 }, + { .name = "last-modified", .rank = 2 }, + { .name = "accept-ranges", .rank = 3 }, + { .name = "if-none-match", .rank = 4 }, + + { .name = "content-length", .rank = 1 }, + + { .name = "accept-encoding", .rank = 1 }, + { .name = "accept-language", .rank = 2 }, + + { .name = "content-encoding", .rank = 1 }, + + { .name = "transfer-encoding", .rank = 1 }, + { .name = "if-modified-since", .rank = 2 }, + + { .name = "content-disposition", .rank = 1 }, +}; + +/* returns the rank of header or 255 if not found */ +int get_hdr_rank(const char *name) +{ + int i; + + for (i = 0; i < sizeof(ranks) / sizeof(ranks[0]); i++) { + if (strcmp(ranks[i].name, name) == 0) + return ranks[i].rank; + } + return 255; +} + +/* sorts first on the length, second on the name, and third on the idx, so that + * headers which appear with multiple occurrences are always met first. + */ +int cmp_idx(const void *l, const void *r) +{ + const struct idxhdr *a = l, *b = r; + int ranka, rankb; + int ret; + + if (a->len < b->len) + return -1; + else if (a->len > b->len) + return 1; + + ranka = get_hdr_rank(a->ptr); + rankb = get_hdr_rank(b->ptr); + + if (ranka < rankb) + return -1; + else if (ranka > rankb) + return 1; + + /* same rank, check for duplicates and use index */ + ret = strcmp(a->ptr, b->ptr); + if (ret != 0) + return ret; + + if (a->idx < b->idx) + return -1; + else if (a->idx > b->idx) + return 1; + else + return 0; +} + +int main(int argc, char **argv) +{ + int pos; + int prev; + int len; + int i; + + for (len = 0; len < 32; len++) + positions[len] = -1; + + for (i = 0; i < HPACK_SHT_SIZE; i++) { + idxhdr[i].ptr = hpack_sht[i].n.ptr; + idxhdr[i].len = hpack_sht[i].n.len; + idxhdr[i].idx = i; + } + + /* sorts all header names by length first, then by name, and finally by + * idx so that we meet smaller headers first, that within a length they + * appear in frequency order, and that multiple occurrences appear with + * the smallest index first. + */ + qsort(&idxhdr[1], HPACK_SHT_SIZE - 1, sizeof(idxhdr[0]), cmp_idx); + + pos = 0; + prev = -1; + for (i = 1; i < HPACK_SHT_SIZE; i++) { + len = idxhdr[i].len; + if (len > 31) { + //printf("skipping %s (len=%d)\n", idxhdr[i].ptr, idxhdr[i].len); + continue; + } + + /* first occurrence of this length? */ + if (positions[len] == -1) + positions[len] = pos; + else if (prev >= 0 && + memcmp(&known_hdr[prev] + 2, idxhdr[i].ptr, len) == 0) { + /* duplicate header field */ + continue; + } + + /* store in the output array */ + + if (pos + 1 + len + 2 >= sizeof(known_hdr)) + abort(); + + prev = pos; + known_hdr[pos++] = len; + known_hdr[pos++] = idxhdr[i].idx; + memcpy(&known_hdr[pos], idxhdr[i].ptr, len); + pos += len; + //printf("%d %d %s\n", len, idxhdr[i].idx, idxhdr[i].ptr); + } + + if (pos + 1 >= sizeof(known_hdr)) + abort(); + known_hdr[pos++] = 0; // size zero ends the stream + + printf("const char hpack_enc_stream[%d] = {\n", pos); + for (i = 0; i < pos; i++) { + if ((i & 7) == 0) + printf("\t /* % 4d: */", i); + + printf(" 0x%02x,", known_hdr[i]); + + if ((i & 7) == 7 || (i == pos - 1)) + putchar('\n'); + } + printf("};\n\n"); + + printf("const signed short hpack_pos_len[32] = {\n"); + for (i = 0; i < 32; i++) { + if ((i & 7) == 0) + printf("\t /* % 4d: */", i); + + printf(" % 4d,", positions[i]); + + if ((i & 7) == 7 || (i == pos - 1)) + putchar('\n'); + } + printf("};\n\n"); + return 0; +} diff --git a/dev/hpack/gen-rht.c b/dev/hpack/gen-rht.c new file mode 100644 index 0000000..4260ffb --- /dev/null +++ b/dev/hpack/gen-rht.c @@ -0,0 +1,369 @@ +/* Reverse Huffman table generator for HPACK decoder - 2017-05-19 Willy Tarreau + * + * rht_bit31_24[256] is indexed on bits 31..24 when < 0xfe + * rht_bit24_17[256] is indexed on bits 24..17 when 31..24 >= 0xfe + * rht_bit15_11_fe[32] is indexed on bits 15..11 when 24..17 == 0xfe + * rht_bit15_8[256] is indexed on bits 15..8 when 24..17 == 0xff + * rht_bit11_4[256] is indexed on bits 11..4 when 15..8 == 0xff + * when 11..4 == 0xff, 3..2 provide the following mapping : + * 00 => 0x0a, 01 => 0x0d, 10 => 0x16, 11 => EOS + */ + +#include +#include +#include +#include + +/* from RFC7541 Appendix B */ +static const struct huff { + uint32_t c; /* code point */ + int b; /* bits */ +} ht[257] = { + [0] = { .c = 0x00001ff8, .b = 13 }, + [1] = { .c = 0x007fffd8, .b = 23 }, + [2] = { .c = 0x0fffffe2, .b = 28 }, + [3] = { .c = 0x0fffffe3, .b = 28 }, + [4] = { .c = 0x0fffffe4, .b = 28 }, + [5] = { .c = 0x0fffffe5, .b = 28 }, + [6] = { .c = 0x0fffffe6, .b = 28 }, + [7] = { .c = 0x0fffffe7, .b = 28 }, + [8] = { .c = 0x0fffffe8, .b = 28 }, + [9] = { .c = 0x00ffffea, .b = 24 }, + [10] = { .c = 0x3ffffffc, .b = 30 }, + [11] = { .c = 0x0fffffe9, .b = 28 }, + [12] = { .c = 0x0fffffea, .b = 28 }, + [13] = { .c = 0x3ffffffd, .b = 30 }, + [14] = { .c = 0x0fffffeb, .b = 28 }, + [15] = { .c = 0x0fffffec, .b = 28 }, + [16] = { .c = 0x0fffffed, .b = 28 }, + [17] = { .c = 0x0fffffee, .b = 28 }, + [18] = { .c = 0x0fffffef, .b = 28 }, + [19] = { .c = 0x0ffffff0, .b = 28 }, + [20] = { .c = 0x0ffffff1, .b = 28 }, + [21] = { .c = 0x0ffffff2, .b = 28 }, + [22] = { .c = 0x3ffffffe, .b = 30 }, + [23] = { .c = 0x0ffffff3, .b = 28 }, + [24] = { .c = 0x0ffffff4, .b = 28 }, + [25] = { .c = 0x0ffffff5, .b = 28 }, + [26] = { .c = 0x0ffffff6, .b = 28 }, + [27] = { .c = 0x0ffffff7, .b = 28 }, + [28] = { .c = 0x0ffffff8, .b = 28 }, + [29] = { .c = 0x0ffffff9, .b = 28 }, + [30] = { .c = 0x0ffffffa, .b = 28 }, + [31] = { .c = 0x0ffffffb, .b = 28 }, + [32] = { .c = 0x00000014, .b = 6 }, + [33] = { .c = 0x000003f8, .b = 10 }, + [34] = { .c = 0x000003f9, .b = 10 }, + [35] = { .c = 0x00000ffa, .b = 12 }, + [36] = { .c = 0x00001ff9, .b = 13 }, + [37] = { .c = 0x00000015, .b = 6 }, + [38] = { .c = 0x000000f8, .b = 8 }, + [39] = { .c = 0x000007fa, .b = 11 }, + [40] = { .c = 0x000003fa, .b = 10 }, + [41] = { .c = 0x000003fb, .b = 10 }, + [42] = { .c = 0x000000f9, .b = 8 }, + [43] = { .c = 0x000007fb, .b = 11 }, + [44] = { .c = 0x000000fa, .b = 8 }, + [45] = { .c = 0x00000016, .b = 6 }, + [46] = { .c = 0x00000017, .b = 6 }, + [47] = { .c = 0x00000018, .b = 6 }, + [48] = { .c = 0x00000000, .b = 5 }, + [49] = { .c = 0x00000001, .b = 5 }, + [50] = { .c = 0x00000002, .b = 5 }, + [51] = { .c = 0x00000019, .b = 6 }, + [52] = { .c = 0x0000001a, .b = 6 }, + [53] = { .c = 0x0000001b, .b = 6 }, + [54] = { .c = 0x0000001c, .b = 6 }, + [55] = { .c = 0x0000001d, .b = 6 }, + [56] = { .c = 0x0000001e, .b = 6 }, + [57] = { .c = 0x0000001f, .b = 6 }, + [58] = { .c = 0x0000005c, .b = 7 }, + [59] = { .c = 0x000000fb, .b = 8 }, + [60] = { .c = 0x00007ffc, .b = 15 }, + [61] = { .c = 0x00000020, .b = 6 }, + [62] = { .c = 0x00000ffb, .b = 12 }, + [63] = { .c = 0x000003fc, .b = 10 }, + [64] = { .c = 0x00001ffa, .b = 13 }, + [65] = { .c = 0x00000021, .b = 6 }, + [66] = { .c = 0x0000005d, .b = 7 }, + [67] = { .c = 0x0000005e, .b = 7 }, + [68] = { .c = 0x0000005f, .b = 7 }, + [69] = { .c = 0x00000060, .b = 7 }, + [70] = { .c = 0x00000061, .b = 7 }, + [71] = { .c = 0x00000062, .b = 7 }, + [72] = { .c = 0x00000063, .b = 7 }, + [73] = { .c = 0x00000064, .b = 7 }, + [74] = { .c = 0x00000065, .b = 7 }, + [75] = { .c = 0x00000066, .b = 7 }, + [76] = { .c = 0x00000067, .b = 7 }, + [77] = { .c = 0x00000068, .b = 7 }, + [78] = { .c = 0x00000069, .b = 7 }, + [79] = { .c = 0x0000006a, .b = 7 }, + [80] = { .c = 0x0000006b, .b = 7 }, + [81] = { .c = 0x0000006c, .b = 7 }, + [82] = { .c = 0x0000006d, .b = 7 }, + [83] = { .c = 0x0000006e, .b = 7 }, + [84] = { .c = 0x0000006f, .b = 7 }, + [85] = { .c = 0x00000070, .b = 7 }, + [86] = { .c = 0x00000071, .b = 7 }, + [87] = { .c = 0x00000072, .b = 7 }, + [88] = { .c = 0x000000fc, .b = 8 }, + [89] = { .c = 0x00000073, .b = 7 }, + [90] = { .c = 0x000000fd, .b = 8 }, + [91] = { .c = 0x00001ffb, .b = 13 }, + [92] = { .c = 0x0007fff0, .b = 19 }, + [93] = { .c = 0x00001ffc, .b = 13 }, + [94] = { .c = 0x00003ffc, .b = 14 }, + [95] = { .c = 0x00000022, .b = 6 }, + [96] = { .c = 0x00007ffd, .b = 15 }, + [97] = { .c = 0x00000003, .b = 5 }, + [98] = { .c = 0x00000023, .b = 6 }, + [99] = { .c = 0x00000004, .b = 5 }, + [100] = { .c = 0x00000024, .b = 6 }, + [101] = { .c = 0x00000005, .b = 5 }, + [102] = { .c = 0x00000025, .b = 6 }, + [103] = { .c = 0x00000026, .b = 6 }, + [104] = { .c = 0x00000027, .b = 6 }, + [105] = { .c = 0x00000006, .b = 5 }, + [106] = { .c = 0x00000074, .b = 7 }, + [107] = { .c = 0x00000075, .b = 7 }, + [108] = { .c = 0x00000028, .b = 6 }, + [109] = { .c = 0x00000029, .b = 6 }, + [110] = { .c = 0x0000002a, .b = 6 }, + [111] = { .c = 0x00000007, .b = 5 }, + [112] = { .c = 0x0000002b, .b = 6 }, + [113] = { .c = 0x00000076, .b = 7 }, + [114] = { .c = 0x0000002c, .b = 6 }, + [115] = { .c = 0x00000008, .b = 5 }, + [116] = { .c = 0x00000009, .b = 5 }, + [117] = { .c = 0x0000002d, .b = 6 }, + [118] = { .c = 0x00000077, .b = 7 }, + [119] = { .c = 0x00000078, .b = 7 }, + [120] = { .c = 0x00000079, .b = 7 }, + [121] = { .c = 0x0000007a, .b = 7 }, + [122] = { .c = 0x0000007b, .b = 7 }, + [123] = { .c = 0x00007ffe, .b = 15 }, + [124] = { .c = 0x000007fc, .b = 11 }, + [125] = { .c = 0x00003ffd, .b = 14 }, + [126] = { .c = 0x00001ffd, .b = 13 }, + [127] = { .c = 0x0ffffffc, .b = 28 }, + [128] = { .c = 0x000fffe6, .b = 20 }, + [129] = { .c = 0x003fffd2, .b = 22 }, + [130] = { .c = 0x000fffe7, .b = 20 }, + [131] = { .c = 0x000fffe8, .b = 20 }, + [132] = { .c = 0x003fffd3, .b = 22 }, + [133] = { .c = 0x003fffd4, .b = 22 }, + [134] = { .c = 0x003fffd5, .b = 22 }, + [135] = { .c = 0x007fffd9, .b = 23 }, + [136] = { .c = 0x003fffd6, .b = 22 }, + [137] = { .c = 0x007fffda, .b = 23 }, + [138] = { .c = 0x007fffdb, .b = 23 }, + [139] = { .c = 0x007fffdc, .b = 23 }, + [140] = { .c = 0x007fffdd, .b = 23 }, + [141] = { .c = 0x007fffde, .b = 23 }, + [142] = { .c = 0x00ffffeb, .b = 24 }, + [143] = { .c = 0x007fffdf, .b = 23 }, + [144] = { .c = 0x00ffffec, .b = 24 }, + [145] = { .c = 0x00ffffed, .b = 24 }, + [146] = { .c = 0x003fffd7, .b = 22 }, + [147] = { .c = 0x007fffe0, .b = 23 }, + [148] = { .c = 0x00ffffee, .b = 24 }, + [149] = { .c = 0x007fffe1, .b = 23 }, + [150] = { .c = 0x007fffe2, .b = 23 }, + [151] = { .c = 0x007fffe3, .b = 23 }, + [152] = { .c = 0x007fffe4, .b = 23 }, + [153] = { .c = 0x001fffdc, .b = 21 }, + [154] = { .c = 0x003fffd8, .b = 22 }, + [155] = { .c = 0x007fffe5, .b = 23 }, + [156] = { .c = 0x003fffd9, .b = 22 }, + [157] = { .c = 0x007fffe6, .b = 23 }, + [158] = { .c = 0x007fffe7, .b = 23 }, + [159] = { .c = 0x00ffffef, .b = 24 }, + [160] = { .c = 0x003fffda, .b = 22 }, + [161] = { .c = 0x001fffdd, .b = 21 }, + [162] = { .c = 0x000fffe9, .b = 20 }, + [163] = { .c = 0x003fffdb, .b = 22 }, + [164] = { .c = 0x003fffdc, .b = 22 }, + [165] = { .c = 0x007fffe8, .b = 23 }, + [166] = { .c = 0x007fffe9, .b = 23 }, + [167] = { .c = 0x001fffde, .b = 21 }, + [168] = { .c = 0x007fffea, .b = 23 }, + [169] = { .c = 0x003fffdd, .b = 22 }, + [170] = { .c = 0x003fffde, .b = 22 }, + [171] = { .c = 0x00fffff0, .b = 24 }, + [172] = { .c = 0x001fffdf, .b = 21 }, + [173] = { .c = 0x003fffdf, .b = 22 }, + [174] = { .c = 0x007fffeb, .b = 23 }, + [175] = { .c = 0x007fffec, .b = 23 }, + [176] = { .c = 0x001fffe0, .b = 21 }, + [177] = { .c = 0x001fffe1, .b = 21 }, + [178] = { .c = 0x003fffe0, .b = 22 }, + [179] = { .c = 0x001fffe2, .b = 21 }, + [180] = { .c = 0x007fffed, .b = 23 }, + [181] = { .c = 0x003fffe1, .b = 22 }, + [182] = { .c = 0x007fffee, .b = 23 }, + [183] = { .c = 0x007fffef, .b = 23 }, + [184] = { .c = 0x000fffea, .b = 20 }, + [185] = { .c = 0x003fffe2, .b = 22 }, + [186] = { .c = 0x003fffe3, .b = 22 }, + [187] = { .c = 0x003fffe4, .b = 22 }, + [188] = { .c = 0x007ffff0, .b = 23 }, + [189] = { .c = 0x003fffe5, .b = 22 }, + [190] = { .c = 0x003fffe6, .b = 22 }, + [191] = { .c = 0x007ffff1, .b = 23 }, + [192] = { .c = 0x03ffffe0, .b = 26 }, + [193] = { .c = 0x03ffffe1, .b = 26 }, + [194] = { .c = 0x000fffeb, .b = 20 }, + [195] = { .c = 0x0007fff1, .b = 19 }, + [196] = { .c = 0x003fffe7, .b = 22 }, + [197] = { .c = 0x007ffff2, .b = 23 }, + [198] = { .c = 0x003fffe8, .b = 22 }, + [199] = { .c = 0x01ffffec, .b = 25 }, + [200] = { .c = 0x03ffffe2, .b = 26 }, + [201] = { .c = 0x03ffffe3, .b = 26 }, + [202] = { .c = 0x03ffffe4, .b = 26 }, + [203] = { .c = 0x07ffffde, .b = 27 }, + [204] = { .c = 0x07ffffdf, .b = 27 }, + [205] = { .c = 0x03ffffe5, .b = 26 }, + [206] = { .c = 0x00fffff1, .b = 24 }, + [207] = { .c = 0x01ffffed, .b = 25 }, + [208] = { .c = 0x0007fff2, .b = 19 }, + [209] = { .c = 0x001fffe3, .b = 21 }, + [210] = { .c = 0x03ffffe6, .b = 26 }, + [211] = { .c = 0x07ffffe0, .b = 27 }, + [212] = { .c = 0x07ffffe1, .b = 27 }, + [213] = { .c = 0x03ffffe7, .b = 26 }, + [214] = { .c = 0x07ffffe2, .b = 27 }, + [215] = { .c = 0x00fffff2, .b = 24 }, + [216] = { .c = 0x001fffe4, .b = 21 }, + [217] = { .c = 0x001fffe5, .b = 21 }, + [218] = { .c = 0x03ffffe8, .b = 26 }, + [219] = { .c = 0x03ffffe9, .b = 26 }, + [220] = { .c = 0x0ffffffd, .b = 28 }, + [221] = { .c = 0x07ffffe3, .b = 27 }, + [222] = { .c = 0x07ffffe4, .b = 27 }, + [223] = { .c = 0x07ffffe5, .b = 27 }, + [224] = { .c = 0x000fffec, .b = 20 }, + [225] = { .c = 0x00fffff3, .b = 24 }, + [226] = { .c = 0x000fffed, .b = 20 }, + [227] = { .c = 0x001fffe6, .b = 21 }, + [228] = { .c = 0x003fffe9, .b = 22 }, + [229] = { .c = 0x001fffe7, .b = 21 }, + [230] = { .c = 0x001fffe8, .b = 21 }, + [231] = { .c = 0x007ffff3, .b = 23 }, + [232] = { .c = 0x003fffea, .b = 22 }, + [233] = { .c = 0x003fffeb, .b = 22 }, + [234] = { .c = 0x01ffffee, .b = 25 }, + [235] = { .c = 0x01ffffef, .b = 25 }, + [236] = { .c = 0x00fffff4, .b = 24 }, + [237] = { .c = 0x00fffff5, .b = 24 }, + [238] = { .c = 0x03ffffea, .b = 26 }, + [239] = { .c = 0x007ffff4, .b = 23 }, + [240] = { .c = 0x03ffffeb, .b = 26 }, + [241] = { .c = 0x07ffffe6, .b = 27 }, + [242] = { .c = 0x03ffffec, .b = 26 }, + [243] = { .c = 0x03ffffed, .b = 26 }, + [244] = { .c = 0x07ffffe7, .b = 27 }, + [245] = { .c = 0x07ffffe8, .b = 27 }, + [246] = { .c = 0x07ffffe9, .b = 27 }, + [247] = { .c = 0x07ffffea, .b = 27 }, + [248] = { .c = 0x07ffffeb, .b = 27 }, + [249] = { .c = 0x0ffffffe, .b = 28 }, + [250] = { .c = 0x07ffffec, .b = 27 }, + [251] = { .c = 0x07ffffed, .b = 27 }, + [252] = { .c = 0x07ffffee, .b = 27 }, + [253] = { .c = 0x07ffffef, .b = 27 }, + [254] = { .c = 0x07fffff0, .b = 27 }, + [255] = { .c = 0x03ffffee, .b = 26 }, + [256] = { .c = 0x3fffffff, .b = 30 }, /* EOS */ +}; + + +int main(int argc, char **argv) +{ + uint32_t c, i, j; + + /* fill first byte */ + printf("struct rht rht_bit31_24[256] = {\n"); + for (j = 0; j < 256; j++) { + for (i = 0; i < sizeof(ht)/sizeof(ht[0]); i++) { + if (ht[i].b > 8) + continue; + c = ht[i].c << (32 - ht[i].b); + + if (((c ^ (j << 24)) & -(1 << (32 - ht[i].b)) & 0xff000000) == 0) { + printf("\t[0x%02x] = { .c = 0x%02x, .l = %d },\n", j, i, ht[i].b); + break; + } + } + } + printf("};\n\n"); + + printf("struct rht rht_bit24_17[256] = {\n"); + for (j = 0; j < 256; j++) { + for (i = 0; i < sizeof(ht)/sizeof(ht[0]); i++) { + if (ht[i].b <= 8 || ht[i].b > 16) + continue; + c = ht[i].c << (32 - ht[i].b); + + if (((c ^ (j << 17)) & -(1 << (32 - ht[i].b)) & 0x01fe0000) == 0) { + printf("\t[0x%02x] = { .c = 0x%02x, .l = %d },\n", j, i, ht[i].b); + break; + } + } + } + printf("};\n\n"); + + printf("struct rht rht_bit15_11_fe[32] = {\n"); + for (j = 0; j < 32; j++) { + for (i = 0; i < sizeof(ht)/sizeof(ht[0]); i++) { + if (ht[i].b <= 16 || ht[i].b > 21) + continue; + c = ht[i].c << (32 - ht[i].b); + if ((c & 0x00ff0000) != 0x00fe0000) + continue; + + if (((c ^ (j << 11)) & -(1 << (32 - ht[i].b)) & 0x0000f800) == 0) { + printf("\t[0x%02x] = { .c = 0x%02x, .l = %d },\n", j, i, ht[i].b); + break; + } + } + } + printf("};\n\n"); + + printf("struct rht rht_bit15_8[256] = {\n"); + for (j = 0; j < 256; j++) { + for (i = 0; i < sizeof(ht)/sizeof(ht[0]); i++) { + if (ht[i].b <= 16 || ht[i].b > 24) + continue; + c = ht[i].c << (32 - ht[i].b); + if ((c & 0x00ff0000) != 0x00ff0000) + continue; + + if (((c ^ (j << 8)) & -(1 << (32 - ht[i].b)) & 0x0000ff00) == 0) { + printf("\t[0x%02x] = { .c = 0x%02x, .l = %d },\n", j, i, ht[i].b); + break; + } + } + } + printf("};\n\n"); + + printf("struct rht rht_bit11_4[256] = {\n"); + /* fill fourth byte after 0xff 0xff 0xf6-0xff. Only 0xfffffffx are not distinguished */ + for (j = 0; j < 256; j++) { + for (i = 0; i < sizeof(ht)/sizeof(ht[0]); i++) { + if (ht[i].b <= 24) + continue; + c = ht[i].c << (32 - ht[i].b); + + if (((c ^ (j << 4)) & -(1 << (32 - ht[i].b)) & 0x00000ff0) == 0) { + //printf("\tj=%02x i=%02x c=%08x l=%d c/l=%08x j/l=%08x xor=%08x\n", j, i, c, ht[i].b, c & -(1 << (32 - ht[i].b)), ((j << 4) & -(1 << (32 - ht[i].b))), (c ^ (j << 4)) & -(1 << (32 - ht[i].b))); + printf("\t[0x%02x] = { .c = 0x%02x, .l = %d },\n", j, i, ht[i].b); + break; + } + } + } + printf("\t/* Note, when l==30, bits 3..2 give 00:0x0a, 01:0x0d, 10:0x16, 11:EOS */\n"); + printf("};\n\n"); + return 0; +} diff --git a/dev/plug_qdisc/README b/dev/plug_qdisc/README new file mode 100644 index 0000000..ccc9bd0 --- /dev/null +++ b/dev/plug_qdisc/README @@ -0,0 +1,59 @@ + ** Plug queueing disciplines ** + + The 'plug' qdisc type is not documented. It is even not supported + by traffic shaping tools like 'tc' from iproute2 package. + + Such qdiscs have already been used by Yelp engineers but outside + of haproxy with libnl-utils tools (especially nl-qdisc-* tools) + to implement a workaround and make haproxy reloads work. + + Indeed with such plug qdiscs coupled with iptables configurations + we are able to temporarily bufferize IP packets and to release them as + needed. So, they may be very useful to "synchronize" TCP sessions + or at higher level to put network applications in states approaching + the ones suspected to occur during bugs. Furthermore to be sure + to produce a correct bug fix, it may be useful to reproduce + as mush as needed such painful bugs. This is where plug qdiscs + may be useful. + + To have an idea about how to use plug qdisc on the command line I highly recommend to + read Willy Tarreau blog here: + + https://www.haproxy.com/blog/truly-seamless-reloads-with-haproxy-no-more-hacks/ + + which refers to this other one from Yelp: + + https://engineeringblog.yelp.com/2015/04/true-zero-downtime-haproxy-reloads.html + + The code found in plug_qdisc.c file already helped in fixing a painful bug hard to + fix because hard to reproduce. To use the API it exports this is quite easy: + + - First your program must call plug_disc_attach() to create if not already created + a plug qdisc and use it (must be done during your application own already existing + initializations). + Note that this function calls plug_qdisc_release_indefinite_buffer() so that to + release already buffered packets before you start your application, + + - then call plug_qdisc_plug_buffer() to start buffering packets incoming to your + plug qdisc. So they won't be delivered to your application, + + - then call plug_qdisc_release_indefinite_buffer() to stop buffering the packets + incoming to your plug qdisc and release those already buffered. + So, that to be deliver them to your application. + + This code is short and simple. But uses several libraries especially libnl-route module + part of libnl library. To compile haproxy and make it use the plug_qdisc.c code we had + to link it against several libnl3 library modules like that: + + -lnl-genl-3 -lnl-route-3 -lnl-3 -lnl-cli-3 + + + - Some references: + Libnl API documentation may be found here: + https://www.infradead.org/~tgr/libnl/doc/api/index.html + + Kernel sources: + http://elixir.free-electrons.com/linux/latest/source/net/sched/sch_plug.c + + Nice website about traffic shaping with queuing disciplines: + http://wiki.linuxwall.info/doku.php/en:ressources:dossiers:networking:traffic_control diff --git a/dev/plug_qdisc/plug_qdisc.c b/dev/plug_qdisc/plug_qdisc.c new file mode 100644 index 0000000..bc47f5d --- /dev/null +++ b/dev/plug_qdisc/plug_qdisc.c @@ -0,0 +1,86 @@ +#include +#include +#include +#include +#include +#include +#include + +/* + * XXX Please, first note that this code is not safe. XXX + * It was developed fast so that to reproduce a bug. + * You will certainly have to adapt it to your application. + * But at least it gives an idea about how to programmatically use plug + * queueing disciplines. + */ + +static struct nl_sock *nl_sock; +static struct nl_cache *link_cache; +static struct rtnl_qdisc *qdisc; +static struct rtnl_tc *tc; + +static int qdisc_init(void) +{ + nl_sock = nl_cli_alloc_socket(); + nl_cli_connect(nl_sock, NETLINK_ROUTE); + link_cache = nl_cli_link_alloc_cache(nl_sock); + qdisc = nl_cli_qdisc_alloc(); + tc = (struct rtnl_tc *)qdisc; + + return 0; +} + +/* Stop buffering and release all buffered and incoming 'qdisc' + * queueing discipline traffic. + */ +int plug_qdisc_release_indefinite_buffer(void) +{ + rtnl_qdisc_plug_release_indefinite(qdisc); + return rtnl_qdisc_add(nl_sock, qdisc, 0); +} + +/* Start buffering incoming 'qdisc' queueing discipline traffic. */ +int plug_qdisc_plug_buffer(void) +{ + rtnl_qdisc_plug_buffer(qdisc); + return rtnl_qdisc_add(nl_sock, qdisc, 0); +} + +/* Create a plug qdisc attached to 'device' network device with 'parent' + * as parent, with 'id' as ID and 'limit' as buffer size. + * This is equivalent to use nl-qdisc-add tool like that: + * $ nl-qdisc-add --dev= --parent= --id= plug --limit + * $ nl-qdisc-add --dev= --parent= --id= --update plug --release-indefinite + */ +int plug_qdisc_attach(char *device, char *parent, char *id, uint32_t limit) +{ + int ret; + + if (!tc && qdisc_init() == -1) + return -1; + + nl_cli_tc_parse_dev(tc, link_cache, device); + nl_cli_tc_parse_parent(tc, parent); + if (!rtnl_tc_get_ifindex(tc)) + return -1; + + if (!rtnl_tc_get_parent(tc)) + return -1; + if (id) + nl_cli_tc_parse_handle(tc, id, 1); + + rtnl_tc_set_kind(tc, "plug"); + if (limit) + rtnl_qdisc_plug_set_limit(qdisc, limit); + + ret = rtnl_qdisc_add(nl_sock, qdisc, NLM_F_CREATE); + if (ret < 0) { + fprintf(stderr, "Could add attach qdisc: %s\n", nl_geterror(ret)); + return -1; + } + /* Release buffer. */ + plug_qdisc_release_indefinite_buffer(); + + return 0; +} + diff --git a/dev/poll/Makefile b/dev/poll/Makefile new file mode 100644 index 0000000..0247099 --- /dev/null +++ b/dev/poll/Makefile @@ -0,0 +1,13 @@ +include ../../include/make/verbose.mk + +CC = cc +OPTIMIZE = -O2 -g +DEFINE = +INCLUDE = +OBJS = poll + +poll: poll.c + $(cmd_CC) $(OPTIMIZE) $(DEFINE) $(INCLUDE) -o $@ $^ + +clean: + rm -f $(OBJS) *.[oas] *~ diff --git a/dev/poll/poll.c b/dev/poll/poll.c new file mode 100644 index 0000000..022c039 --- /dev/null +++ b/dev/poll/poll.c @@ -0,0 +1,445 @@ +#define _GNU_SOURCE // for POLLRDHUP +#include +#include +#include + +#ifdef __linux__ +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* for OSes which don't have it */ +#ifndef POLLRDHUP +#define POLLRDHUP 0 +#endif + +#ifndef MSG_NOSIGNAL +#define MSG_NOSIGNAL 0 +#endif +#ifndef MSG_MORE +#define MSG_MORE 0 +#endif + +int verbose = 0; +int cmd = 0; +int cmdstep = 0; +int zero = 0; +int one = 1; +int lfd = -1; +int cfd = -1; +int sfd = -1; +int connected = 0; +int use_epoll = 0; +struct sockaddr_in saddr, caddr; +socklen_t salen, calen; + +static inline const char *side(int fd) +{ + if (fd == lfd) + return "l"; + if (fd == sfd) + return "s"; + if (fd == cfd) + return "c"; + return "?"; +} + +void usage(const char *arg0) +{ + printf("Usage: %s [ arg [[,...]] ] ...\n" + "args:\n" + " -h display this help\n" + " -v verbose mode (shows ret values)\n" + " -e use epoll instead of poll\n" + " -c perform on client side socket\n" + " -s perform on server side socket\n" + " -l perform on listening socket\n" + "\n" + "actions for -c/-s/-l (multiple may be delimited by commas) :\n" + " con connect to listener, implicit before first -c/-s\n" + " acc accept on listener, implicit before first -s\n" + " snd send a few bytes of data\n" + " mor send a few bytes of data with MSG_MORE\n" + " rcv receive a few bytes of data\n" + " drn drain: receive till zero\n" + " shr SHUT_RD : shutdown read side\n" + " shw SHUT_WR : shutdown write side\n" + " shb SHUT_RDWR : shutdown both sides\n" + " lin disable lingering on the socket\n" + " clo close the file descriptor\n" + " pol poll() for any event\n" + "\n", arg0); +} + +void die(const char *msg) +{ + if (msg) + fprintf(stderr, "%s\n", msg); + exit(1); +} + +const char *get_errno(int ret) +{ + static char errmsg[100]; + + if (ret >= 0) + return ""; + + snprintf(errmsg, sizeof(errmsg), " (%s)", strerror(errno)); + return errmsg; +} + +void do_acc(int fd) +{ + int ret; + + calen = sizeof(caddr); + ret = accept(lfd, (struct sockaddr*)&caddr, &calen); + if (sfd < 0) + sfd = ret; + if (verbose) + printf("cmd #%d stp #%d: %s(%s=%d): ret=%d%s\n", cmd, cmdstep, __FUNCTION__ + 3, side(fd), fd, ret, get_errno(ret)); +} + +void do_con(int fd) +{ + int ret; + + ret = connect(cfd, (const struct sockaddr*)&saddr, salen); + if (verbose) + printf("cmd #%d stp #%d: %s(%s=%d): ret=%d%s\n", cmd, cmdstep, __FUNCTION__ + 3, side(fd), fd, ret, get_errno(ret)); + connected = 1; +} + +void do_snd(int fd) +{ + int ret; + + ret = send(fd, "foo", 3, MSG_NOSIGNAL|MSG_DONTWAIT); + if (verbose) + printf("cmd #%d stp #%d: %s(%s=%d): ret=%d%s\n", cmd, cmdstep, __FUNCTION__ + 3, side(fd), fd, ret, get_errno(ret)); +} + +void do_mor(int fd) +{ + int ret; + + ret = send(fd, "foo", 3, MSG_NOSIGNAL|MSG_DONTWAIT|MSG_MORE); + if (verbose) + printf("cmd #%d stp #%d: %s(%s=%d): ret=%d%s\n", cmd, cmdstep, __FUNCTION__ + 3, side(fd), fd, ret, get_errno(ret)); +} + +void do_rcv(int fd) +{ + char buf[10]; + int ret; + + ret = recv(fd, buf, sizeof(buf), MSG_DONTWAIT); + if (verbose) + printf("cmd #%d stp #%d: %s(%s=%d): ret=%d%s\n", cmd, cmdstep, __FUNCTION__ + 3, side(fd), fd, ret, get_errno(ret)); +} + +void do_drn(int fd) +{ + char buf[16384]; + int total = -1; + int ret; + + while (1) { + ret = recv(fd, buf, sizeof(buf), 0); + if (ret <= 0) + break; + if (total < 0) + total = 0; + total += ret; + } + + if (verbose) + printf("cmd #%d stp #%d: %s(%s=%d): ret=%d%s\n", cmd, cmdstep, __FUNCTION__ + 3, side(fd), fd, total, get_errno(ret)); +} + +void do_shr(int fd) +{ + int ret; + + ret = shutdown(fd, SHUT_RD); + if (verbose) + printf("cmd #%d stp #%d: %s(%s=%d): ret=%d%s\n", cmd, cmdstep, __FUNCTION__ + 3, side(fd), fd, ret, get_errno(ret)); +} + +void do_shw(int fd) +{ + int ret; + + ret = shutdown(fd, SHUT_WR); + if (verbose) + printf("cmd #%d stp #%d: %s(%s=%d): ret=%d%s\n", cmd, cmdstep, __FUNCTION__ + 3, side(fd), fd, ret, get_errno(ret)); +} + +void do_shb(int fd) +{ + int ret; + + ret = shutdown(fd, SHUT_RDWR); + if (verbose) + printf("cmd #%d stp #%d: %s(%s=%d): ret=%d%s\n", cmd, cmdstep, __FUNCTION__ + 3, side(fd), fd, ret, get_errno(ret)); +} + +void do_lin(int fd) +{ + struct linger nolinger = { .l_onoff = 1, .l_linger = 0 }; + int ret; + + ret = setsockopt(fd, SOL_SOCKET, SO_LINGER, &nolinger, sizeof(nolinger)); + if (verbose) + printf("cmd #%d stp #%d: %s(%s=%d): ret=%d%s\n", cmd, cmdstep, __FUNCTION__ + 3, side(fd), fd, ret, get_errno(ret)); +} + +void do_clo(int fd) +{ + int ret; + + ret = close(fd); + if (verbose) + printf("cmd #%d stp #%d: %s(%s=%d): ret=%d%s\n", cmd, cmdstep, __FUNCTION__ + 3, side(fd), fd, ret, get_errno(ret)); +} + +void do_pol(int fd) +{ + struct pollfd fds = { .fd = fd, .events = POLLIN|POLLOUT|POLLRDHUP, .revents=0 }; + int flags, flag; + int ret; + +#ifdef __linux__ + while (use_epoll) { + struct epoll_event evt; + static int epoll_fd = -1; + + if (epoll_fd == -1) + epoll_fd = epoll_create(1024); + if (epoll_fd == -1) + break; + evt.events = EPOLLIN | EPOLLOUT | EPOLLRDHUP; + evt.data.fd = fd; + epoll_ctl(epoll_fd, EPOLL_CTL_ADD, fd, &evt); + ret = epoll_wait(epoll_fd, &evt, 1, 0); + + if (verbose) { + printf("cmd #%d stp #%d: %s(%s=%d): ret=%d%s ev=%#x ", cmd, cmdstep, __FUNCTION__ + 3, side(fd), fd, ret, get_errno(ret), ret > 0 ? evt.events : 0); + if (ret > 0 && evt.events) { + putchar('('); + + for (flags = evt.events; flags; flags ^= flag) { + flag = flags ^ (flags & (flags - 1)); // keep lowest bit only + switch (flag) { + case EPOLLIN: printf("IN"); break; + case EPOLLOUT: printf("OUT"); break; + case EPOLLPRI: printf("PRI"); break; + case EPOLLHUP: printf("HUP"); break; + case EPOLLERR: printf("ERR"); break; + case EPOLLRDHUP: printf("RDHUP"); break; + default: printf("???[%#x]", flag); break; + } + if (flags ^ flag) + putchar(' '); + } + putchar(')'); + } + putchar('\n'); + } + + evt.data.fd = fd; + epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, &evt); + return; + } +#endif + ret = poll(&fds, 1, 0); + if (verbose) { + printf("cmd #%d stp #%d: %s(%s=%d): ret=%d%s ev=%#x ", cmd, cmdstep, __FUNCTION__ + 3, side(fd), fd, ret, get_errno(ret), ret > 0 ? fds.revents : 0); + if (ret > 0 && fds.revents) { + putchar('('); + + for (flags = fds.revents; flags; flags ^= flag) { + flag = flags ^ (flags & (flags - 1)); // keep lowest bit only + switch (flag) { + case POLLIN: printf("IN"); break; + case POLLOUT: printf("OUT"); break; + case POLLPRI: printf("PRI"); break; + case POLLHUP: printf("HUP"); break; + case POLLERR: printf("ERR"); break; + case POLLNVAL: printf("NVAL"); break; +#if POLLRDHUP + case POLLRDHUP: printf("RDHUP"); break; +#endif + default: printf("???[%#x]", flag); break; + } + if (flags ^ flag) + putchar(' '); + } + putchar(')'); + } + putchar('\n'); + } +} + +int main(int argc, char **argv) +{ + const char *arg0; + char *word, *next; + int fd; + + /* listener */ + lfd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); + if (lfd < 0) + die("socket(l)"); + + setsockopt(lfd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); + + memset(&saddr, 0, sizeof(saddr)); + saddr.sin_family = AF_INET; + saddr.sin_port = htons(0); + salen = sizeof(saddr); + + if (bind(lfd, (struct sockaddr *)&saddr, salen) < 0) + die("bind()"); + + if (listen(lfd, 1000) < 0) + die("listen()"); + + if (getsockname(lfd, (struct sockaddr *)&saddr, &salen) < 0) + die("getsockname()"); + + + /* client */ + cfd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); + if (cfd < 0) + die("socket(c)"); + + arg0 = argv[0]; + if (argc < 2) { + usage(arg0); + exit(1); + } + + write(1, "#### BEGIN ####\n", 16); // add a visible delimiter in the traces + + while (argc > 1) { + argc--; argv++; + if (**argv != '-') { + usage(arg0); + exit(1); + } + + fd = -1; + switch (argv[0][1]) { + case 'h' : + usage(arg0); + exit(0); + break; + case 'v' : + verbose++; + break; + case 'e' : + use_epoll = 1; + break; + case 'c' : + cmd++; cmdstep = 0; + if (!connected) { + do_con(cfd); + /* connection is pending in accept queue, accept() will either be + * explicit with "-l acc" below, or implicit on "-s " + */ + } + fd = cfd; + break; + case 's' : + cmd++; cmdstep = 0; + if (!connected) + do_con(cfd); + if (sfd < 0) + do_acc(lfd); + if (sfd < 0) + die("accept()"); + fd = sfd; + break; + case 'l' : + cmd++; cmdstep = 0; + fd = lfd; + break; + default : usage(arg0); exit(1); break; + } + + if (fd >= 0) { /* an action is required */ + if (argc < 2) { + usage(arg0); + exit(1); + } + + for (word = argv[1]; word && *word; word = next) { + next = strchr(word, ','); + if (next) + *(next++) = 0; + cmdstep++; + if (strcmp(word, "acc") == 0) { + do_acc(fd); + } + else if (strcmp(word, "con") == 0) { + do_con(fd); + } + else if (strcmp(word, "snd") == 0) { + do_snd(fd); + } + else if (strcmp(word, "mor") == 0) { + do_mor(fd); + } + else if (strcmp(word, "rcv") == 0) { + do_rcv(fd); + } + else if (strcmp(word, "drn") == 0) { + do_drn(fd); + } + else if (strcmp(word, "shb") == 0) { + do_shb(fd); + } + else if (strcmp(word, "shr") == 0) { + do_shr(fd); + } + else if (strcmp(word, "shw") == 0) { + do_shw(fd); + } + else if (strcmp(word, "lin") == 0) { + do_lin(fd); + } + else if (strcmp(word, "clo") == 0) { + do_clo(fd); + } + else if (strcmp(word, "pol") == 0) { + do_pol(fd); + } + else { + printf("Ignoring unknown action '%s' in step #%d of cmd #%d\n", word, cmdstep, cmd); + } + } + argc--; argv++; + } + } + + write(1, "#### END ####\n", 14); // add a visible delimiter in the traces + + if (!cmd) { + printf("No command was requested!\n"); + usage(arg0); + exit(1); + } + + return 0; +} diff --git a/dev/qpack/decode.c b/dev/qpack/decode.c new file mode 100644 index 0000000..b2d233a --- /dev/null +++ b/dev/qpack/decode.c @@ -0,0 +1,171 @@ +/* + * QPACK stream decoder. Decode a series of hex codes on stdin using one line + * per H3 HEADERS frame. Silently skip spaces, tabs, CR, '-' and ','. + * + * Compilation via Makefile + * + * Example run: + * echo 0000d1d7508b089d5c0b8170dc101a699fc15f5085ed6989397f | ./dev/qpack/decode + */ + +#include +#include +#include +#include + +#define MAX_RQ_SIZE 65536 +#define MAX_HDR_NUM 1000 + +#define QPACK_STANDALONE + +#define USE_OPENSSL +#define USE_QUIC + +#include +#include +#include +#include + +char line[MAX_RQ_SIZE * 3 + 3]; +uint8_t bin[MAX_RQ_SIZE]; + +char tmp_buf[MAX_RQ_SIZE]; +struct buffer buf = { .area = tmp_buf, .data = 0, .size = sizeof(tmp_buf) }; + +#define DEBUG_QPACK +#include "../src/hpack-huff.c" +#include "../src/qpack-dec.c" +#include "../src/qpack-tbl.c" + +/* define to compile with BUG_ON/ABORT_NOW statements */ +void ha_backtrace_to_stderr(void) +{ +} + +/* taken from dev/hpack/decode.c */ +int hex2bin(const char *hex, uint8_t *bin, int size) +{ + int a, b, c; + uint8_t code; + int len = 0; + + a = b = -1; + + for (; *hex; hex++) { + c = *hex; + if (c == ' ' || c == '\t' || c == '\r' || + c == '-' || c == ',') + continue; + + if (c == '\n' || c == '#') + break; + + if (c >= '0' && c <= '9') + c -= '0'; + else if (c >= 'a' && c <= 'f') + c -= 'a' - 10; + else if (c >= 'A' && c <= 'F') + c -= 'A' - 10; + else + return -1; + + if (a == -1) + a = c; + else + b = c; + + if (b == -1) + continue; + + code = (a << 4) | b; + a = b = -1; + if (len >= size) + return -2; + + bin[len] = code; + len++; + } + if (a >= 0 || b >= 0) + return -3; + return len; +} + +/* taken from src/tools.c */ +void debug_hexdump(FILE *out, const char *pfx, const char *buf, + unsigned int baseaddr, int len) +{ + unsigned int i; + int b, j; + + for (i = 0; i < (len + (baseaddr & 15)); i += 16) { + b = i - (baseaddr & 15); + fprintf(out, "%s%08x: ", pfx ? pfx : "", i + (baseaddr & ~15)); + for (j = 0; j < 8; j++) { + if (b + j >= 0 && b + j < len) + fprintf(out, "%02x ", (unsigned char)buf[b + j]); + else + fprintf(out, " "); + } + + if (b + j >= 0 && b + j < len) + fputc('-', out); + else + fputc(' ', out); + + for (j = 8; j < 16; j++) { + if (b + j >= 0 && b + j < len) + fprintf(out, " %02x", (unsigned char)buf[b + j]); + else + fprintf(out, " "); + } + + fprintf(out, " "); + for (j = 0; j < 16; j++) { + if (b + j >= 0 && b + j < len) { + if (isprint((unsigned char)buf[b + j])) + fputc((unsigned char)buf[b + j], out); + else + fputc('.', out); + } + else + fputc(' ', out); + } + fputc('\n', out); + } +} + +int main(int argc, char **argv) +{ + struct http_hdr hdrs[MAX_HDR_NUM]; + int len, outlen, hdr_idx; + + do { + if (!fgets(line, sizeof(line), stdin)) + break; + + if ((len = hex2bin(line, bin, MAX_RQ_SIZE)) < 0) + break; + + outlen = qpack_decode_fs(bin, len, &buf, hdrs, + sizeof(hdrs) / sizeof(hdrs[0])); + if (outlen < 0) { + fprintf(stderr, "QPACK decoding failed: %d\n", outlen); + continue; + } + + hdr_idx = 0; + fprintf(stderr, "<<< Found %d headers:\n", outlen); + while (1) { + if (isteq(hdrs[hdr_idx].n, ist(""))) + break; + + fprintf(stderr, "%.*s: %.*s\n", + (int)hdrs[hdr_idx].n.len, hdrs[hdr_idx].n.ptr, + (int)hdrs[hdr_idx].v.len, hdrs[hdr_idx].v.ptr); + + ++hdr_idx; + } + } while (1); + + return EXIT_SUCCESS; +} diff --git a/dev/sslkeylogger/sslkeylogger.lua b/dev/sslkeylogger/sslkeylogger.lua new file mode 100644 index 0000000..e67bf77 --- /dev/null +++ b/dev/sslkeylogger/sslkeylogger.lua @@ -0,0 +1,47 @@ +--[[ + This script can be used to decipher SSL traffic coming through haproxy. It + must first be loaded in the global section of haproxy configuration with + TLS keys logging activated : + + tune.ssl.keylog on + lua-load sslkeylogger.lua + + Then a http-request rule can be inserted for the desired frontend : + http-request lua.sslkeylog + + The generated keylog file can then be injected into wireshark to decipher a + network capture. +]] + +local function sslkeylog(txn, filename) + local fields = { + CLIENT_EARLY_TRAFFIC_SECRET = function() return txn.f:ssl_fc_client_early_traffic_secret() end, + CLIENT_HANDSHAKE_TRAFFIC_SECRET = function() return txn.f:ssl_fc_client_handshake_traffic_secret() end, + SERVER_HANDSHAKE_TRAFFIC_SECRET = function() return txn.f:ssl_fc_server_handshake_traffic_secret() end, + CLIENT_TRAFFIC_SECRET_0 = function() return txn.f:ssl_fc_client_traffic_secret_0() end, + SERVER_TRAFFIC_SECRET_0 = function() return txn.f:ssl_fc_server_traffic_secret_0() end, + EXPORTER_SECRET = function() return txn.f:ssl_fc_exporter_secret() end, + EARLY_EXPORTER_SECRET = function() return txn.f:ssl_fc_early_exporter_secret() end + } + + local client_random = txn.c:hex(txn.f:ssl_fc_client_random()) + + -- ensure that a key is written only once by using a session variable + if not txn:get_var('sess.sslkeylogdone') then + local file, err = io.open(filename, 'a') + if file then + for fieldname, fetch in pairs(fields) do + if fetch() then + file:write(string.format('%s %s %s\n', fieldname, client_random, fetch())) + end + end + file:close() + else + core.Warning("Cannot open SSL log file: " .. err .. ".") + end + + txn:set_var('sess.sslkeylogdone', true) + end +end + +core.register_action('sslkeylog', { 'http-req' }, sslkeylog, 1) diff --git a/dev/tcploop/Makefile b/dev/tcploop/Makefile new file mode 100644 index 0000000..6d0a0c2 --- /dev/null +++ b/dev/tcploop/Makefile @@ -0,0 +1,13 @@ +include ../../include/make/verbose.mk + +CC = gcc +OPTIMIZE = -O2 -g +DEFINE = +INCLUDE = +OBJS = tcploop + +tcploop: tcploop.c + $(cmd_CC) $(OPTIMIZE) $(DEFINE) $(INCLUDE) -o $@ $^ + +clean: + rm -f $(OBJS) *.[oas] *~ diff --git a/dev/tcploop/tcploop.c b/dev/tcploop/tcploop.c new file mode 100644 index 0000000..c091f10 --- /dev/null +++ b/dev/tcploop/tcploop.c @@ -0,0 +1,1055 @@ +/* + * TCP client and server for bug hunting + * + * Copyright (C) 2016 Willy Tarreau + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#define _GNU_SOURCE // for POLLRDHUP +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __linux__ +#include +#endif + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* for OSes which don't have it */ +#ifndef POLLRDHUP +#define POLLRDHUP 0 +#endif + +#ifndef MSG_MORE +#define MSG_MORE 0 +#endif + +struct err_msg { + int size; + int len; + char msg[0]; +}; + +const int zero = 0; +const int one = 1; +const struct linger nolinger = { .l_onoff = 1, .l_linger = 0 }; + +#define TRASH_SIZE 65536 +static char trash[TRASH_SIZE]; + +volatile int nbproc = 0; +static struct timeval start_time; +static int showtime; +static int verbose; +static int use_epoll; +static int pid; +static int sock_type = SOCK_STREAM; +static int sock_proto = IPPROTO_TCP; + + +/* display the message and exit with the code */ +__attribute__((noreturn)) void die(int code, const char *format, ...) +{ + va_list args; + + if (format) { + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + } + exit(code); +} + +/* display the usage message and exit with the code */ +__attribute__((noreturn)) void usage(int code, const char *arg0) +{ + die(code, + "Usage : %s [options]* [:]port [*]\n" + "\n" + "options :\n" + " -v : verbose\n" + " -u : use UDP instead of TCP (limited)\n" + " -U : use UNIX instead of TCP (limited, addr must have one '/')\n" + " -t|-tt|-ttt : show time (msec / relative / absolute)\n" + " -e : use epoll instead of poll on Linux\n" + "actions :\n" + " A[] : Accepts incoming sockets and closes count-1\n" + " Note: fd=accept(fd)\n" + " B[[ip]:port] : Bind a new socket to ip:port or default one if unspecified.\n" + " Note: fd=socket,bind(fd)\n" + " C[[ip]:port] : Connects to ip:port or default ones if unspecified.\n" + " Note: fd=socket,connect(fd)\n" + " D : Disconnect (connect to AF_UNSPEC)\n" + " E[] : Echo this amount of bytes. 0=infinite. unset=any amount.\n" + " F : FIN : shutdown(SHUT_WR)\n" + " G : disable lingering\n" + " I : wait for Input data to be present (POLLIN)\n" + " J : Jump back to oldest post-fork/post-accept action\n" + " K : kill the connection and go on with next operation\n" + " L[] : Listens to ip:port and optionally sets backlog\n" + " Note: fd=socket,bind(fd),listen(fd)\n" + " N : fork New process, limited to concurrent (default 1)\n" + " O : wait for Output queue to be empty (POLLOUT + TIOCOUTQ)\n" + " P[