From 77e50caaf2ef81cd91075cf836fed0e75718ffb4 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 13 Apr 2024 23:12:02 +0200 Subject: Adding debian version 1.8.3-2. Signed-off-by: Daniel Baumann --- debian/vendor-h2o/deps/brotli/python/README.md | 5 + debian/vendor-h2o/deps/brotli/python/bro.py | 132 +++++++++ .../vendor-h2o/deps/brotli/python/brotlimodule.cc | 302 +++++++++++++++++++++ .../deps/brotli/python/tests/compatibility_test.py | 27 ++ .../brotli/python/tests/custom_dictionary_test.py | 36 +++ .../deps/brotli/python/tests/roundtrip_test.py | 43 +++ .../deps/brotli/python/tests/test_utils.py | 36 +++ 7 files changed, 581 insertions(+) create mode 100644 debian/vendor-h2o/deps/brotli/python/README.md create mode 100755 debian/vendor-h2o/deps/brotli/python/bro.py create mode 100644 debian/vendor-h2o/deps/brotli/python/brotlimodule.cc create mode 100755 debian/vendor-h2o/deps/brotli/python/tests/compatibility_test.py create mode 100644 debian/vendor-h2o/deps/brotli/python/tests/custom_dictionary_test.py create mode 100755 debian/vendor-h2o/deps/brotli/python/tests/roundtrip_test.py create mode 100644 debian/vendor-h2o/deps/brotli/python/tests/test_utils.py (limited to 'debian/vendor-h2o/deps/brotli/python') diff --git a/debian/vendor-h2o/deps/brotli/python/README.md b/debian/vendor-h2o/deps/brotli/python/README.md new file mode 100644 index 0000000..e787228 --- /dev/null +++ b/debian/vendor-h2o/deps/brotli/python/README.md @@ -0,0 +1,5 @@ +This directory contains Python brotli wrapper module and roundtrip tests. + +To build module execute `python setup.py build_ext` from root project directory. + +To test module run `python setup.py test`. diff --git a/debian/vendor-h2o/deps/brotli/python/bro.py b/debian/vendor-h2o/deps/brotli/python/bro.py new file mode 100755 index 0000000..c6f74ce --- /dev/null +++ b/debian/vendor-h2o/deps/brotli/python/bro.py @@ -0,0 +1,132 @@ +#! /usr/bin/env python +"""bro %s -- compression/decompression utility using the Brotli algorithm.""" + +from __future__ import print_function +import argparse +import sys +import os +import brotli +import platform + + +# default values of encoder parameters +DEFAULT_PARAMS = { + 'mode': brotli.MODE_GENERIC, + 'quality': 11, + 'lgwin': 22, + 'lgblock': 0, +} + + +def get_binary_stdio(stream): + """ Return the specified standard input, output or errors stream as a + 'raw' buffer object suitable for reading/writing binary data from/to it. + """ + assert stream in ['stdin', 'stdout', 'stderr'], "invalid stream name" + stdio = getattr(sys, stream) + if sys.version_info[0] < 3: + if sys.platform == 'win32': + # set I/O stream binary flag on python2.x (Windows) + runtime = platform.python_implementation() + if runtime == "PyPy": + # the msvcrt trick doesn't work in pypy, so I use fdopen + mode = "rb" if stream == "stdin" else "wb" + stdio = os.fdopen(stdio.fileno(), mode, 0) + else: + # this works with CPython -- untested on other implementations + import msvcrt + msvcrt.setmode(stdio.fileno(), os.O_BINARY) + return stdio + else: + # get 'buffer' attribute to read/write binary data on python3.x + if hasattr(stdio, 'buffer'): + return stdio.buffer + else: + orig_stdio = getattr(sys, "__%s__" % stream) + return orig_stdio.buffer + + +def main(args=None): + + parser = argparse.ArgumentParser( + prog='bro.py', + description="Compression/decompression utility using the Brotli algorithm.") + parser.add_argument('--version', action='version', version=brotli.__version__) + parser.add_argument('-i', '--input', metavar='FILE', type=str, dest='infile', + help='Input file', default=None) + parser.add_argument('-o', '--output', metavar='FILE', type=str, dest='outfile', + help='Output file', default=None) + parser.add_argument('-f', '--force', action='store_true', + help='Overwrite existing output file', default=False) + parser.add_argument('-d', '--decompress', action='store_true', + help='Decompress input file', default=False) + params = parser.add_argument_group('optional encoder parameters') + params.add_argument('-m', '--mode', metavar="MODE", type=int, choices=[0, 1, 2], + help='The compression mode can be 0 for generic input, ' + '1 for UTF-8 encoded text, or 2 for WOFF 2.0 font data. ' + 'Defaults to 0.') + params.add_argument('-q', '--quality', metavar="QUALITY", type=int, + choices=list(range(0, 12)), + help='Controls the compression-speed vs compression-density ' + 'tradeoff. The higher the quality, the slower the ' + 'compression. Range is 0 to 11. Defaults to 11.') + params.add_argument('--lgwin', metavar="LGWIN", type=int, + choices=list(range(10, 25)), + help='Base 2 logarithm of the sliding window size. Range is ' + '10 to 24. Defaults to 22.') + params.add_argument('--lgblock', metavar="LGBLOCK", type=int, + choices=[0] + list(range(16, 25)), + help='Base 2 logarithm of the maximum input block size. ' + 'Range is 16 to 24. If set to 0, the value will be set based ' + 'on the quality. Defaults to 0.') + params.add_argument('--custom-dictionary', metavar="FILE", type=str, dest='dictfile', + help='Custom dictionary file.', default = None) + # set default values using global DEFAULT_PARAMS dictionary + parser.set_defaults(**DEFAULT_PARAMS) + + options = parser.parse_args(args=args) + + if options.infile: + if not os.path.isfile(options.infile): + parser.error('file "%s" not found' % options.infile) + with open(options.infile, "rb") as infile: + data = infile.read() + else: + if sys.stdin.isatty(): + # interactive console, just quit + parser.error('no input') + infile = get_binary_stdio('stdin') + data = infile.read() + + if options.outfile: + if os.path.isfile(options.outfile) and not options.force: + parser.error('output file exists') + outfile = open(options.outfile, "wb") + else: + outfile = get_binary_stdio('stdout') + + if options.dictfile: + if not os.path.isfile(options.dictfile): + parser.error('file "%s" not found' % options.dictfile) + with open(options.dictfile, "rb") as dictfile: + custom_dictionary = dictfile.read() + else: + custom_dictionary = '' + + + try: + if options.decompress: + data = brotli.decompress(data, dictionary=custom_dictionary) + else: + data = brotli.compress( + data, mode=options.mode, quality=options.quality, + lgwin=options.lgwin, lgblock=options.lgblock, dictionary=custom_dictionary) + except brotli.error as e: + parser.exit(1,'bro: error: %s: %s' % (e, options.infile or 'sys.stdin')) + + outfile.write(data) + outfile.close() + + +if __name__ == '__main__': + main() diff --git a/debian/vendor-h2o/deps/brotli/python/brotlimodule.cc b/debian/vendor-h2o/deps/brotli/python/brotlimodule.cc new file mode 100644 index 0000000..936d1a4 --- /dev/null +++ b/debian/vendor-h2o/deps/brotli/python/brotlimodule.cc @@ -0,0 +1,302 @@ +#define PY_SSIZE_T_CLEAN 1 +#include +#include +#include "../enc/encode.h" +#include "../dec/decode.h" +#include "../tools/version.h" + +#if PY_MAJOR_VERSION >= 3 +#define PyInt_Check PyLong_Check +#define PyInt_AsLong PyLong_AsLong +#endif + +using namespace brotli; + +static PyObject *BrotliError; + +static int as_bounded_int(PyObject *o, int* result, int lower_bound, int upper_bound) { + long value = PyInt_AsLong(o); + if ((value < (long) lower_bound) || (value > (long) upper_bound)) { + return 0; + } + *result = (int) value; + return 1; +} + +static int mode_convertor(PyObject *o, BrotliParams::Mode *mode) { + if (!PyInt_Check(o)) { + PyErr_SetString(BrotliError, "Invalid mode"); + return 0; + } + + int mode_value = -1; + if (!as_bounded_int(o, &mode_value, 0, 255)) { + PyErr_SetString(BrotliError, "Invalid mode"); + return 0; + } + *mode = (BrotliParams::Mode) mode_value; + if (*mode != BrotliParams::MODE_GENERIC && + *mode != BrotliParams::MODE_TEXT && + *mode != BrotliParams::MODE_FONT) { + PyErr_SetString(BrotliError, "Invalid mode"); + return 0; + } + + return 1; +} + +static int quality_convertor(PyObject *o, int *quality) { + if (!PyInt_Check(o)) { + PyErr_SetString(BrotliError, "Invalid quality"); + return 0; + } + + if (!as_bounded_int(o, quality, 0, 11)) { + PyErr_SetString(BrotliError, "Invalid quality. Range is 0 to 11."); + return 0; + } + + return 1; +} + +static int lgwin_convertor(PyObject *o, int *lgwin) { + if (!PyInt_Check(o)) { + PyErr_SetString(BrotliError, "Invalid lgwin"); + return 0; + } + + if (!as_bounded_int(o, lgwin, 10, 24)) { + PyErr_SetString(BrotliError, "Invalid lgwin. Range is 10 to 24."); + return 0; + } + + return 1; +} + +static int lgblock_convertor(PyObject *o, int *lgblock) { + if (!PyInt_Check(o)) { + PyErr_SetString(BrotliError, "Invalid lgblock"); + return 0; + } + + if (!as_bounded_int(o, lgblock, 0, 24) || (*lgblock != 0 && *lgblock < 16)) { + PyErr_SetString(BrotliError, "Invalid lgblock. Can be 0 or in range 16 to 24."); + return 0; + } + + return 1; +} + +PyDoc_STRVAR(compress__doc__, +"Compress a byte string.\n" +"\n" +"Signature:\n" +" compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0, dictionary='')\n" +"\n" +"Args:\n" +" string (bytes): The input data.\n" +" mode (int, optional): The compression mode can be MODE_GENERIC (default),\n" +" MODE_TEXT (for UTF-8 format text input) or MODE_FONT (for WOFF 2.0). \n" +" quality (int, optional): Controls the compression-speed vs compression-\n" +" density tradeoff. The higher the quality, the slower the compression.\n" +" Range is 0 to 11. Defaults to 11.\n" +" lgwin (int, optional): Base 2 logarithm of the sliding window size. Range\n" +" is 10 to 24. Defaults to 22.\n" +" lgblock (int, optional): Base 2 logarithm of the maximum input block size.\n" +" Range is 16 to 24. If set to 0, the value will be set based on the\n" +" quality. Defaults to 0.\n" +" dictionary (bytes, optional): Custom dictionary. Only last sliding window\n" +" size bytes will be used.\n" +"\n" +"Returns:\n" +" The compressed byte string.\n" +"\n" +"Raises:\n" +" brotli.error: If arguments are invalid, or compressor fails.\n"); + +static PyObject* brotli_compress(PyObject *self, PyObject *args, PyObject *keywds) { + PyObject *ret = NULL; + uint8_t *input, *output, *custom_dictionary; + size_t length, output_length, custom_dictionary_length; + BrotliParams::Mode mode = (BrotliParams::Mode) -1; + int quality = -1; + int lgwin = -1; + int lgblock = -1; + int ok; + + static const char *kwlist[] = { + "string", "mode", "quality", "lgwin", "lgblock", "dictionary", NULL}; + + custom_dictionary = NULL; + custom_dictionary_length = 0; + + ok = PyArg_ParseTupleAndKeywords(args, keywds, "s#|O&O&O&O&s#:compress", + const_cast(kwlist), + &input, &length, + &mode_convertor, &mode, + &quality_convertor, &quality, + &lgwin_convertor, &lgwin, + &lgblock_convertor, &lgblock, + &custom_dictionary, &custom_dictionary_length); + if (!ok) + return NULL; + + output_length = length + (length >> 2) + 10240; + output = new uint8_t[output_length]; + + BrotliParams params; + if ((int) mode != -1) + params.mode = mode; + if (quality != -1) + params.quality = quality; + if (lgwin != -1) + params.lgwin = lgwin; + if (lgblock != -1) + params.lgblock = lgblock; + + if (custom_dictionary_length == 0) { + ok = BrotliCompressBuffer(params, length, input, + &output_length, output); + } else { + uint8_t *custom_dictionary_start = custom_dictionary; + BrotliMemIn in(input, length); + BrotliMemOut out(output, output_length); + size_t sliding_window_size = ((size_t)1) << params.lgwin; + if (custom_dictionary_length > sliding_window_size) { + custom_dictionary_start += custom_dictionary_length - sliding_window_size; + custom_dictionary_length = sliding_window_size; + } + ok = BrotliCompressWithCustomDictionary(custom_dictionary_length, + custom_dictionary_start, params, &in, &out); + output_length = out.position(); + } + + if (ok) { + ret = PyBytes_FromStringAndSize((char*)output, output_length); + } else { + PyErr_SetString(BrotliError, "BrotliCompressBuffer failed"); + } + + delete[] output; + + return ret; +} + +PyDoc_STRVAR(decompress__doc__, +"Decompress a compressed byte string.\n" +"\n" +"Signature:\n" +" decompress(string)\n" +"\n" +"Args:\n" +" string (bytes): The compressed input data.\n" +" dictionary (bytes, optional): Custom dictionary. MUST be the same data\n" +" as passed to compress method.\n" +"\n" +"Returns:\n" +" The decompressed byte string.\n" +"\n" +"Raises:\n" +" brotli.error: If decompressor fails.\n"); + +static PyObject* brotli_decompress(PyObject *self, PyObject *args, PyObject *keywds) { + PyObject *ret = NULL; + const uint8_t *input, *custom_dictionary; + size_t length, custom_dictionary_length; + int ok; + + static const char *kwlist[] = {"string", "dictionary", NULL}; + + custom_dictionary = NULL; + custom_dictionary_length = 0; + + ok = PyArg_ParseTupleAndKeywords(args, keywds, "s#|s#:decompress", + const_cast(kwlist), + &input, &length, + &custom_dictionary, &custom_dictionary_length); + if (!ok) + return NULL; + + std::vector output; + const size_t kBufferSize = 65536; + uint8_t* buffer = new uint8_t[kBufferSize]; + BrotliState state; + BrotliStateInit(&state); + if (custom_dictionary_length != 0) { + BrotliSetCustomDictionary(custom_dictionary_length, custom_dictionary, &state); + } + + BrotliResult result = BROTLI_RESULT_NEEDS_MORE_OUTPUT; + while (result == BROTLI_RESULT_NEEDS_MORE_OUTPUT) { + size_t available_out = kBufferSize; + uint8_t* next_out = buffer; + size_t total_out = 0; + result = BrotliDecompressStream(&length, &input, + &available_out, &next_out, + &total_out, &state); + size_t used_out = kBufferSize - available_out; + if (used_out != 0) + output.insert(output.end(), buffer, buffer + used_out); + } + ok = result == BROTLI_RESULT_SUCCESS; + if (ok) { + ret = PyBytes_FromStringAndSize((char*)(output.size() ? &output[0] : NULL), output.size()); + } else { + PyErr_SetString(BrotliError, "BrotliDecompress failed"); + } + + BrotliStateCleanup(&state); + delete[] buffer; + + return ret; +} + +static PyMethodDef brotli_methods[] = { + {"compress", (PyCFunction)brotli_compress, METH_VARARGS | METH_KEYWORDS, compress__doc__}, + {"decompress", (PyCFunction)brotli_decompress, METH_VARARGS | METH_KEYWORDS, decompress__doc__}, + {NULL, NULL, 0, NULL} +}; + +PyDoc_STRVAR(brotli__doc__, +"The functions in this module allow compression and decompression using the\n" +"Brotli library.\n\n"); + +#if PY_MAJOR_VERSION >= 3 +#define INIT_BROTLI PyInit_brotli +#define CREATE_BROTLI PyModule_Create(&brotli_module) +#define RETURN_BROTLI return m + +static struct PyModuleDef brotli_module = { + PyModuleDef_HEAD_INIT, + "brotli", + brotli__doc__, + 0, + brotli_methods, + NULL, + NULL, + NULL +}; +#else +#define INIT_BROTLI initbrotli +#define CREATE_BROTLI Py_InitModule3("brotli", brotli_methods, brotli__doc__) +#define RETURN_BROTLI return +#endif + +PyMODINIT_FUNC INIT_BROTLI(void) { + PyObject *m = CREATE_BROTLI; + + BrotliError = PyErr_NewException((char*) "brotli.error", NULL, NULL); + + if (BrotliError != NULL) { + Py_INCREF(BrotliError); + PyModule_AddObject(m, "error", BrotliError); + } + + PyModule_AddIntConstant(m, "MODE_GENERIC", (int) BrotliParams::MODE_GENERIC); + PyModule_AddIntConstant(m, "MODE_TEXT", (int) BrotliParams::MODE_TEXT); + PyModule_AddIntConstant(m, "MODE_FONT", (int) BrotliParams::MODE_FONT); + + PyModule_AddStringConstant(m, "__version__", BROTLI_VERSION); + + RETURN_BROTLI; +} diff --git a/debian/vendor-h2o/deps/brotli/python/tests/compatibility_test.py b/debian/vendor-h2o/deps/brotli/python/tests/compatibility_test.py new file mode 100755 index 0000000..668c9ec --- /dev/null +++ b/debian/vendor-h2o/deps/brotli/python/tests/compatibility_test.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +from __future__ import print_function +import glob +import sys +import os +from subprocess import check_call + +from test_utils import PYTHON, BRO, TEST_ENV, diff_q + + +os.chdir(os.path.abspath("../../tests")) +for filename in glob.glob("testdata/*.compressed*"): + filename = os.path.abspath(filename) + print('Testing decompression of file "%s"' % os.path.basename(filename)) + expected = filename.split(".compressed")[0] + uncompressed = expected + ".uncompressed" + check_call([PYTHON, BRO, "-f", "-d", "-i", filename, "-o", uncompressed], + env=TEST_ENV) + if diff_q(uncompressed, expected) != 0: + sys.exit(1) + # Test the streaming version + with open(filename, "rb") as infile, open(uncompressed, "wb") as outfile: + check_call([PYTHON, BRO, '-d'], stdin=infile, stdout=outfile, + env=TEST_ENV) + if diff_q(uncompressed, expected) != 0: + sys.exit(1) + os.unlink(uncompressed) diff --git a/debian/vendor-h2o/deps/brotli/python/tests/custom_dictionary_test.py b/debian/vendor-h2o/deps/brotli/python/tests/custom_dictionary_test.py new file mode 100644 index 0000000..afbf07a --- /dev/null +++ b/debian/vendor-h2o/deps/brotli/python/tests/custom_dictionary_test.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +from __future__ import print_function +import sys +import os +from subprocess import check_call, Popen, PIPE + +from test_utils import PYTHON, BRO, TEST_ENV, diff_q + + +INPUTS = """\ +testdata/alice29.txt +testdata/asyoulik.txt +testdata/lcet10.txt +testdata/plrabn12.txt +../enc/encode.cc +../enc/dictionary.h +../dec/decode.c +%s +""" % BRO + +os.chdir(os.path.abspath("../../tests")) +for filename in INPUTS.splitlines(): + for quality in (1, 6, 9, 11): + for lgwin in (10, 15, 20, 24): + filename = os.path.abspath(filename) + print('Roundtrip testing file "%s" at quality %d with lg(win)=%d and auto-custom-dictionary' % + (os.path.basename(filename), quality, lgwin)) + compressed = os.path.splitext(filename)[0] + ".custom_bro" + uncompressed = os.path.splitext(filename)[0] + ".custom_unbro" + check_call([PYTHON, BRO, "-f", "-q", str(quality), "-i", filename, + "-o", compressed, "--lgwin", str(lgwin), + "--custom-dictionary", filename], env=TEST_ENV) + check_call([PYTHON, BRO, "-f", "-d", "-i", compressed, "-o", + uncompressed, "--custom-dictionary", filename], env=TEST_ENV) + if diff_q(filename, uncompressed) != 0: + sys.exit(1) diff --git a/debian/vendor-h2o/deps/brotli/python/tests/roundtrip_test.py b/debian/vendor-h2o/deps/brotli/python/tests/roundtrip_test.py new file mode 100755 index 0000000..719a7b7 --- /dev/null +++ b/debian/vendor-h2o/deps/brotli/python/tests/roundtrip_test.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +from __future__ import print_function +import sys +import os +from subprocess import check_call, Popen, PIPE + +from test_utils import PYTHON, BRO, TEST_ENV, diff_q + + +INPUTS = """\ +testdata/alice29.txt +testdata/asyoulik.txt +testdata/lcet10.txt +testdata/plrabn12.txt +../enc/encode.cc +../enc/dictionary.h +../dec/decode.c +%s +""" % BRO + +os.chdir(os.path.abspath("../../tests")) +for filename in INPUTS.splitlines(): + for quality in (1, 6, 9, 11): + filename = os.path.abspath(filename) + print('Roundtrip testing file "%s" at quality %d' % + (os.path.basename(filename), quality)) + compressed = os.path.splitext(filename)[0] + ".bro" + uncompressed = os.path.splitext(filename)[0] + ".unbro" + check_call([PYTHON, BRO, "-f", "-q", str(quality), "-i", filename, + "-o", compressed], env=TEST_ENV) + check_call([PYTHON, BRO, "-f", "-d", "-i", compressed, "-o", + uncompressed], env=TEST_ENV) + if diff_q(filename, uncompressed) != 0: + sys.exit(1) + # Test the streaming version + with open(filename, "rb") as infile, \ + open(uncompressed, "wb") as outfile: + p = Popen([PYTHON, BRO, "-q", str(quality)], stdin=infile, + stdout=PIPE, env=TEST_ENV) + check_call([PYTHON, BRO, "-d"], stdin=p.stdout, stdout=outfile, + env=TEST_ENV) + if diff_q(filename, uncompressed) != 0: + sys.exit(1) diff --git a/debian/vendor-h2o/deps/brotli/python/tests/test_utils.py b/debian/vendor-h2o/deps/brotli/python/tests/test_utils.py new file mode 100644 index 0000000..381b64e --- /dev/null +++ b/debian/vendor-h2o/deps/brotli/python/tests/test_utils.py @@ -0,0 +1,36 @@ +from __future__ import print_function +import sys +import os +import sysconfig +import filecmp + + +def diff_q(first_file, second_file): + """Simulate call to POSIX diff with -q argument""" + if not filecmp.cmp(first_file, second_file, shallow=False): + print("Files %s and %s differ" % (first_file, second_file), + file=sys.stderr) + return 1 + return 0 + + +PYTHON = sys.executable or "python" + +# 'bro.py' script should be in parent directory +BRO = os.path.abspath("../bro.py") + +# get platform- and version-specific build/lib folder +platform_lib_name = "lib.{platform}-{version[0]}.{version[1]}".format( + platform=sysconfig.get_platform(), + version=sys.version_info) + +# by default, distutils' build base is in the same location as setup.py +build_base = os.path.abspath(os.path.join("..", "..", "build")) +build_lib = os.path.join(build_base, platform_lib_name) + +# prepend build/lib to PYTHONPATH environment variable +TEST_ENV = os.environ.copy() +if 'PYTHONPATH' not in TEST_ENV: + TEST_ENV['PYTHONPATH'] = build_lib +else: + TEST_ENV['PYTHONPATH'] = build_lib + os.pathsep + TEST_ENV['PYTHONPATH'] -- cgit v1.2.3