diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 18:45:59 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 18:45:59 +0000 |
commit | 19fcec84d8d7d21e796c7624e521b60d28ee21ed (patch) | |
tree | 42d26aa27d1e3f7c0b8bd3fd14e7d7082f5008dc /src/isa-l/igzip/igzip_semi_dyn_file_perf.c | |
parent | Initial commit. (diff) | |
download | ceph-upstream.tar.xz ceph-upstream.zip |
Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/isa-l/igzip/igzip_semi_dyn_file_perf.c')
-rw-r--r-- | src/isa-l/igzip/igzip_semi_dyn_file_perf.c | 334 |
1 files changed, 334 insertions, 0 deletions
diff --git a/src/isa-l/igzip/igzip_semi_dyn_file_perf.c b/src/isa-l/igzip/igzip_semi_dyn_file_perf.c new file mode 100644 index 000000000..79e7d2754 --- /dev/null +++ b/src/isa-l/igzip/igzip_semi_dyn_file_perf.c @@ -0,0 +1,334 @@ +/********************************************************************** + Copyright(c) 2011-2016 Intel Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#define _FILE_OFFSET_BITS 64 +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <getopt.h> +#include "igzip_lib.h" +#include "test.h" + +#define MIN_BUF_SIZE (4 * 1024) +#define MIN_TEST_LOOPS 10 +#ifndef RUN_MEM_SIZE +# define RUN_MEM_SIZE 500000000 +#endif + +#define DEFAULT_SEG_SIZE (512 * 1024) +#define DEFAULT_SAMPLE_SIZE (32 * 1024) + +int usage(void) +{ + fprintf(stderr, + "Usage: igzip_semi_dynamic [options] <infile>\n" + " -h help\n" + " -v (don't) validate output by inflate and compare\n" + " -t <type> 1:stateless 0:(default)stateful\n" + " -c <size> chunk size default=%d\n" + " -s <size> sample size default=%d\n" + " -o <file> output file\n", DEFAULT_SEG_SIZE, DEFAULT_SAMPLE_SIZE); + exit(0); +} + +int str_to_i(char *s) +{ +#define ARG_MAX 32 + + int i = atoi(s); + int len = strnlen(s, ARG_MAX); + if (len < 2 || len == ARG_MAX) + return i; + + switch (s[len - 1]) { + case 'k': + i *= 1024; + break; + case 'K': + i *= 1000; + break; + case 'm': + i *= (1024 * 1024); + break; + case 'M': + i *= (1000 * 1000); + break; + case 'g': + i *= (1024 * 1024 * 1024); + break; + case 'G': + i *= (1000 * 1000 * 1000); + break; + } + return i; +} + +void semi_dyn_stateless_perf(struct isal_zstream *stream, uint8_t * inbuf, + uint64_t infile_size, uint8_t * outbuf, uint64_t outbuf_size, + int segment_size, int hist_size) +{ + struct isal_huff_histogram histogram; + struct isal_hufftables hufftable; + + isal_deflate_stateless_init(stream); + stream->end_of_stream = 0; + stream->flush = FULL_FLUSH; + stream->next_in = inbuf; + stream->next_out = outbuf; + int remaining = infile_size; + int chunk_size = segment_size; + + while (remaining > 0) { + // Generate custom hufftables on sample + memset(&histogram, 0, sizeof(struct isal_huff_histogram)); + if (remaining < segment_size * 2) { + chunk_size = remaining; + stream->end_of_stream = 1; + } + int hist_rem = (hist_size > chunk_size) ? chunk_size : hist_size; + isal_update_histogram(stream->next_in, hist_rem, &histogram); + + if (hist_rem == chunk_size) + isal_create_hufftables_subset(&hufftable, &histogram); + else + isal_create_hufftables(&hufftable, &histogram); + + // Compress with custom table + stream->avail_in = chunk_size; + stream->avail_out = chunk_size + 8 * (1 + (chunk_size >> 16)); + stream->hufftables = &hufftable; + remaining -= chunk_size; + isal_deflate_stateless(stream); + if (stream->avail_in != 0) + break; + } +} + +void semi_dyn_stateful_perf(struct isal_zstream *stream, uint8_t * inbuf, + uint64_t infile_size, uint8_t * outbuf, uint64_t outbuf_size, + int segment_size, int hist_size) +{ + struct isal_huff_histogram histogram; + struct isal_hufftables hufftable; + + isal_deflate_init(stream); + stream->end_of_stream = 0; + stream->flush = SYNC_FLUSH; + stream->next_in = inbuf; + stream->next_out = outbuf; + stream->avail_out = outbuf_size; + int remaining = infile_size; + int chunk_size = segment_size; + + while (remaining > 0) { + // Generate custom hufftables on sample + memset(&histogram, 0, sizeof(struct isal_huff_histogram)); + if (remaining < segment_size * 2) { + chunk_size = remaining; + stream->end_of_stream = 1; + } + int hist_rem = (hist_size > chunk_size) ? chunk_size : hist_size; + isal_update_histogram(stream->next_in, hist_rem, &histogram); + + if (hist_rem == chunk_size) + isal_create_hufftables_subset(&hufftable, &histogram); + else + isal_create_hufftables(&hufftable, &histogram); + + // Compress with custom table + stream->avail_in = chunk_size; + stream->hufftables = &hufftable; + remaining -= chunk_size; + isal_deflate(stream); + if (stream->internal_state.state != ZSTATE_NEW_HDR) + break; + } +} + +int main(int argc, char *argv[]) +{ + FILE *in = stdin, *out = NULL; + unsigned char *inbuf, *outbuf; + int i = 0, c; + uint64_t infile_size, outbuf_size; + int segment_size = DEFAULT_SEG_SIZE; + int sample_size = DEFAULT_SAMPLE_SIZE; + int check_output = 1; + int do_stateless = 0, do_stateful = 1; + int ret = 0; + char *out_file_name = NULL; + struct isal_zstream stream; + + while ((c = getopt(argc, argv, "vht:c:s:o:")) != -1) { + switch (c) { + case 'v': + check_output ^= 1; + break; + case 't': + if (atoi(optarg) == 1) { + do_stateful = 0; + do_stateless = 1; + } + break; + case 'c': + segment_size = str_to_i(optarg); + break; + case 's': + sample_size = str_to_i(optarg); + break; + case 'o': + out_file_name = optarg; + break; + case 'h': + default: + usage(); + break; + } + } + + // Open input file + if (optind < argc) { + if (!(in = fopen(argv[optind], "rb"))) { + fprintf(stderr, "Can't open %s for reading\n", argv[optind]); + exit(1); + } + } else + usage(); + + // Optionally open output file + if (out_file_name != NULL) { + if (!(out = fopen(out_file_name, "wb"))) { + fprintf(stderr, "Can't open %s for writing\n", out_file_name); + exit(1); + } + } + + printf("Window Size: %d K\n", IGZIP_HIST_SIZE / 1024); + + /* + * Allocate space for entire input file and output + * (assuming some possible expansion on output size) + */ + infile_size = get_filesize(in); + if (infile_size == 0) { + printf("Input file has zero length\n"); + usage(); + } + + outbuf_size = infile_size * 1.30 > MIN_BUF_SIZE ? infile_size * 1.30 : MIN_BUF_SIZE; + + if (NULL == (inbuf = malloc(infile_size))) { + fprintf(stderr, "Can't allocate input buffer memory\n"); + exit(0); + } + if (NULL == (outbuf = malloc(outbuf_size))) { + fprintf(stderr, "Can't allocate output buffer memory\n"); + exit(0); + } + + int hist_size = sample_size > segment_size ? segment_size : sample_size; + + printf("semi-dynamic sample=%d segment=%d %s\n", hist_size, segment_size, + do_stateful ? "stateful" : "stateless"); + printf("igzip_file_perf: %s\n", argv[optind]); + + // Read complete input file into buffer + stream.avail_in = (uint32_t) fread(inbuf, 1, infile_size, in); + if (stream.avail_in != infile_size) { + fprintf(stderr, "Couldn't fit all of input file into buffer\n"); + exit(0); + } + + struct perf start; + + if (do_stateful) { + BENCHMARK(&start, BENCHMARK_TIME, + semi_dyn_stateful_perf(&stream, inbuf, infile_size, outbuf, + outbuf_size, segment_size, hist_size) + ); + } + + if (do_stateless) { + BENCHMARK(&start, BENCHMARK_TIME, + semi_dyn_stateless_perf(&stream, inbuf, infile_size, outbuf, + outbuf_size, segment_size, hist_size)); + } + + if (stream.avail_in != 0) { + printf("Could not compress all of inbuf\n"); + ret = 1; + } + + printf(" file %s - in_size=%lu out_size=%d iter=%d ratio=%3.1f%%\n", argv[optind], + infile_size, stream.total_out, i, 100.0 * stream.total_out / infile_size); + + printf("igzip_semi_dyn_file: "); + perf_print(start, (long long)infile_size); + + if (out != NULL) { + printf("writing %s\n", out_file_name); + fwrite(outbuf, 1, stream.total_out, out); + fclose(out); + } + + fclose(in); + + if (check_output) { + unsigned char *inflate_buf; + struct inflate_state istate; + + if (NULL == (inflate_buf = malloc(infile_size))) { + fprintf(stderr, "Can't allocate reconstruct buffer memory\n"); + exit(0); + } + isal_inflate_init(&istate); + istate.next_in = outbuf; + istate.avail_in = stream.total_out; + istate.next_out = inflate_buf; + istate.avail_out = infile_size; + int check = isal_inflate(&istate); + + if (memcmp(inflate_buf, inbuf, infile_size)) { + printf("inflate check Fail\n"); + printf(" ret %d total_inflate=%d\n", check, istate.total_out); + for (i = 0; i < infile_size; i++) { + if (inbuf[i] != inflate_buf[i]) { + printf(" first diff at offset=%d\n", i); + break; + } + } + ret = 1; + } else + printf("inflate check Pass\n"); + free(inflate_buf); + } + + printf("End of igzip_semi_dyn_file_perf\n\n"); + return ret; +} |