diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-05 12:08:03 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-05 12:08:18 +0000 |
commit | 5da14042f70711ea5cf66e034699730335462f66 (patch) | |
tree | 0f6354ccac934ed87a2d555f45be4c831cf92f4a /src/fluent-bit/lib/tutf8e | |
parent | Releasing debian version 1.44.3-2. (diff) | |
download | netdata-5da14042f70711ea5cf66e034699730335462f66.tar.xz netdata-5da14042f70711ea5cf66e034699730335462f66.zip |
Merging upstream version 1.45.3+dfsg.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/fluent-bit/lib/tutf8e')
-rw-r--r-- | src/fluent-bit/lib/tutf8e/CMakeLists.txt | 17 | ||||
-rw-r--r-- | src/fluent-bit/lib/tutf8e/LICENSE | 21 | ||||
-rw-r--r-- | src/fluent-bit/lib/tutf8e/README.md | 84 | ||||
-rwxr-xr-x | src/fluent-bit/lib/tutf8e/codegen.py | 496 | ||||
-rw-r--r-- | src/fluent-bit/lib/tutf8e/include/tutf8e.h | 138 | ||||
-rw-r--r-- | src/fluent-bit/lib/tutf8e/src/tutf8e.c | 680 | ||||
-rw-r--r-- | src/fluent-bit/lib/tutf8e/test/test.c | 474 |
7 files changed, 1910 insertions, 0 deletions
diff --git a/src/fluent-bit/lib/tutf8e/CMakeLists.txt b/src/fluent-bit/lib/tutf8e/CMakeLists.txt new file mode 100644 index 000000000..a056ef83c --- /dev/null +++ b/src/fluent-bit/lib/tutf8e/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 2.8) +project(tutf8e C) + +# Not supported: -std=c90 (lacks support for inline) +# Supported: -std=gnu90, -std=c99 or -std=gnu99 + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Os -Wall") + +include_directories(include) +add_library(tutf8e STATIC src/tutf8e.c) +set_property(TARGET tutf8e PROPERTY C_STANDARD 99) +set_property(TARGET tutf8e PROPERTY C_EXTENSIONS OFF) + +add_executable(tutf8e-test test/test.c) +target_link_libraries(tutf8e-test tutf8e) +set_property(TARGET tutf8e-test PROPERTY C_STANDARD 99) +set_property(TARGET tutf8e-test PROPERTY C_EXTENSIONS ON) diff --git a/src/fluent-bit/lib/tutf8e/LICENSE b/src/fluent-bit/lib/tutf8e/LICENSE new file mode 100644 index 000000000..82ba453aa --- /dev/null +++ b/src/fluent-bit/lib/tutf8e/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Nigel Stewart + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/fluent-bit/lib/tutf8e/README.md b/src/fluent-bit/lib/tutf8e/README.md new file mode 100644 index 000000000..a630a1b75 --- /dev/null +++ b/src/fluent-bit/lib/tutf8e/README.md @@ -0,0 +1,84 @@ +# tutf8e + + *Tute Feighty* + + A tiny UTF-8 encoder for C. + +## Goals + + * As small and fast as possible + * Narrowly scoped to one-step UTF-8 encoding in C + * Link only what you need and use + * MIT licence + +## Supported Encodings + + * [iso-8859-1](https://en.wikipedia.org/wiki/ISO/IEC_8859-1) Latin-1 Western European + * [iso-8859-2](https://en.wikipedia.org/wiki/ISO/IEC_8859-2) Latin-2 East European + * [iso-8859-3](https://en.wikipedia.org/wiki/ISO/IEC_8859-3) Latin-3 South European + * [iso-8859-4](https://en.wikipedia.org/wiki/ISO/IEC_8859-4) Latin-4 North European + * [iso-8859-5](https://en.wikipedia.org/wiki/ISO/IEC_8859-5) Part 5: Latin/Cyrillic + * [iso-8859-6](https://en.wikipedia.org/wiki/ISO/IEC_8859-6) Part 6: Latin/Arabic + * [iso-8859-7](https://en.wikipedia.org/wiki/ISO/IEC_8859-7) Part 7: Latin/Greek + * [iso-8859-8](https://en.wikipedia.org/wiki/ISO/IEC_8859-8) Part 8: Latin/Hebrew + * [iso-8859-9](https://en.wikipedia.org/wiki/ISO/IEC_8859-9) Latin-5 Turkish + * [iso-8859-10](https://en.wikipedia.org/wiki/ISO/IEC_8859-10) Latin-6 Nordic + * [iso-8859-11](https://en.wikipedia.org/wiki/ISO/IEC_8859-11) Part 11: Latin/Thai + * [iso-8859-13](https://en.wikipedia.org/wiki/ISO/IEC_8859-13) Latin-7 Baltic Rim + * [iso-8859-14](https://en.wikipedia.org/wiki/ISO/IEC_8859-14) Latin-8 Celtic + * [iso-8859-15](https://en.wikipedia.org/wiki/ISO/IEC_8859-15) Latin-9 Western European + * [iso-8859-16](https://en.wikipedia.org/wiki/ISO/IEC_8859-16) Latin-10 South-Eastern European + * [windows-1250](https://en.wikipedia.org/wiki/Windows-1250) Central European and Eastern European + * [windows-1251](https://en.wikipedia.org/wiki/Windows-1251) Cyrillic + * [windows-1252](https://en.wikipedia.org/wiki/Windows-1252) English + * [windows-1253](https://en.wikipedia.org/wiki/Windows-1253) Greek + * [windows-1254](https://en.wikipedia.org/wiki/Windows-1254) Turkish + * [windows-1255](https://en.wikipedia.org/wiki/Windows-1255) Hebrew + * [windows-1256](https://en.wikipedia.org/wiki/Windows-1256) Arabic + * [windows-1257](https://en.wikipedia.org/wiki/Windows-1257) Baltic + * [windows-1258](https://en.wikipedia.org/wiki/Windows-1258) Vietnamese + +## Test Procedure + +``` +$ ./codegen.py + +$ gcc src/* test/test.c -Iinclude + +$ ./a.out +A quick brown fox jumps over the lazy dog +Nechť již hříšné saxofony ďáblů rozezvučí síň úděsnými tóny waltzu, tanga a quickstepu. +Pijamalı hasta yağız şoföre çabucak güvendi. +Põdur Zagrebi tšellomängija-följetonist Ciqo külmetas kehvas garaažis +В чащах юга жил бы цитрус? Да, но фальшивый экземпляр! +διαφυλάξτε γενικά τη ζωή σας από βαθειά ψυχικά τραύματα +עטלף אבק נס דרך מזגן שהתפוצץ כי חם +Pijamalı hasta yağız şoföre çabucak güvendi. +Flygande bäckasiner söka hwila på mjuka tuvor. +เป็นมนุษย์สุดประเสริฐเลิศคุณค่า กว่าบรรดาฝูงสัตว์เดรัจฉาน จงฝ่าฟันพัฒนาวิชาการ อย่าล้างผลาญฤๅเข่นฆ่าบีฑาใคร ไม่ถือโทษโกรธแช่งซัดฮึดฮัดด่า หัดอภัยเหมือนกีฬาอัชฌาสัย ปฏิบัติประพฤติกฎกำหนดใจ พูดจาให้จ๊ะๆ จ๋าๆ น่าฟังเอยฯ +Jeżu klątw, spłódź Finom część gry hańb! +11 passed, 0 failed tests +``` + +## How small is it? + +512 bytes + overhead per encoding. + +``` +$ for i in src/*; do gcc -c $i -O1; done +$ du -bhc *.o | grep total +32K total + +$ for i in src/*; do gcc -c $i -O3; done +$ du -bhc *.o | grep total +32K total + +$ for i in src/*; do gcc -c $i -Os; done +$ du -bhc *.o | grep total +28K total +``` + +## Related + + * [iconv](https://www.gnu.org/software/libiconv/) + * [icu](http://site.icu-project.org/) diff --git a/src/fluent-bit/lib/tutf8e/codegen.py b/src/fluent-bit/lib/tutf8e/codegen.py new file mode 100755 index 000000000..8d7c2c635 --- /dev/null +++ b/src/fluent-bit/lib/tutf8e/codegen.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 + +encodings = [ + 'windows-1250', 'windows-1251', 'windows-1252', + 'windows-1253', 'windows-1254', 'windows-1255', + 'windows-1256', 'windows-1257', 'windows-1258', + 'iso-8859-1', 'iso-8859-2', 'iso-8859-3', 'iso-8859-4', + 'iso-8859-5', 'iso-8859-6', 'iso-8859-7', 'iso-8859-8', + 'iso-8859-9', 'iso-8859-10', 'iso-8859-11', 'iso-8859-13', + 'iso-8859-14', 'iso-8859-15', 'iso-8859-16' +] + +with open('include/tutf8e.h', 'w') as include: + + include.write(''' +#ifndef TUTF8E_H +#define TUTF8E_H + +#include <stddef.h> /* size_t */ +#include <stdint.h> /* uint16_t */ + +/*************** Internal API ***************/ + +/* NUL-terminated C-string API */ + +extern int tutf8e_string_length(const uint16_t *table, const char *input, const char *invalid, size_t *input_length, size_t *output_length); +extern int tutf8e_string_encode(const uint16_t *table, const char *input, const char *invalid, char *output, size_t *output_length); + +/* Known-length buffer API */ + +extern int tutf8e_buffer_length(const uint16_t *table, const char *input, size_t input_length, const char *invalid, size_t *output_length); +extern int tutf8e_buffer_encode(const uint16_t *table, const char *input, size_t input_length, const char *invalid, char *output, size_t *output_length); + +/*************** Public API ***************/ + +/* Opaque handle type */ + +typedef void *TUTF8encoder; + +/* Query encoder by name */ + +extern TUTF8encoder tutf8e_encoder(const char *encoding); + +#define TUTF8E_OK 0 /* Success */ +#define TUTF8E_INVALID 1 /* Invalid input character */ +#define TUTF8E_TOOLONG 2 /* Insufficient output buffer */ + +/* + * tutf8e_encoder_string_length + * + * Determine the length of input and UTF8 encoded output of NUL-terminated string + * Performance: single pass O(n) + * + * output NUL terminator not counted + * + * - TUTF8E_INVALID if input character is not convertable + * - TUTF8E_OK for success + */ + +static inline int tutf8e_encoder_string_length(const TUTF8encoder encoder, const char *input, const char *invalid, size_t *input_length, size_t *output_length) +{ + return tutf8e_string_length((const uint16_t *) encoder, input, invalid, input_length, output_length); +} + +/* + * tutf8e_encoder_string_encode + * + * UTF8 encode NUL-terminated string + * Performance: two pass O(n) + * + * output string is NUL terminated + * output_length is output buffer size for input + * output_length is encoded length for output, including NUL + * + * - TUTF8E_TOOLONG if output buffer insuficient + * - TUTF8E_INVALID if input character is not convertable + * - TUTF8E_OK for success + */ + +static inline int tutf8e_encoder_string_encode(const TUTF8encoder encoder, const char *input, const char *invalid, char *output, size_t *output_length) +{ + return tutf8e_string_encode((const uint16_t *) encoder, input, invalid, output, output_length); +} + +/* Known-length buffer API */ + +/* + * tutf8e_encoder_buffer_length + * + * Determine the length of input and UTF8 encoded output of string + * Performance: single pass O(n) + * + * output NUL terminator not counted + * + * - TUTF8E_INVALID if input character is not convertable + * - TUTF8E_OK for success + */ + +static inline int tutf8e_encoder_buffer_length(const TUTF8encoder encoder, const char *input, const char *invalid, size_t input_length, size_t *length) +{ + return tutf8e_buffer_length((const uint16_t *) encoder, input, input_length, invalid, length); +} + +/* + * tutf8e_encoder_buffer_encode + * + * UTF8 encode string + * Performance: two pass O(n) + * + * output string is not NUL terminated + * + * output_length is output buffer size for input + * output_length is encoded length for output + * + * - TUTF8E_TOOLONG if output buffer insuficient + * - TUTF8E_INVALID if input character is not convertable + * - TUTF8E_OK for success + */ + +static inline int tutf8e_encoder_buffer_encode(const TUTF8encoder encoder, const char *input, size_t input_length, const char *invalid, char *output, size_t *output_length) +{ + return tutf8e_buffer_encode((const uint16_t *) encoder, input, input_length, invalid, output, output_length); +} + +''') + + include.write('/* Supported encoders */\n\n') + for e in sorted(encodings): + name = e.replace('-', '_').lower() + include.write('extern const TUTF8encoder tutf8e_encoder_%s;\n'%(name)) + + include.write('\n') + include.write('#endif\n') + +with open('src/tutf8e.c', 'w') as src: + + src.write(''' +#include <tutf8e.h> + +#include <string.h> + +int tutf8e_string_length(const uint16_t *table, const char *input, const char *invalid, size_t *input_length, size_t *output_length) +{ + const size_t invalid_length = invalid ? strlen(invalid) : 0; + + const unsigned char *i; + for (i = (const unsigned char *) input; *i; ++i, (*input_length)++) { + const uint16_t c = table[*i]; + if (c<0x80) { + *output_length += 1; + continue; + } + if (c<0x800) { + *output_length += 2; + continue; + } + if (c<0xffff) { + *output_length += 3; + continue; + } + if (invalid) { + *output_length += invalid_length; + } + else { + return TUTF8E_INVALID; + } + } + return TUTF8E_OK; +} + +int tutf8e_string_encode(const uint16_t *table, const char *input, const char *invalid, char *output, size_t *output_length) +{ + int ret; + size_t input_length = 0; + size_t encoded_length = 0; + if (!(ret = tutf8e_string_length(table, input, invalid, &input_length, &encoded_length))) + { + if (encoded_length+1 > *output_length) return TUTF8E_TOOLONG; + if (!(ret = tutf8e_buffer_encode(table, input, input_length, invalid, output, output_length))) + { + output[encoded_length] = 0; + return TUTF8E_OK; + } + } + return ret; +} + +int tutf8e_buffer_length +( + const uint16_t *table, + const char *input, + size_t input_length, + const char *invalid, + size_t *length +) +{ + const size_t invalid_length = invalid ? strlen(invalid) : 0; + + const unsigned char *i; + for (i = (const unsigned char *) input; input_length; ++i, --input_length) { + const uint16_t c = table[*i]; + if (c<0x80) { + ++*length; + continue; + } + if (c<0x800) { + *length += 2; + continue; + } + if (c<0xffff) { + *length += 3; + continue; + } + if (invalid) { + *length += invalid_length; + } + else { + return TUTF8E_INVALID; + } + } + return TUTF8E_OK; +} + +int tutf8e_buffer_encode +( + const uint16_t *table, + const char *input, + size_t input_length, + const char *invalid, + char *output, + size_t *output_length +) +{ + size_t invalid_length = invalid ? strlen(invalid) : 0; + + size_t left = *output_length; + unsigned char *o = (unsigned char *) output; + const unsigned char *i; + for (i = (const unsigned char *) input; input_length; ++i, --input_length) { + const uint16_t c = table[*i]; + if (c<0x80) { + if (left<1) return TUTF8E_TOOLONG; + *(o++) = c; + left -= 1; + continue; + } + if (c<0x800) { + if (left<2) return TUTF8E_TOOLONG; + *(o++) = 0xc0 | (c>>6); + *(o++) = 0x80 | (c&0x3f); + left -= 2; + continue; + } + if (c<0xffff) { + if (left<3) return TUTF8E_TOOLONG; + *(o++) = 0xe0 | (c>>12); + *(o++) = 0x80 | ((c>>6)&0x3f); + *(o++) = 0x80 | (c&0x3f); + left -= 3; + continue; + } + if (invalid) + { + if (left<invalid_length) return TUTF8E_TOOLONG; + if (invalid_length) { + memcpy(o, invalid, invalid_length); + o += invalid_length; + left -= invalid_length; + } + } + else { + return TUTF8E_INVALID; + } + } + *output_length -= left; + return TUTF8E_OK; +} +''') + + for e in sorted(encodings): + + mapping = {} + domain = [] + + name = e.replace('-', '_').lower() + + v = [] + for i in range(0,256): + try: + v.append(ord(bytes([i]).decode(e)[0])) + except: + v.append(0xffff) + pass + + src.write('\n') + src.write('const uint16_t tutf8e_%s_utf8[256] =\n'%(name)) + src.write('{\n') + for i in range(0,256,16): + src.write(' %s,\n'%(', '.join([ '0x%04x'%(i) for i in v[i:i+16]]))) + src.write('};\n') + + src.write('\n') + for e in sorted(encodings): + name = e.replace('-', '_').lower() + src.write('const TUTF8encoder tutf8e_encoder_%s = (TUTF8encoder) tutf8e_%s_utf8;\n'%(name, name)) + + src.write(''' +TUTF8encoder tutf8e_encoder(const char *encoding) +{ +''') + for e in sorted(encodings): + name = e.replace('-', '_').lower() + src.write(' if (!strcmp(encoding, "%s")) return tutf8e_encoder_%s;\n'%(e, name)) + src.write(''' + return NULL; +} +''') + +for e in sorted(encodings): + + mapping = {} + domain = [] + + name = e.replace('-', '_').lower() + with open('include/tutf8e/%s.h'%(name), 'w') as include: + + include.write(''' +#ifndef TUTF8E_%s_H +#define TUTF8E_%s_H + +#include <tutf8e.h> +'''%(name.upper(), name.upper())) + + include.write(''' +static inline int tutf8e_%s_string_length(const char *input, const char *invalid, size_t *input_length, size_t *output_length) +{ + return tutf8e_encoder_string_length(tutf8e_encoder_%s, input, invalid, input_length, output_length); +} +'''%(name, name)) + + include.write(''' +static inline int tutf8e_%s_string_encode(const char *input, const char *invalid, char *output, size_t *output_length) +{ + return tutf8e_encoder_string_encode(tutf8e_encoder_%s, input, invalid, output, output_length); +} +'''%(name, name)) + + include.write(''' +static inline int tutf8e_%s_buffer_length(const char *i, size_t input_length, const char *invalid, size_t *length) +{ + return tutf8e_encoder_buffer_length(tutf8e_encoder_%s, input, input_length, invalid, length); +} +'''%(name, name)) + + include.write(''' +static inline int tutf8e_%s_buffer_encode(const char *i, size_t input_length, const char *invalid, char *output, size_t *output_length) +{ + return tutf8e_encoder_buffer_encode(tutf8e_encoder_%s, input, input_length, invalid, output, output_length); +} +'''%(name, name)) + + include.write('\n') + include.write('#endif\n') + +# TESTS + +# List of pangrams +# http://clagnut.com/blog/2380/ + +tests = [ + ('english', 'iso-8859-1', 'A quick brown fox jumps over the lazy dog'), + ('finnish', 'iso-8859-1', 'Albert osti fagotin ja töräytti puhkuvan melodian.'), + ('czech', 'iso-8859-2', 'Nechť již hříšné saxofony ďáblů rozezvučí síň úděsnými tóny waltzu, tanga a quickstepu.'), + ('turkish', 'iso-8859-3', 'Pijamalı hasta yağız şoföre çabucak güvendi.'), + ('estonian', 'iso-8859-4', 'Põdur Zagrebi tšellomängija-följetonist Ciqo külmetas kehvas garaažis'), + ('russian', 'iso-8859-5', 'В чащах юга жил бы цитрус? Да, но фальшивый экземпляр!'), + ('greek', 'iso-8859-7', 'διαφυλάξτε γενικά τη ζωή σας από βαθειά ψυχικά τραύματα'), + ('hebrew', 'iso-8859-8', 'עטלף אבק נס דרך מזגן שהתפוצץ כי חם'), + ('turkish2', 'iso-8859-9', 'Pijamalı hasta yağız şoföre çabucak güvendi.'), + ('swedish', 'iso-8859-10', 'Flygande bäckasiner söka hwila på mjuka tuvor.'), + ('thai', 'iso-8859-11', 'เป็นมนุษย์สุดประเสริฐเลิศคุณค่า กว่าบรรดาฝูงสัตว์เดรัจฉาน จงฝ่าฟันพัฒนาวิชาการ อย่าล้างผลาญฤๅเข่นฆ่าบีฑาใคร ไม่ถือโทษโกรธแช่งซัดฮึดฮัดด่า หัดอภัยเหมือนกีฬาอัชฌาสัย ปฏิบัติประพฤติกฎกำหนดใจ พูดจาให้จ๊ะๆ จ๋าๆ น่าฟังเอยฯ'), + ('polish', 'iso-8859-13', 'Jeżu klątw, spłódź Finom część gry hańb!') +] + +with open('test/test.c', 'w') as test: + + test.write('#include <tutf8e.h>\n') + test.write('\n') + # for e in sorted(encodings): + # name = e.replace('-', '_').lower() + # test.write('#include <tutf8e/%s.h>\n'%(name)) + # test.write('\n') + + test.write('#include <stdio.h>\n') + test.write('#include <string.h>\n') + test.write('#include <stdlib.h>\n') + test.write('\n') + test.write('int main(int argc, char *argv[])\n') + test.write('{\n') + test.write(' int pass = 0;\n') + test.write(' int fail = 0;\n') + test.write(' int ret;\n') + test.write(' char *copy;\n') + test.write(' size_t input_length, output_length;\n') + test.write(' char buffer[1024];\n') + # test.write(' char *encoded;\n') + test.write('\n') + + for i in tests: + if i[1] in encodings: + test.write(' static const char %s[] = {\n'%(i[0])) + data = [i for i in i[2].encode(i[1])] + [ 0 ] + for i in range(0, len(data), 24): + test.write(' %s,\n'%(', '.join([ '0x%02x'%(j) for j in data[i:i+24]]))) + test.write(' };\n') + + test.write('\n') + for i in tests: + if i[1] in encodings: + test.write(' static const char %sUTF8[] = {\n'%(i[0])) + data = [i for i in i[2].encode('utf-8')] + [ 0 ] + for i in range(0, len(data), 24): + test.write(' %s,\n'%(', '.join([ '0x%02x'%(j) for j in data[i:i+24]]))) + test.write(' };\n') + + test.write('\n /* string encode to UTF8, error on invalid input */\n') + for i in tests: + if i[1] in encodings: + name = i[1].replace('-', '_').lower() + test.write(' output_length = sizeof(buffer);\n') + test.write(' ret = tutf8e_encoder_string_encode(tutf8e_encoder_%s, %s, NULL, buffer, &output_length);\n'%(name, i[0])) + test.write(' if (!ret && !strcmp(buffer, %sUTF8)) {\n'%(i[0])) + test.write(' printf("%s\\n", buffer);\n') + test.write(' pass++;\n') + test.write(' } else {\n') + test.write(' printf("Failed to encode %s test\\n");\n'%(i[0])) + test.write(' fail++;\n') + test.write(' }\n') + test.write('\n') + + test.write('\n /* buffer encode to UTF8, error on invalid input */\n') + for i in tests: + if i[1] in encodings: + name = i[1].replace('-', '_').lower() + test.write(' input_length = strlen(%s);\n'%(i[0])) + test.write(' output_length = sizeof(buffer);\n') + test.write(' ret = tutf8e_encoder_buffer_encode(tutf8e_encoder_%s, %s, input_length, NULL, buffer, &output_length);\n'%(name, i[0])) + test.write(' if (!ret && (output_length+1)==sizeof(%sUTF8) && !strncmp(buffer, %sUTF8, output_length)) {\n'%(i[0], i[0])) + test.write(' pass++;\n') + test.write(' } else {\n') + test.write(' printf("Failed to encode %s test\\n");\n'%(i[0])) + test.write(' fail++;\n') + test.write(' }\n') + test.write('\n') + + test.write('\n /* string encode to UTF8, first input character invalid -> ? */\n') + for i in tests: + if i[1] in ['iso-8859-6', 'iso-8859-7', 'iso-8859-11']: + name = i[1].replace('-', '_').lower() + test.write(' output_length = sizeof(buffer);\n') + test.write(' copy = strdup(%s);\n'%(i[0])) + test.write(' copy[0] = 255;\n') + test.write(' buffer[0] = 255;\n') + test.write(' ret = tutf8e_encoder_string_encode(tutf8e_encoder_%s, copy, "?", buffer, &output_length);\n'%(name)) + test.write(' if (!ret && buffer[0]==\'?\') {\n') + test.write(' printf("%s\\n", buffer);\n') + test.write(' pass++;\n') + test.write(' } else {\n') + test.write(' printf("Failed to encode %s test\\n");\n'%(i[0])) + test.write(' fail++;\n') + test.write(' }\n') + test.write(' free(copy);\n') + test.write('\n') + + test.write('\n /* string encode to UTF8, first input character invalid -> [INVALID] */\n') + for i in tests: + if i[1] in ['iso-8859-6', 'iso-8859-7', 'iso-8859-11']: + name = i[1].replace('-', '_').lower() + test.write(' output_length = sizeof(buffer);\n') + test.write(' copy = strdup(%s);\n'%(i[0])) + test.write(' copy[0] = 255;\n') + test.write(' buffer[0] = 255;\n') + test.write(' ret = tutf8e_encoder_string_encode(tutf8e_encoder_%s, copy, "[INVALID]", buffer, &output_length);\n'%(name)) + test.write(' if (!ret && !strncmp(buffer, "[INVALID]", 9)) {\n') + test.write(' printf("%s\\n", buffer);\n') + test.write(' pass++;\n') + test.write(' } else {\n') + test.write(' printf("Failed to encode %s test\\n");\n'%(i[0])) + test.write(' fail++;\n') + test.write(' }\n') + test.write(' free(copy);\n') + test.write('\n') + + test.write(' printf("%d passed, %d failed tests\\n", pass, fail);\n') + + test.write('}\n') diff --git a/src/fluent-bit/lib/tutf8e/include/tutf8e.h b/src/fluent-bit/lib/tutf8e/include/tutf8e.h new file mode 100644 index 000000000..478f514b5 --- /dev/null +++ b/src/fluent-bit/lib/tutf8e/include/tutf8e.h @@ -0,0 +1,138 @@ + +#ifndef TUTF8E_H +#define TUTF8E_H + +#include <stddef.h> /* size_t */ +#include <stdint.h> /* uint16_t */ + +/*************** Internal API ***************/ + +/* NUL-terminated C-string API */ + +extern int tutf8e_string_length(const uint16_t *table, const char *input, const char *invalid, size_t *input_length, size_t *output_length); +extern int tutf8e_string_encode(const uint16_t *table, const char *input, const char *invalid, char *output, size_t *output_length); + +/* Known-length buffer API */ + +extern int tutf8e_buffer_length(const uint16_t *table, const char *input, size_t input_length, const char *invalid, size_t *output_length); +extern int tutf8e_buffer_encode(const uint16_t *table, const char *input, size_t input_length, const char *invalid, char *output, size_t *output_length); + +/*************** Public API ***************/ + +/* Opaque handle type */ + +typedef void *TUTF8encoder; + +/* Query encoder by name */ + +extern TUTF8encoder tutf8e_encoder(const char *encoding); + +#define TUTF8E_OK 0 /* Success */ +#define TUTF8E_INVALID 1 /* Invalid input character */ +#define TUTF8E_TOOLONG 2 /* Insufficient output buffer */ + +/* + * tutf8e_encoder_string_length + * + * Determine the length of input and UTF8 encoded output of NUL-terminated string + * Performance: single pass O(n) + * + * output NUL terminator not counted + * + * - TUTF8E_INVALID if input character is not convertable + * - TUTF8E_OK for success + */ + +static inline int tutf8e_encoder_string_length(const TUTF8encoder encoder, const char *input, const char *invalid, size_t *input_length, size_t *output_length) +{ + return tutf8e_string_length((const uint16_t *) encoder, input, invalid, input_length, output_length); +} + +/* + * tutf8e_encoder_string_encode + * + * UTF8 encode NUL-terminated string + * Performance: two pass O(n) + * + * output string is NUL terminated + * output_length is output buffer size for input + * output_length is encoded length for output, including NUL + * + * - TUTF8E_TOOLONG if output buffer insuficient + * - TUTF8E_INVALID if input character is not convertable + * - TUTF8E_OK for success + */ + +static inline int tutf8e_encoder_string_encode(const TUTF8encoder encoder, const char *input, const char *invalid, char *output, size_t *output_length) +{ + return tutf8e_string_encode((const uint16_t *) encoder, input, invalid, output, output_length); +} + +/* Known-length buffer API */ + +/* + * tutf8e_encoder_buffer_length + * + * Determine the length of input and UTF8 encoded output of string + * Performance: single pass O(n) + * + * output NUL terminator not counted + * + * - TUTF8E_INVALID if input character is not convertable + * - TUTF8E_OK for success + */ + +static inline int tutf8e_encoder_buffer_length(const TUTF8encoder encoder, const char *input, const char *invalid, size_t input_length, size_t *length) +{ + return tutf8e_buffer_length((const uint16_t *) encoder, input, input_length, invalid, length); +} + +/* + * tutf8e_encoder_buffer_encode + * + * UTF8 encode string + * Performance: two pass O(n) + * + * output string is not NUL terminated + * + * output_length is output buffer size for input + * output_length is encoded length for output + * + * - TUTF8E_TOOLONG if output buffer insuficient + * - TUTF8E_INVALID if input character is not convertable + * - TUTF8E_OK for success + */ + +static inline int tutf8e_encoder_buffer_encode(const TUTF8encoder encoder, const char *input, size_t input_length, const char *invalid, char *output, size_t *output_length) +{ + return tutf8e_buffer_encode((const uint16_t *) encoder, input, input_length, invalid, output, output_length); +} + +/* Supported encoders */ + +extern const TUTF8encoder tutf8e_encoder_iso_8859_1; +extern const TUTF8encoder tutf8e_encoder_iso_8859_10; +extern const TUTF8encoder tutf8e_encoder_iso_8859_11; +extern const TUTF8encoder tutf8e_encoder_iso_8859_13; +extern const TUTF8encoder tutf8e_encoder_iso_8859_14; +extern const TUTF8encoder tutf8e_encoder_iso_8859_15; +extern const TUTF8encoder tutf8e_encoder_iso_8859_16; +extern const TUTF8encoder tutf8e_encoder_iso_8859_2; +extern const TUTF8encoder tutf8e_encoder_iso_8859_3; +extern const TUTF8encoder tutf8e_encoder_iso_8859_4; +extern const TUTF8encoder tutf8e_encoder_iso_8859_5; +extern const TUTF8encoder tutf8e_encoder_iso_8859_6; +extern const TUTF8encoder tutf8e_encoder_iso_8859_7; +extern const TUTF8encoder tutf8e_encoder_iso_8859_8; +extern const TUTF8encoder tutf8e_encoder_iso_8859_9; +extern const TUTF8encoder tutf8e_encoder_windows_1250; +extern const TUTF8encoder tutf8e_encoder_windows_1251; +extern const TUTF8encoder tutf8e_encoder_windows_1252; +extern const TUTF8encoder tutf8e_encoder_windows_1253; +extern const TUTF8encoder tutf8e_encoder_windows_1254; +extern const TUTF8encoder tutf8e_encoder_windows_1255; +extern const TUTF8encoder tutf8e_encoder_windows_1256; +extern const TUTF8encoder tutf8e_encoder_windows_1257; +extern const TUTF8encoder tutf8e_encoder_windows_1258; + +#endif diff --git a/src/fluent-bit/lib/tutf8e/src/tutf8e.c b/src/fluent-bit/lib/tutf8e/src/tutf8e.c new file mode 100644 index 000000000..d91325123 --- /dev/null +++ b/src/fluent-bit/lib/tutf8e/src/tutf8e.c @@ -0,0 +1,680 @@ + +#include <tutf8e.h> + +#include <string.h> + +int tutf8e_string_length(const uint16_t *table, const char *input, const char *invalid, size_t *input_length, size_t *output_length) +{ + const size_t invalid_length = invalid ? strlen(invalid) : 0; + + const unsigned char *i; + for (i = (const unsigned char *) input; *i; ++i, (*input_length)++) { + const uint16_t c = table[*i]; + if (c<0x80) { + *output_length += 1; + continue; + } + if (c<0x800) { + *output_length += 2; + continue; + } + if (c<0xffff) { + *output_length += 3; + continue; + } + if (invalid) { + *output_length += invalid_length; + } + else { + return TUTF8E_INVALID; + } + } + return TUTF8E_OK; +} + +int tutf8e_string_encode(const uint16_t *table, const char *input, const char *invalid, char *output, size_t *output_length) +{ + int ret; + size_t input_length = 0; + size_t encoded_length = 0; + + ret = tutf8e_string_length(table, input, invalid, &input_length, &encoded_length); + if (!ret) + { + if (encoded_length+1 > *output_length) return TUTF8E_TOOLONG; + + ret = tutf8e_buffer_encode(table, input, input_length, invalid, output, output_length); + if (!ret) + { + output[encoded_length] = 0; + return TUTF8E_OK; + } + } + return ret; +} + +int tutf8e_buffer_length +( + const uint16_t *table, + const char *input, + size_t input_length, + const char *invalid, + size_t *length +) +{ + const size_t invalid_length = invalid ? strlen(invalid) : 0; + + const unsigned char *i; + for (i = (const unsigned char *) input; input_length; ++i, --input_length) { + const uint16_t c = table[*i]; + if (c<0x80) { + ++*length; + continue; + } + if (c<0x800) { + *length += 2; + continue; + } + if (c<0xffff) { + *length += 3; + continue; + } + if (invalid) { + *length += invalid_length; + } + else { + return TUTF8E_INVALID; + } + } + return TUTF8E_OK; +} + +int tutf8e_buffer_encode +( + const uint16_t *table, + const char *input, + size_t input_length, + const char *invalid, + char *output, + size_t *output_length +) +{ + size_t invalid_length = invalid ? strlen(invalid) : 0; + + size_t left = *output_length; + unsigned char *o = (unsigned char *) output; + const unsigned char *i; + for (i = (const unsigned char *) input; input_length; ++i, --input_length) { + const uint16_t c = table[*i]; + if (c<0x80) { + if (left<1) return TUTF8E_TOOLONG; + *(o++) = (uint8_t) c; + left -= 1; + continue; + } + if (c<0x800) { + if (left<2) return TUTF8E_TOOLONG; + *(o++) = 0xc0 | (uint8_t) (c>>6); + *(o++) = 0x80 | (uint8_t) (c&0x3f); + left -= 2; + continue; + } + if (c<0xffff) { + if (left<3) return TUTF8E_TOOLONG; + *(o++) = 0xe0 | (c>>12); + *(o++) = 0x80 | ((c>>6)&0x3f); + *(o++) = 0x80 | (c&0x3f); + left -= 3; + continue; + } + if (invalid) + { + if (left<invalid_length) return TUTF8E_TOOLONG; + if (invalid_length) { + memcpy(o, invalid, invalid_length); + o += invalid_length; + left -= invalid_length; + } + } + else { + return TUTF8E_INVALID; + } + } + *output_length -= left; + return TUTF8E_OK; +} + +const uint16_t tutf8e_iso_8859_1_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff, +}; + +const uint16_t tutf8e_iso_8859_10_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0104, 0x0112, 0x0122, 0x012a, 0x0128, 0x0136, 0x00a7, 0x013b, 0x0110, 0x0160, 0x0166, 0x017d, 0x00ad, 0x016a, 0x014a, + 0x00b0, 0x0105, 0x0113, 0x0123, 0x012b, 0x0129, 0x0137, 0x00b7, 0x013c, 0x0111, 0x0161, 0x0167, 0x017e, 0x2015, 0x016b, 0x014b, + 0x0100, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x012e, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x0116, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x0145, 0x014c, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x0168, 0x00d8, 0x0172, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, + 0x0101, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x012f, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x0117, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x0146, 0x014d, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x0169, 0x00f8, 0x0173, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x0138, +}; + +const uint16_t tutf8e_iso_8859_11_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0e01, 0x0e02, 0x0e03, 0x0e04, 0x0e05, 0x0e06, 0x0e07, 0x0e08, 0x0e09, 0x0e0a, 0x0e0b, 0x0e0c, 0x0e0d, 0x0e0e, 0x0e0f, + 0x0e10, 0x0e11, 0x0e12, 0x0e13, 0x0e14, 0x0e15, 0x0e16, 0x0e17, 0x0e18, 0x0e19, 0x0e1a, 0x0e1b, 0x0e1c, 0x0e1d, 0x0e1e, 0x0e1f, + 0x0e20, 0x0e21, 0x0e22, 0x0e23, 0x0e24, 0x0e25, 0x0e26, 0x0e27, 0x0e28, 0x0e29, 0x0e2a, 0x0e2b, 0x0e2c, 0x0e2d, 0x0e2e, 0x0e2f, + 0x0e30, 0x0e31, 0x0e32, 0x0e33, 0x0e34, 0x0e35, 0x0e36, 0x0e37, 0x0e38, 0x0e39, 0x0e3a, 0xffff, 0xffff, 0xffff, 0xffff, 0x0e3f, + 0x0e40, 0x0e41, 0x0e42, 0x0e43, 0x0e44, 0x0e45, 0x0e46, 0x0e47, 0x0e48, 0x0e49, 0x0e4a, 0x0e4b, 0x0e4c, 0x0e4d, 0x0e4e, 0x0e4f, + 0x0e50, 0x0e51, 0x0e52, 0x0e53, 0x0e54, 0x0e55, 0x0e56, 0x0e57, 0x0e58, 0x0e59, 0x0e5a, 0x0e5b, 0xffff, 0xffff, 0xffff, 0xffff, +}; + +const uint16_t tutf8e_iso_8859_13_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x201d, 0x00a2, 0x00a3, 0x00a4, 0x201e, 0x00a6, 0x00a7, 0x00d8, 0x00a9, 0x0156, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00c6, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x201c, 0x00b5, 0x00b6, 0x00b7, 0x00f8, 0x00b9, 0x0157, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00e6, + 0x0104, 0x012e, 0x0100, 0x0106, 0x00c4, 0x00c5, 0x0118, 0x0112, 0x010c, 0x00c9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012a, 0x013b, + 0x0160, 0x0143, 0x0145, 0x00d3, 0x014c, 0x00d5, 0x00d6, 0x00d7, 0x0172, 0x0141, 0x015a, 0x016a, 0x00dc, 0x017b, 0x017d, 0x00df, + 0x0105, 0x012f, 0x0101, 0x0107, 0x00e4, 0x00e5, 0x0119, 0x0113, 0x010d, 0x00e9, 0x017a, 0x0117, 0x0123, 0x0137, 0x012b, 0x013c, + 0x0161, 0x0144, 0x0146, 0x00f3, 0x014d, 0x00f5, 0x00f6, 0x00f7, 0x0173, 0x0142, 0x015b, 0x016b, 0x00fc, 0x017c, 0x017e, 0x2019, +}; + +const uint16_t tutf8e_iso_8859_14_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x1e02, 0x1e03, 0x00a3, 0x010a, 0x010b, 0x1e0a, 0x00a7, 0x1e80, 0x00a9, 0x1e82, 0x1e0b, 0x1ef2, 0x00ad, 0x00ae, 0x0178, + 0x1e1e, 0x1e1f, 0x0120, 0x0121, 0x1e40, 0x1e41, 0x00b6, 0x1e56, 0x1e81, 0x1e57, 0x1e83, 0x1e60, 0x1ef3, 0x1e84, 0x1e85, 0x1e61, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x0174, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x1e6a, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x0176, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x0175, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x1e6b, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x0177, 0x00ff, +}; + +const uint16_t tutf8e_iso_8859_15_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x20ac, 0x00a5, 0x0160, 0x00a7, 0x0161, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x017d, 0x00b5, 0x00b6, 0x00b7, 0x017e, 0x00b9, 0x00ba, 0x00bb, 0x0152, 0x0153, 0x0178, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff, +}; + +const uint16_t tutf8e_iso_8859_16_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0104, 0x0105, 0x0141, 0x20ac, 0x201e, 0x0160, 0x00a7, 0x0161, 0x00a9, 0x0218, 0x00ab, 0x0179, 0x00ad, 0x017a, 0x017b, + 0x00b0, 0x00b1, 0x010c, 0x0142, 0x017d, 0x201d, 0x00b6, 0x00b7, 0x017e, 0x010d, 0x0219, 0x00bb, 0x0152, 0x0153, 0x0178, 0x017c, + 0x00c0, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0106, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x0110, 0x0143, 0x00d2, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x015a, 0x0170, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x0118, 0x021a, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x0107, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x0111, 0x0144, 0x00f2, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x015b, 0x0171, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x0119, 0x021b, 0x00ff, +}; + +const uint16_t tutf8e_iso_8859_2_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0104, 0x02d8, 0x0141, 0x00a4, 0x013d, 0x015a, 0x00a7, 0x00a8, 0x0160, 0x015e, 0x0164, 0x0179, 0x00ad, 0x017d, 0x017b, + 0x00b0, 0x0105, 0x02db, 0x0142, 0x00b4, 0x013e, 0x015b, 0x02c7, 0x00b8, 0x0161, 0x015f, 0x0165, 0x017a, 0x02dd, 0x017e, 0x017c, + 0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e, + 0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7, 0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df, + 0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f, + 0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7, 0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9, +}; + +const uint16_t tutf8e_iso_8859_3_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0126, 0x02d8, 0x00a3, 0x00a4, 0xffff, 0x0124, 0x00a7, 0x00a8, 0x0130, 0x015e, 0x011e, 0x0134, 0x00ad, 0xffff, 0x017b, + 0x00b0, 0x0127, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x0125, 0x00b7, 0x00b8, 0x0131, 0x015f, 0x011f, 0x0135, 0x00bd, 0xffff, 0x017c, + 0x00c0, 0x00c1, 0x00c2, 0xffff, 0x00c4, 0x010a, 0x0108, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0xffff, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x0120, 0x00d6, 0x00d7, 0x011c, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x016c, 0x015c, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0xffff, 0x00e4, 0x010b, 0x0109, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0xffff, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x0121, 0x00f6, 0x00f7, 0x011d, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x016d, 0x015d, 0x02d9, +}; + +const uint16_t tutf8e_iso_8859_4_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0104, 0x0138, 0x0156, 0x00a4, 0x0128, 0x013b, 0x00a7, 0x00a8, 0x0160, 0x0112, 0x0122, 0x0166, 0x00ad, 0x017d, 0x00af, + 0x00b0, 0x0105, 0x02db, 0x0157, 0x00b4, 0x0129, 0x013c, 0x02c7, 0x00b8, 0x0161, 0x0113, 0x0123, 0x0167, 0x014a, 0x017e, 0x014b, + 0x0100, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x012e, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x0116, 0x00cd, 0x00ce, 0x012a, + 0x0110, 0x0145, 0x014c, 0x0136, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x0172, 0x00da, 0x00db, 0x00dc, 0x0168, 0x016a, 0x00df, + 0x0101, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x012f, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x0117, 0x00ed, 0x00ee, 0x012b, + 0x0111, 0x0146, 0x014d, 0x0137, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x0173, 0x00fa, 0x00fb, 0x00fc, 0x0169, 0x016b, 0x02d9, +}; + +const uint16_t tutf8e_iso_8859_5_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, 0x0406, 0x0407, 0x0408, 0x0409, 0x040a, 0x040b, 0x040c, 0x00ad, 0x040e, 0x040f, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, + 0x2116, 0x0451, 0x0452, 0x0453, 0x0454, 0x0455, 0x0456, 0x0457, 0x0458, 0x0459, 0x045a, 0x045b, 0x045c, 0x00a7, 0x045e, 0x045f, +}; + +const uint16_t tutf8e_iso_8859_6_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0xffff, 0xffff, 0xffff, 0x00a4, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0x060c, 0x00ad, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0x061b, 0xffff, 0xffff, 0xffff, 0x061f, + 0xffff, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f, + 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637, 0x0638, 0x0639, 0x063a, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647, 0x0648, 0x0649, 0x064a, 0x064b, 0x064c, 0x064d, 0x064e, 0x064f, + 0x0650, 0x0651, 0x0652, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, +}; + +const uint16_t tutf8e_iso_8859_7_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x2018, 0x2019, 0x00a3, 0x20ac, 0x20af, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x037a, 0x00ab, 0x00ac, 0x00ad, 0xffff, 0x2015, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x0384, 0x0385, 0x0386, 0x00b7, 0x0388, 0x0389, 0x038a, 0x00bb, 0x038c, 0x00bd, 0x038e, 0x038f, + 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f, + 0x03a0, 0x03a1, 0xffff, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, 0x03a8, 0x03a9, 0x03aa, 0x03ab, 0x03ac, 0x03ad, 0x03ae, 0x03af, + 0x03b0, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, + 0x03c0, 0x03c1, 0x03c2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, 0x03c8, 0x03c9, 0x03ca, 0x03cb, 0x03cc, 0x03cd, 0x03ce, 0xffff, +}; + +const uint16_t tutf8e_iso_8859_8_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0xffff, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00d7, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00f7, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0x2017, + 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0x05d7, 0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0x05dd, 0x05de, 0x05df, + 0x05e0, 0x05e1, 0x05e2, 0x05e3, 0x05e4, 0x05e5, 0x05e6, 0x05e7, 0x05e8, 0x05e9, 0x05ea, 0xffff, 0xffff, 0x200e, 0x200f, 0xffff, +}; + +const uint16_t tutf8e_iso_8859_9_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x011e, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x0130, 0x015e, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x011f, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x0131, 0x015f, 0x00ff, +}; + +const uint16_t tutf8e_windows_1250_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xffff, 0x201a, 0xffff, 0x201e, 0x2026, 0x2020, 0x2021, 0xffff, 0x2030, 0x0160, 0x2039, 0x015a, 0x0164, 0x017d, 0x0179, + 0xffff, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0xffff, 0x2122, 0x0161, 0x203a, 0x015b, 0x0165, 0x017e, 0x017a, + 0x00a0, 0x02c7, 0x02d8, 0x0141, 0x00a4, 0x0104, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x015e, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x017b, + 0x00b0, 0x00b1, 0x02db, 0x0142, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x0105, 0x015f, 0x00bb, 0x013d, 0x02dd, 0x013e, 0x017c, + 0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e, + 0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7, 0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df, + 0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f, + 0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7, 0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9, +}; + +const uint16_t tutf8e_windows_1251_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0402, 0x0403, 0x201a, 0x0453, 0x201e, 0x2026, 0x2020, 0x2021, 0x20ac, 0x2030, 0x0409, 0x2039, 0x040a, 0x040c, 0x040b, 0x040f, + 0x0452, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0xffff, 0x2122, 0x0459, 0x203a, 0x045a, 0x045c, 0x045b, 0x045f, + 0x00a0, 0x040e, 0x045e, 0x0408, 0x00a4, 0x0490, 0x00a6, 0x00a7, 0x0401, 0x00a9, 0x0404, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x0407, + 0x00b0, 0x00b1, 0x0406, 0x0456, 0x0491, 0x00b5, 0x00b6, 0x00b7, 0x0451, 0x2116, 0x0454, 0x00bb, 0x0458, 0x0405, 0x0455, 0x0457, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, +}; + +const uint16_t tutf8e_windows_1252_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xffff, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0xffff, 0x017d, 0xffff, + 0xffff, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0xffff, 0x017e, 0x0178, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff, +}; + +const uint16_t tutf8e_windows_1253_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xffff, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0xffff, 0x2030, 0xffff, 0x2039, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0xffff, 0x2122, 0xffff, 0x203a, 0xffff, 0xffff, 0xffff, 0xffff, + 0x00a0, 0x0385, 0x0386, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0xffff, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x2015, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x0384, 0x00b5, 0x00b6, 0x00b7, 0x0388, 0x0389, 0x038a, 0x00bb, 0x038c, 0x00bd, 0x038e, 0x038f, + 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f, + 0x03a0, 0x03a1, 0xffff, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, 0x03a8, 0x03a9, 0x03aa, 0x03ab, 0x03ac, 0x03ad, 0x03ae, 0x03af, + 0x03b0, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, + 0x03c0, 0x03c1, 0x03c2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, 0x03c8, 0x03c9, 0x03ca, 0x03cb, 0x03cc, 0x03cd, 0x03ce, 0xffff, +}; + +const uint16_t tutf8e_windows_1254_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xffff, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0xffff, 0xffff, 0xffff, + 0xffff, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0xffff, 0xffff, 0x0178, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x011e, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x0130, 0x015e, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x011f, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x0131, 0x015f, 0x00ff, +}; + +const uint16_t tutf8e_windows_1255_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xffff, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0xffff, 0x2039, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x02dc, 0x2122, 0xffff, 0x203a, 0xffff, 0xffff, 0xffff, 0xffff, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x20aa, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00d7, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00f7, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x05b0, 0x05b1, 0x05b2, 0x05b3, 0x05b4, 0x05b5, 0x05b6, 0x05b7, 0x05b8, 0x05b9, 0xffff, 0x05bb, 0x05bc, 0x05bd, 0x05be, 0x05bf, + 0x05c0, 0x05c1, 0x05c2, 0x05c3, 0x05f0, 0x05f1, 0x05f2, 0x05f3, 0x05f4, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0x05d7, 0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0x05dd, 0x05de, 0x05df, + 0x05e0, 0x05e1, 0x05e2, 0x05e3, 0x05e4, 0x05e5, 0x05e6, 0x05e7, 0x05e8, 0x05e9, 0x05ea, 0xffff, 0xffff, 0x200e, 0x200f, 0xffff, +}; + +const uint16_t tutf8e_windows_1256_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0x067e, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688, + 0x06af, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x06a9, 0x2122, 0x0691, 0x203a, 0x0153, 0x200c, 0x200d, 0x06ba, + 0x00a0, 0x060c, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x06be, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x061b, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x061f, + 0x06c1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f, + 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00d7, 0x0637, 0x0638, 0x0639, 0x063a, 0x0640, 0x0641, 0x0642, 0x0643, + 0x00e0, 0x0644, 0x00e2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x0649, 0x064a, 0x00ee, 0x00ef, + 0x064b, 0x064c, 0x064d, 0x064e, 0x00f4, 0x064f, 0x0650, 0x00f7, 0x0651, 0x00f9, 0x0652, 0x00fb, 0x00fc, 0x200e, 0x200f, 0x06d2, +}; + +const uint16_t tutf8e_windows_1257_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xffff, 0x201a, 0xffff, 0x201e, 0x2026, 0x2020, 0x2021, 0xffff, 0x2030, 0xffff, 0x2039, 0xffff, 0x00a8, 0x02c7, 0x00b8, + 0xffff, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0xffff, 0x2122, 0xffff, 0x203a, 0xffff, 0x00af, 0x02db, 0xffff, + 0x00a0, 0xffff, 0x00a2, 0x00a3, 0x00a4, 0xffff, 0x00a6, 0x00a7, 0x00d8, 0x00a9, 0x0156, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00c6, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00f8, 0x00b9, 0x0157, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00e6, + 0x0104, 0x012e, 0x0100, 0x0106, 0x00c4, 0x00c5, 0x0118, 0x0112, 0x010c, 0x00c9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012a, 0x013b, + 0x0160, 0x0143, 0x0145, 0x00d3, 0x014c, 0x00d5, 0x00d6, 0x00d7, 0x0172, 0x0141, 0x015a, 0x016a, 0x00dc, 0x017b, 0x017d, 0x00df, + 0x0105, 0x012f, 0x0101, 0x0107, 0x00e4, 0x00e5, 0x0119, 0x0113, 0x010d, 0x00e9, 0x017a, 0x0117, 0x0123, 0x0137, 0x012b, 0x013c, + 0x0161, 0x0144, 0x0146, 0x00f3, 0x014d, 0x00f5, 0x00f6, 0x00f7, 0x0173, 0x0142, 0x015b, 0x016b, 0x00fc, 0x017c, 0x017e, 0x02d9, +}; + +const uint16_t tutf8e_windows_1258_utf8[256] = +{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x20ac, 0xffff, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0xffff, 0x2039, 0x0152, 0xffff, 0xffff, 0xffff, + 0xffff, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x02dc, 0x2122, 0xffff, 0x203a, 0x0153, 0xffff, 0xffff, 0x0178, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x0300, 0x00cd, 0x00ce, 0x00cf, + 0x0110, 0x00d1, 0x0309, 0x00d3, 0x00d4, 0x01a0, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x01af, 0x0303, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x0301, 0x00ed, 0x00ee, 0x00ef, + 0x0111, 0x00f1, 0x0323, 0x00f3, 0x00f4, 0x01a1, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x01b0, 0x20ab, 0x00ff, +}; + +const TUTF8encoder tutf8e_encoder_iso_8859_1 = (TUTF8encoder) tutf8e_iso_8859_1_utf8; +const TUTF8encoder tutf8e_encoder_iso_8859_10 = (TUTF8encoder) tutf8e_iso_8859_10_utf8; +const TUTF8encoder tutf8e_encoder_iso_8859_11 = (TUTF8encoder) tutf8e_iso_8859_11_utf8; +const TUTF8encoder tutf8e_encoder_iso_8859_13 = (TUTF8encoder) tutf8e_iso_8859_13_utf8; +const TUTF8encoder tutf8e_encoder_iso_8859_14 = (TUTF8encoder) tutf8e_iso_8859_14_utf8; +const TUTF8encoder tutf8e_encoder_iso_8859_15 = (TUTF8encoder) tutf8e_iso_8859_15_utf8; +const TUTF8encoder tutf8e_encoder_iso_8859_16 = (TUTF8encoder) tutf8e_iso_8859_16_utf8; +const TUTF8encoder tutf8e_encoder_iso_8859_2 = (TUTF8encoder) tutf8e_iso_8859_2_utf8; +const TUTF8encoder tutf8e_encoder_iso_8859_3 = (TUTF8encoder) tutf8e_iso_8859_3_utf8; +const TUTF8encoder tutf8e_encoder_iso_8859_4 = (TUTF8encoder) tutf8e_iso_8859_4_utf8; +const TUTF8encoder tutf8e_encoder_iso_8859_5 = (TUTF8encoder) tutf8e_iso_8859_5_utf8; +const TUTF8encoder tutf8e_encoder_iso_8859_6 = (TUTF8encoder) tutf8e_iso_8859_6_utf8; +const TUTF8encoder tutf8e_encoder_iso_8859_7 = (TUTF8encoder) tutf8e_iso_8859_7_utf8; +const TUTF8encoder tutf8e_encoder_iso_8859_8 = (TUTF8encoder) tutf8e_iso_8859_8_utf8; +const TUTF8encoder tutf8e_encoder_iso_8859_9 = (TUTF8encoder) tutf8e_iso_8859_9_utf8; +const TUTF8encoder tutf8e_encoder_windows_1250 = (TUTF8encoder) tutf8e_windows_1250_utf8; +const TUTF8encoder tutf8e_encoder_windows_1251 = (TUTF8encoder) tutf8e_windows_1251_utf8; +const TUTF8encoder tutf8e_encoder_windows_1252 = (TUTF8encoder) tutf8e_windows_1252_utf8; +const TUTF8encoder tutf8e_encoder_windows_1253 = (TUTF8encoder) tutf8e_windows_1253_utf8; +const TUTF8encoder tutf8e_encoder_windows_1254 = (TUTF8encoder) tutf8e_windows_1254_utf8; +const TUTF8encoder tutf8e_encoder_windows_1255 = (TUTF8encoder) tutf8e_windows_1255_utf8; +const TUTF8encoder tutf8e_encoder_windows_1256 = (TUTF8encoder) tutf8e_windows_1256_utf8; +const TUTF8encoder tutf8e_encoder_windows_1257 = (TUTF8encoder) tutf8e_windows_1257_utf8; +const TUTF8encoder tutf8e_encoder_windows_1258 = (TUTF8encoder) tutf8e_windows_1258_utf8; + +TUTF8encoder tutf8e_encoder(const char *encoding) +{ + if (!strcmp(encoding, "iso-8859-1")) return tutf8e_encoder_iso_8859_1; + if (!strcmp(encoding, "iso-8859-10")) return tutf8e_encoder_iso_8859_10; + if (!strcmp(encoding, "iso-8859-11")) return tutf8e_encoder_iso_8859_11; + if (!strcmp(encoding, "iso-8859-13")) return tutf8e_encoder_iso_8859_13; + if (!strcmp(encoding, "iso-8859-14")) return tutf8e_encoder_iso_8859_14; + if (!strcmp(encoding, "iso-8859-15")) return tutf8e_encoder_iso_8859_15; + if (!strcmp(encoding, "iso-8859-16")) return tutf8e_encoder_iso_8859_16; + if (!strcmp(encoding, "iso-8859-2")) return tutf8e_encoder_iso_8859_2; + if (!strcmp(encoding, "iso-8859-3")) return tutf8e_encoder_iso_8859_3; + if (!strcmp(encoding, "iso-8859-4")) return tutf8e_encoder_iso_8859_4; + if (!strcmp(encoding, "iso-8859-5")) return tutf8e_encoder_iso_8859_5; + if (!strcmp(encoding, "iso-8859-6")) return tutf8e_encoder_iso_8859_6; + if (!strcmp(encoding, "iso-8859-7")) return tutf8e_encoder_iso_8859_7; + if (!strcmp(encoding, "iso-8859-8")) return tutf8e_encoder_iso_8859_8; + if (!strcmp(encoding, "iso-8859-9")) return tutf8e_encoder_iso_8859_9; + if (!strcmp(encoding, "windows-1250")) return tutf8e_encoder_windows_1250; + if (!strcmp(encoding, "windows-1251")) return tutf8e_encoder_windows_1251; + if (!strcmp(encoding, "windows-1252")) return tutf8e_encoder_windows_1252; + if (!strcmp(encoding, "windows-1253")) return tutf8e_encoder_windows_1253; + if (!strcmp(encoding, "windows-1254")) return tutf8e_encoder_windows_1254; + if (!strcmp(encoding, "windows-1255")) return tutf8e_encoder_windows_1255; + if (!strcmp(encoding, "windows-1256")) return tutf8e_encoder_windows_1256; + if (!strcmp(encoding, "windows-1257")) return tutf8e_encoder_windows_1257; + if (!strcmp(encoding, "windows-1258")) return tutf8e_encoder_windows_1258; + + return NULL; +} diff --git a/src/fluent-bit/lib/tutf8e/test/test.c b/src/fluent-bit/lib/tutf8e/test/test.c new file mode 100644 index 000000000..618cb1f79 --- /dev/null +++ b/src/fluent-bit/lib/tutf8e/test/test.c @@ -0,0 +1,474 @@ +#include <tutf8e.h> + +#include <stdio.h> +#include <string.h> +#include <stdlib.h> + +int main(int argc, char *argv[]) +{ + int pass = 0; + int fail = 0; + int ret; + char *copy; + size_t input_length, output_length; + char buffer[1024]; + + static const char english[] = { + 0x41, 0x20, 0x71, 0x75, 0x69, 0x63, 0x6b, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x6e, 0x20, 0x66, 0x6f, 0x78, 0x20, 0x6a, 0x75, 0x6d, 0x70, 0x73, 0x20, + 0x6f, 0x76, 0x65, 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6c, 0x61, 0x7a, 0x79, 0x20, 0x64, 0x6f, 0x67, 0x00, + }; + static const char finnish[] = { + 0x41, 0x6c, 0x62, 0x65, 0x72, 0x74, 0x20, 0x6f, 0x73, 0x74, 0x69, 0x20, 0x66, 0x61, 0x67, 0x6f, 0x74, 0x69, 0x6e, 0x20, 0x6a, 0x61, 0x20, 0x74, + 0xf6, 0x72, 0xe4, 0x79, 0x74, 0x74, 0x69, 0x20, 0x70, 0x75, 0x68, 0x6b, 0x75, 0x76, 0x61, 0x6e, 0x20, 0x6d, 0x65, 0x6c, 0x6f, 0x64, 0x69, 0x61, + 0x6e, 0x2e, 0x00, + }; + static const char czech[] = { + 0x4e, 0x65, 0x63, 0x68, 0xbb, 0x20, 0x6a, 0x69, 0xbe, 0x20, 0x68, 0xf8, 0xed, 0xb9, 0x6e, 0xe9, 0x20, 0x73, 0x61, 0x78, 0x6f, 0x66, 0x6f, 0x6e, + 0x79, 0x20, 0xef, 0xe1, 0x62, 0x6c, 0xf9, 0x20, 0x72, 0x6f, 0x7a, 0x65, 0x7a, 0x76, 0x75, 0xe8, 0xed, 0x20, 0x73, 0xed, 0xf2, 0x20, 0xfa, 0x64, + 0xec, 0x73, 0x6e, 0xfd, 0x6d, 0x69, 0x20, 0x74, 0xf3, 0x6e, 0x79, 0x20, 0x77, 0x61, 0x6c, 0x74, 0x7a, 0x75, 0x2c, 0x20, 0x74, 0x61, 0x6e, 0x67, + 0x61, 0x20, 0x61, 0x20, 0x71, 0x75, 0x69, 0x63, 0x6b, 0x73, 0x74, 0x65, 0x70, 0x75, 0x2e, 0x00, + }; + static const char turkish[] = { + 0x50, 0x69, 0x6a, 0x61, 0x6d, 0x61, 0x6c, 0xb9, 0x20, 0x68, 0x61, 0x73, 0x74, 0x61, 0x20, 0x79, 0x61, 0xbb, 0xb9, 0x7a, 0x20, 0xba, 0x6f, 0x66, + 0xf6, 0x72, 0x65, 0x20, 0xe7, 0x61, 0x62, 0x75, 0x63, 0x61, 0x6b, 0x20, 0x67, 0xfc, 0x76, 0x65, 0x6e, 0x64, 0x69, 0x2e, 0x00, + }; + static const char estonian[] = { + 0x50, 0xf5, 0x64, 0x75, 0x72, 0x20, 0x5a, 0x61, 0x67, 0x72, 0x65, 0x62, 0x69, 0x20, 0x74, 0xb9, 0x65, 0x6c, 0x6c, 0x6f, 0x6d, 0xe4, 0x6e, 0x67, + 0x69, 0x6a, 0x61, 0x2d, 0x66, 0xf6, 0x6c, 0x6a, 0x65, 0x74, 0x6f, 0x6e, 0x69, 0x73, 0x74, 0x20, 0x43, 0x69, 0x71, 0x6f, 0x20, 0x6b, 0xfc, 0x6c, + 0x6d, 0x65, 0x74, 0x61, 0x73, 0x20, 0x6b, 0x65, 0x68, 0x76, 0x61, 0x73, 0x20, 0x67, 0x61, 0x72, 0x61, 0x61, 0xbe, 0x69, 0x73, 0x00, + }; + static const char russian[] = { + 0xb2, 0x20, 0xe7, 0xd0, 0xe9, 0xd0, 0xe5, 0x20, 0xee, 0xd3, 0xd0, 0x20, 0xd6, 0xd8, 0xdb, 0x20, 0xd1, 0xeb, 0x20, 0xe6, 0xd8, 0xe2, 0xe0, 0xe3, + 0xe1, 0x3f, 0x20, 0xb4, 0xd0, 0x2c, 0x20, 0xdd, 0xde, 0x20, 0xe4, 0xd0, 0xdb, 0xec, 0xe8, 0xd8, 0xd2, 0xeb, 0xd9, 0x20, 0xed, 0xda, 0xd7, 0xd5, + 0xdc, 0xdf, 0xdb, 0xef, 0xe0, 0x21, 0x00, + }; + static const char greek[] = { + 0xe4, 0xe9, 0xe1, 0xf6, 0xf5, 0xeb, 0xdc, 0xee, 0xf4, 0xe5, 0x20, 0xe3, 0xe5, 0xed, 0xe9, 0xea, 0xdc, 0x20, 0xf4, 0xe7, 0x20, 0xe6, 0xf9, 0xde, + 0x20, 0xf3, 0xe1, 0xf2, 0x20, 0xe1, 0xf0, 0xfc, 0x20, 0xe2, 0xe1, 0xe8, 0xe5, 0xe9, 0xdc, 0x20, 0xf8, 0xf5, 0xf7, 0xe9, 0xea, 0xdc, 0x20, 0xf4, + 0xf1, 0xe1, 0xfd, 0xec, 0xe1, 0xf4, 0xe1, 0x00, + }; + static const char hebrew[] = { + 0xf2, 0xe8, 0xec, 0xf3, 0x20, 0xe0, 0xe1, 0xf7, 0x20, 0xf0, 0xf1, 0x20, 0xe3, 0xf8, 0xea, 0x20, 0xee, 0xe6, 0xe2, 0xef, 0x20, 0xf9, 0xe4, 0xfa, + 0xf4, 0xe5, 0xf6, 0xf5, 0x20, 0xeb, 0xe9, 0x20, 0xe7, 0xed, 0x00, + }; + static const char turkish2[] = { + 0x50, 0x69, 0x6a, 0x61, 0x6d, 0x61, 0x6c, 0xfd, 0x20, 0x68, 0x61, 0x73, 0x74, 0x61, 0x20, 0x79, 0x61, 0xf0, 0xfd, 0x7a, 0x20, 0xfe, 0x6f, 0x66, + 0xf6, 0x72, 0x65, 0x20, 0xe7, 0x61, 0x62, 0x75, 0x63, 0x61, 0x6b, 0x20, 0x67, 0xfc, 0x76, 0x65, 0x6e, 0x64, 0x69, 0x2e, 0x00, + }; + static const char swedish[] = { + 0x46, 0x6c, 0x79, 0x67, 0x61, 0x6e, 0x64, 0x65, 0x20, 0x62, 0xe4, 0x63, 0x6b, 0x61, 0x73, 0x69, 0x6e, 0x65, 0x72, 0x20, 0x73, 0xf6, 0x6b, 0x61, + 0x20, 0x68, 0x77, 0x69, 0x6c, 0x61, 0x20, 0x70, 0xe5, 0x20, 0x6d, 0x6a, 0x75, 0x6b, 0x61, 0x20, 0x74, 0x75, 0x76, 0x6f, 0x72, 0x2e, 0x00, + }; + static const char thai[] = { + 0xe0, 0xbb, 0xe7, 0xb9, 0xc1, 0xb9, 0xd8, 0xc9, 0xc2, 0xec, 0xca, 0xd8, 0xb4, 0xbb, 0xc3, 0xd0, 0xe0, 0xca, 0xc3, 0xd4, 0xb0, 0xe0, 0xc5, 0xd4, + 0xc8, 0xa4, 0xd8, 0xb3, 0xa4, 0xe8, 0xd2, 0x20, 0xa1, 0xc7, 0xe8, 0xd2, 0xba, 0xc3, 0xc3, 0xb4, 0xd2, 0xbd, 0xd9, 0xa7, 0xca, 0xd1, 0xb5, 0xc7, + 0xec, 0xe0, 0xb4, 0xc3, 0xd1, 0xa8, 0xa9, 0xd2, 0xb9, 0x20, 0xa8, 0xa7, 0xbd, 0xe8, 0xd2, 0xbf, 0xd1, 0xb9, 0xbe, 0xd1, 0xb2, 0xb9, 0xd2, 0xc7, + 0xd4, 0xaa, 0xd2, 0xa1, 0xd2, 0xc3, 0x20, 0xcd, 0xc2, 0xe8, 0xd2, 0xc5, 0xe9, 0xd2, 0xa7, 0xbc, 0xc5, 0xd2, 0xad, 0xc4, 0xe5, 0xe0, 0xa2, 0xe8, + 0xb9, 0xa6, 0xe8, 0xd2, 0xba, 0xd5, 0xb1, 0xd2, 0xe3, 0xa4, 0xc3, 0x20, 0xe4, 0xc1, 0xe8, 0xb6, 0xd7, 0xcd, 0xe2, 0xb7, 0xc9, 0xe2, 0xa1, 0xc3, + 0xb8, 0xe1, 0xaa, 0xe8, 0xa7, 0xab, 0xd1, 0xb4, 0xce, 0xd6, 0xb4, 0xce, 0xd1, 0xb4, 0xb4, 0xe8, 0xd2, 0x20, 0xcb, 0xd1, 0xb4, 0xcd, 0xc0, 0xd1, + 0xc2, 0xe0, 0xcb, 0xc1, 0xd7, 0xcd, 0xb9, 0xa1, 0xd5, 0xcc, 0xd2, 0xcd, 0xd1, 0xaa, 0xac, 0xd2, 0xca, 0xd1, 0xc2, 0x20, 0xbb, 0xaf, 0xd4, 0xba, + 0xd1, 0xb5, 0xd4, 0xbb, 0xc3, 0xd0, 0xbe, 0xc4, 0xb5, 0xd4, 0xa1, 0xae, 0xa1, 0xd3, 0xcb, 0xb9, 0xb4, 0xe3, 0xa8, 0x20, 0xbe, 0xd9, 0xb4, 0xa8, + 0xd2, 0xe3, 0xcb, 0xe9, 0xa8, 0xea, 0xd0, 0xe6, 0x20, 0xa8, 0xeb, 0xd2, 0xe6, 0x20, 0xb9, 0xe8, 0xd2, 0xbf, 0xd1, 0xa7, 0xe0, 0xcd, 0xc2, 0xcf, + 0x00, + }; + static const char polish[] = { + 0x4a, 0x65, 0xfd, 0x75, 0x20, 0x6b, 0x6c, 0xe0, 0x74, 0x77, 0x2c, 0x20, 0x73, 0x70, 0xf9, 0xf3, 0x64, 0xea, 0x20, 0x46, 0x69, 0x6e, 0x6f, 0x6d, + 0x20, 0x63, 0x7a, 0xe6, 0xfa, 0xe3, 0x20, 0x67, 0x72, 0x79, 0x20, 0x68, 0x61, 0xf1, 0x62, 0x21, 0x00, + }; + + static const char englishUTF8[] = { + 0x41, 0x20, 0x71, 0x75, 0x69, 0x63, 0x6b, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x6e, 0x20, 0x66, 0x6f, 0x78, 0x20, 0x6a, 0x75, 0x6d, 0x70, 0x73, 0x20, + 0x6f, 0x76, 0x65, 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6c, 0x61, 0x7a, 0x79, 0x20, 0x64, 0x6f, 0x67, 0x00, + }; + static const char finnishUTF8[] = { + 0x41, 0x6c, 0x62, 0x65, 0x72, 0x74, 0x20, 0x6f, 0x73, 0x74, 0x69, 0x20, 0x66, 0x61, 0x67, 0x6f, 0x74, 0x69, 0x6e, 0x20, 0x6a, 0x61, 0x20, 0x74, + 0xc3, 0xb6, 0x72, 0xc3, 0xa4, 0x79, 0x74, 0x74, 0x69, 0x20, 0x70, 0x75, 0x68, 0x6b, 0x75, 0x76, 0x61, 0x6e, 0x20, 0x6d, 0x65, 0x6c, 0x6f, 0x64, + 0x69, 0x61, 0x6e, 0x2e, 0x00, + }; + static const char czechUTF8[] = { + 0x4e, 0x65, 0x63, 0x68, 0xc5, 0xa5, 0x20, 0x6a, 0x69, 0xc5, 0xbe, 0x20, 0x68, 0xc5, 0x99, 0xc3, 0xad, 0xc5, 0xa1, 0x6e, 0xc3, 0xa9, 0x20, 0x73, + 0x61, 0x78, 0x6f, 0x66, 0x6f, 0x6e, 0x79, 0x20, 0xc4, 0x8f, 0xc3, 0xa1, 0x62, 0x6c, 0xc5, 0xaf, 0x20, 0x72, 0x6f, 0x7a, 0x65, 0x7a, 0x76, 0x75, + 0xc4, 0x8d, 0xc3, 0xad, 0x20, 0x73, 0xc3, 0xad, 0xc5, 0x88, 0x20, 0xc3, 0xba, 0x64, 0xc4, 0x9b, 0x73, 0x6e, 0xc3, 0xbd, 0x6d, 0x69, 0x20, 0x74, + 0xc3, 0xb3, 0x6e, 0x79, 0x20, 0x77, 0x61, 0x6c, 0x74, 0x7a, 0x75, 0x2c, 0x20, 0x74, 0x61, 0x6e, 0x67, 0x61, 0x20, 0x61, 0x20, 0x71, 0x75, 0x69, + 0x63, 0x6b, 0x73, 0x74, 0x65, 0x70, 0x75, 0x2e, 0x00, + }; + static const char turkishUTF8[] = { + 0x50, 0x69, 0x6a, 0x61, 0x6d, 0x61, 0x6c, 0xc4, 0xb1, 0x20, 0x68, 0x61, 0x73, 0x74, 0x61, 0x20, 0x79, 0x61, 0xc4, 0x9f, 0xc4, 0xb1, 0x7a, 0x20, + 0xc5, 0x9f, 0x6f, 0x66, 0xc3, 0xb6, 0x72, 0x65, 0x20, 0xc3, 0xa7, 0x61, 0x62, 0x75, 0x63, 0x61, 0x6b, 0x20, 0x67, 0xc3, 0xbc, 0x76, 0x65, 0x6e, + 0x64, 0x69, 0x2e, 0x00, + }; + static const char estonianUTF8[] = { + 0x50, 0xc3, 0xb5, 0x64, 0x75, 0x72, 0x20, 0x5a, 0x61, 0x67, 0x72, 0x65, 0x62, 0x69, 0x20, 0x74, 0xc5, 0xa1, 0x65, 0x6c, 0x6c, 0x6f, 0x6d, 0xc3, + 0xa4, 0x6e, 0x67, 0x69, 0x6a, 0x61, 0x2d, 0x66, 0xc3, 0xb6, 0x6c, 0x6a, 0x65, 0x74, 0x6f, 0x6e, 0x69, 0x73, 0x74, 0x20, 0x43, 0x69, 0x71, 0x6f, + 0x20, 0x6b, 0xc3, 0xbc, 0x6c, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x20, 0x6b, 0x65, 0x68, 0x76, 0x61, 0x73, 0x20, 0x67, 0x61, 0x72, 0x61, 0x61, 0xc5, + 0xbe, 0x69, 0x73, 0x00, + }; + static const char russianUTF8[] = { + 0xd0, 0x92, 0x20, 0xd1, 0x87, 0xd0, 0xb0, 0xd1, 0x89, 0xd0, 0xb0, 0xd1, 0x85, 0x20, 0xd1, 0x8e, 0xd0, 0xb3, 0xd0, 0xb0, 0x20, 0xd0, 0xb6, 0xd0, + 0xb8, 0xd0, 0xbb, 0x20, 0xd0, 0xb1, 0xd1, 0x8b, 0x20, 0xd1, 0x86, 0xd0, 0xb8, 0xd1, 0x82, 0xd1, 0x80, 0xd1, 0x83, 0xd1, 0x81, 0x3f, 0x20, 0xd0, + 0x94, 0xd0, 0xb0, 0x2c, 0x20, 0xd0, 0xbd, 0xd0, 0xbe, 0x20, 0xd1, 0x84, 0xd0, 0xb0, 0xd0, 0xbb, 0xd1, 0x8c, 0xd1, 0x88, 0xd0, 0xb8, 0xd0, 0xb2, + 0xd1, 0x8b, 0xd0, 0xb9, 0x20, 0xd1, 0x8d, 0xd0, 0xba, 0xd0, 0xb7, 0xd0, 0xb5, 0xd0, 0xbc, 0xd0, 0xbf, 0xd0, 0xbb, 0xd1, 0x8f, 0xd1, 0x80, 0x21, + 0x00, + }; + static const char greekUTF8[] = { + 0xce, 0xb4, 0xce, 0xb9, 0xce, 0xb1, 0xcf, 0x86, 0xcf, 0x85, 0xce, 0xbb, 0xce, 0xac, 0xce, 0xbe, 0xcf, 0x84, 0xce, 0xb5, 0x20, 0xce, 0xb3, 0xce, + 0xb5, 0xce, 0xbd, 0xce, 0xb9, 0xce, 0xba, 0xce, 0xac, 0x20, 0xcf, 0x84, 0xce, 0xb7, 0x20, 0xce, 0xb6, 0xcf, 0x89, 0xce, 0xae, 0x20, 0xcf, 0x83, + 0xce, 0xb1, 0xcf, 0x82, 0x20, 0xce, 0xb1, 0xcf, 0x80, 0xcf, 0x8c, 0x20, 0xce, 0xb2, 0xce, 0xb1, 0xce, 0xb8, 0xce, 0xb5, 0xce, 0xb9, 0xce, 0xac, + 0x20, 0xcf, 0x88, 0xcf, 0x85, 0xcf, 0x87, 0xce, 0xb9, 0xce, 0xba, 0xce, 0xac, 0x20, 0xcf, 0x84, 0xcf, 0x81, 0xce, 0xb1, 0xcf, 0x8d, 0xce, 0xbc, + 0xce, 0xb1, 0xcf, 0x84, 0xce, 0xb1, 0x00, + }; + static const char hebrewUTF8[] = { + 0xd7, 0xa2, 0xd7, 0x98, 0xd7, 0x9c, 0xd7, 0xa3, 0x20, 0xd7, 0x90, 0xd7, 0x91, 0xd7, 0xa7, 0x20, 0xd7, 0xa0, 0xd7, 0xa1, 0x20, 0xd7, 0x93, 0xd7, + 0xa8, 0xd7, 0x9a, 0x20, 0xd7, 0x9e, 0xd7, 0x96, 0xd7, 0x92, 0xd7, 0x9f, 0x20, 0xd7, 0xa9, 0xd7, 0x94, 0xd7, 0xaa, 0xd7, 0xa4, 0xd7, 0x95, 0xd7, + 0xa6, 0xd7, 0xa5, 0x20, 0xd7, 0x9b, 0xd7, 0x99, 0x20, 0xd7, 0x97, 0xd7, 0x9d, 0x00, + }; + static const char turkish2UTF8[] = { + 0x50, 0x69, 0x6a, 0x61, 0x6d, 0x61, 0x6c, 0xc4, 0xb1, 0x20, 0x68, 0x61, 0x73, 0x74, 0x61, 0x20, 0x79, 0x61, 0xc4, 0x9f, 0xc4, 0xb1, 0x7a, 0x20, + 0xc5, 0x9f, 0x6f, 0x66, 0xc3, 0xb6, 0x72, 0x65, 0x20, 0xc3, 0xa7, 0x61, 0x62, 0x75, 0x63, 0x61, 0x6b, 0x20, 0x67, 0xc3, 0xbc, 0x76, 0x65, 0x6e, + 0x64, 0x69, 0x2e, 0x00, + }; + static const char swedishUTF8[] = { + 0x46, 0x6c, 0x79, 0x67, 0x61, 0x6e, 0x64, 0x65, 0x20, 0x62, 0xc3, 0xa4, 0x63, 0x6b, 0x61, 0x73, 0x69, 0x6e, 0x65, 0x72, 0x20, 0x73, 0xc3, 0xb6, + 0x6b, 0x61, 0x20, 0x68, 0x77, 0x69, 0x6c, 0x61, 0x20, 0x70, 0xc3, 0xa5, 0x20, 0x6d, 0x6a, 0x75, 0x6b, 0x61, 0x20, 0x74, 0x75, 0x76, 0x6f, 0x72, + 0x2e, 0x00, + }; + static const char thaiUTF8[] = { + 0xe0, 0xb9, 0x80, 0xe0, 0xb8, 0x9b, 0xe0, 0xb9, 0x87, 0xe0, 0xb8, 0x99, 0xe0, 0xb8, 0xa1, 0xe0, 0xb8, 0x99, 0xe0, 0xb8, 0xb8, 0xe0, 0xb8, 0xa9, + 0xe0, 0xb8, 0xa2, 0xe0, 0xb9, 0x8c, 0xe0, 0xb8, 0xaa, 0xe0, 0xb8, 0xb8, 0xe0, 0xb8, 0x94, 0xe0, 0xb8, 0x9b, 0xe0, 0xb8, 0xa3, 0xe0, 0xb8, 0xb0, + 0xe0, 0xb9, 0x80, 0xe0, 0xb8, 0xaa, 0xe0, 0xb8, 0xa3, 0xe0, 0xb8, 0xb4, 0xe0, 0xb8, 0x90, 0xe0, 0xb9, 0x80, 0xe0, 0xb8, 0xa5, 0xe0, 0xb8, 0xb4, + 0xe0, 0xb8, 0xa8, 0xe0, 0xb8, 0x84, 0xe0, 0xb8, 0xb8, 0xe0, 0xb8, 0x93, 0xe0, 0xb8, 0x84, 0xe0, 0xb9, 0x88, 0xe0, 0xb8, 0xb2, 0x20, 0xe0, 0xb8, + 0x81, 0xe0, 0xb8, 0xa7, 0xe0, 0xb9, 0x88, 0xe0, 0xb8, 0xb2, 0xe0, 0xb8, 0x9a, 0xe0, 0xb8, 0xa3, 0xe0, 0xb8, 0xa3, 0xe0, 0xb8, 0x94, 0xe0, 0xb8, + 0xb2, 0xe0, 0xb8, 0x9d, 0xe0, 0xb8, 0xb9, 0xe0, 0xb8, 0x87, 0xe0, 0xb8, 0xaa, 0xe0, 0xb8, 0xb1, 0xe0, 0xb8, 0x95, 0xe0, 0xb8, 0xa7, 0xe0, 0xb9, + 0x8c, 0xe0, 0xb9, 0x80, 0xe0, 0xb8, 0x94, 0xe0, 0xb8, 0xa3, 0xe0, 0xb8, 0xb1, 0xe0, 0xb8, 0x88, 0xe0, 0xb8, 0x89, 0xe0, 0xb8, 0xb2, 0xe0, 0xb8, + 0x99, 0x20, 0xe0, 0xb8, 0x88, 0xe0, 0xb8, 0x87, 0xe0, 0xb8, 0x9d, 0xe0, 0xb9, 0x88, 0xe0, 0xb8, 0xb2, 0xe0, 0xb8, 0x9f, 0xe0, 0xb8, 0xb1, 0xe0, + 0xb8, 0x99, 0xe0, 0xb8, 0x9e, 0xe0, 0xb8, 0xb1, 0xe0, 0xb8, 0x92, 0xe0, 0xb8, 0x99, 0xe0, 0xb8, 0xb2, 0xe0, 0xb8, 0xa7, 0xe0, 0xb8, 0xb4, 0xe0, + 0xb8, 0x8a, 0xe0, 0xb8, 0xb2, 0xe0, 0xb8, 0x81, 0xe0, 0xb8, 0xb2, 0xe0, 0xb8, 0xa3, 0x20, 0xe0, 0xb8, 0xad, 0xe0, 0xb8, 0xa2, 0xe0, 0xb9, 0x88, + 0xe0, 0xb8, 0xb2, 0xe0, 0xb8, 0xa5, 0xe0, 0xb9, 0x89, 0xe0, 0xb8, 0xb2, 0xe0, 0xb8, 0x87, 0xe0, 0xb8, 0x9c, 0xe0, 0xb8, 0xa5, 0xe0, 0xb8, 0xb2, + 0xe0, 0xb8, 0x8d, 0xe0, 0xb8, 0xa4, 0xe0, 0xb9, 0x85, 0xe0, 0xb9, 0x80, 0xe0, 0xb8, 0x82, 0xe0, 0xb9, 0x88, 0xe0, 0xb8, 0x99, 0xe0, 0xb8, 0x86, + 0xe0, 0xb9, 0x88, 0xe0, 0xb8, 0xb2, 0xe0, 0xb8, 0x9a, 0xe0, 0xb8, 0xb5, 0xe0, 0xb8, 0x91, 0xe0, 0xb8, 0xb2, 0xe0, 0xb9, 0x83, 0xe0, 0xb8, 0x84, + 0xe0, 0xb8, 0xa3, 0x20, 0xe0, 0xb9, 0x84, 0xe0, 0xb8, 0xa1, 0xe0, 0xb9, 0x88, 0xe0, 0xb8, 0x96, 0xe0, 0xb8, 0xb7, 0xe0, 0xb8, 0xad, 0xe0, 0xb9, + 0x82, 0xe0, 0xb8, 0x97, 0xe0, 0xb8, 0xa9, 0xe0, 0xb9, 0x82, 0xe0, 0xb8, 0x81, 0xe0, 0xb8, 0xa3, 0xe0, 0xb8, 0x98, 0xe0, 0xb9, 0x81, 0xe0, 0xb8, + 0x8a, 0xe0, 0xb9, 0x88, 0xe0, 0xb8, 0x87, 0xe0, 0xb8, 0x8b, 0xe0, 0xb8, 0xb1, 0xe0, 0xb8, 0x94, 0xe0, 0xb8, 0xae, 0xe0, 0xb8, 0xb6, 0xe0, 0xb8, + 0x94, 0xe0, 0xb8, 0xae, 0xe0, 0xb8, 0xb1, 0xe0, 0xb8, 0x94, 0xe0, 0xb8, 0x94, 0xe0, 0xb9, 0x88, 0xe0, 0xb8, 0xb2, 0x20, 0xe0, 0xb8, 0xab, 0xe0, + 0xb8, 0xb1, 0xe0, 0xb8, 0x94, 0xe0, 0xb8, 0xad, 0xe0, 0xb8, 0xa0, 0xe0, 0xb8, 0xb1, 0xe0, 0xb8, 0xa2, 0xe0, 0xb9, 0x80, 0xe0, 0xb8, 0xab, 0xe0, + 0xb8, 0xa1, 0xe0, 0xb8, 0xb7, 0xe0, 0xb8, 0xad, 0xe0, 0xb8, 0x99, 0xe0, 0xb8, 0x81, 0xe0, 0xb8, 0xb5, 0xe0, 0xb8, 0xac, 0xe0, 0xb8, 0xb2, 0xe0, + 0xb8, 0xad, 0xe0, 0xb8, 0xb1, 0xe0, 0xb8, 0x8a, 0xe0, 0xb8, 0x8c, 0xe0, 0xb8, 0xb2, 0xe0, 0xb8, 0xaa, 0xe0, 0xb8, 0xb1, 0xe0, 0xb8, 0xa2, 0x20, + 0xe0, 0xb8, 0x9b, 0xe0, 0xb8, 0x8f, 0xe0, 0xb8, 0xb4, 0xe0, 0xb8, 0x9a, 0xe0, 0xb8, 0xb1, 0xe0, 0xb8, 0x95, 0xe0, 0xb8, 0xb4, 0xe0, 0xb8, 0x9b, + 0xe0, 0xb8, 0xa3, 0xe0, 0xb8, 0xb0, 0xe0, 0xb8, 0x9e, 0xe0, 0xb8, 0xa4, 0xe0, 0xb8, 0x95, 0xe0, 0xb8, 0xb4, 0xe0, 0xb8, 0x81, 0xe0, 0xb8, 0x8e, + 0xe0, 0xb8, 0x81, 0xe0, 0xb8, 0xb3, 0xe0, 0xb8, 0xab, 0xe0, 0xb8, 0x99, 0xe0, 0xb8, 0x94, 0xe0, 0xb9, 0x83, 0xe0, 0xb8, 0x88, 0x20, 0xe0, 0xb8, + 0x9e, 0xe0, 0xb8, 0xb9, 0xe0, 0xb8, 0x94, 0xe0, 0xb8, 0x88, 0xe0, 0xb8, 0xb2, 0xe0, 0xb9, 0x83, 0xe0, 0xb8, 0xab, 0xe0, 0xb9, 0x89, 0xe0, 0xb8, + 0x88, 0xe0, 0xb9, 0x8a, 0xe0, 0xb8, 0xb0, 0xe0, 0xb9, 0x86, 0x20, 0xe0, 0xb8, 0x88, 0xe0, 0xb9, 0x8b, 0xe0, 0xb8, 0xb2, 0xe0, 0xb9, 0x86, 0x20, + 0xe0, 0xb8, 0x99, 0xe0, 0xb9, 0x88, 0xe0, 0xb8, 0xb2, 0xe0, 0xb8, 0x9f, 0xe0, 0xb8, 0xb1, 0xe0, 0xb8, 0x87, 0xe0, 0xb9, 0x80, 0xe0, 0xb8, 0xad, + 0xe0, 0xb8, 0xa2, 0xe0, 0xb8, 0xaf, 0x00, + }; + static const char polishUTF8[] = { + 0x4a, 0x65, 0xc5, 0xbc, 0x75, 0x20, 0x6b, 0x6c, 0xc4, 0x85, 0x74, 0x77, 0x2c, 0x20, 0x73, 0x70, 0xc5, 0x82, 0xc3, 0xb3, 0x64, 0xc5, 0xba, 0x20, + 0x46, 0x69, 0x6e, 0x6f, 0x6d, 0x20, 0x63, 0x7a, 0xc4, 0x99, 0xc5, 0x9b, 0xc4, 0x87, 0x20, 0x67, 0x72, 0x79, 0x20, 0x68, 0x61, 0xc5, 0x84, 0x62, + 0x21, 0x00, + }; + + /* string encode to UTF8, error on invalid input */ + output_length = sizeof(buffer); + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_1, english, NULL, buffer, &output_length); + if (!ret && !strcmp(buffer, englishUTF8)) { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode english test\n"); + fail++; + } + + output_length = sizeof(buffer); + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_1, finnish, NULL, buffer, &output_length); + if (!ret && !strcmp(buffer, finnishUTF8)) { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode finnish test\n"); + fail++; + } + + output_length = sizeof(buffer); + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_2, czech, NULL, buffer, &output_length); + if (!ret && !strcmp(buffer, czechUTF8)) { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode czech test\n"); + fail++; + } + + output_length = sizeof(buffer); + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_3, turkish, NULL, buffer, &output_length); + if (!ret && !strcmp(buffer, turkishUTF8)) { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode turkish test\n"); + fail++; + } + + output_length = sizeof(buffer); + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_4, estonian, NULL, buffer, &output_length); + if (!ret && !strcmp(buffer, estonianUTF8)) { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode estonian test\n"); + fail++; + } + + output_length = sizeof(buffer); + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_5, russian, NULL, buffer, &output_length); + if (!ret && !strcmp(buffer, russianUTF8)) { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode russian test\n"); + fail++; + } + + output_length = sizeof(buffer); + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_7, greek, NULL, buffer, &output_length); + if (!ret && !strcmp(buffer, greekUTF8)) { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode greek test\n"); + fail++; + } + + output_length = sizeof(buffer); + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_8, hebrew, NULL, buffer, &output_length); + if (!ret && !strcmp(buffer, hebrewUTF8)) { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode hebrew test\n"); + fail++; + } + + output_length = sizeof(buffer); + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_9, turkish2, NULL, buffer, &output_length); + if (!ret && !strcmp(buffer, turkish2UTF8)) { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode turkish2 test\n"); + fail++; + } + + output_length = sizeof(buffer); + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_10, swedish, NULL, buffer, &output_length); + if (!ret && !strcmp(buffer, swedishUTF8)) { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode swedish test\n"); + fail++; + } + + output_length = sizeof(buffer); + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_11, thai, NULL, buffer, &output_length); + if (!ret && !strcmp(buffer, thaiUTF8)) { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode thai test\n"); + fail++; + } + + output_length = sizeof(buffer); + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_13, polish, NULL, buffer, &output_length); + if (!ret && !strcmp(buffer, polishUTF8)) { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode polish test\n"); + fail++; + } + + + /* buffer encode to UTF8, error on invalid input */ + input_length = strlen(english); + output_length = sizeof(buffer); + ret = tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_1, english, input_length, NULL, buffer, &output_length); + if (!ret && (output_length+1)==sizeof(englishUTF8) && !strncmp(buffer, englishUTF8, output_length)) { + pass++; + } else { + printf("Failed to encode english test\n"); + fail++; + } + + input_length = strlen(finnish); + output_length = sizeof(buffer); + ret = tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_1, finnish, input_length, NULL, buffer, &output_length); + if (!ret && (output_length+1)==sizeof(finnishUTF8) && !strncmp(buffer, finnishUTF8, output_length)) { + pass++; + } else { + printf("Failed to encode finnish test\n"); + fail++; + } + + input_length = strlen(czech); + output_length = sizeof(buffer); + ret = tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_2, czech, input_length, NULL, buffer, &output_length); + if (!ret && (output_length+1)==sizeof(czechUTF8) && !strncmp(buffer, czechUTF8, output_length)) { + pass++; + } else { + printf("Failed to encode czech test\n"); + fail++; + } + + input_length = strlen(turkish); + output_length = sizeof(buffer); + ret = tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_3, turkish, input_length, NULL, buffer, &output_length); + if (!ret && (output_length+1)==sizeof(turkishUTF8) && !strncmp(buffer, turkishUTF8, output_length)) { + pass++; + } else { + printf("Failed to encode turkish test\n"); + fail++; + } + + input_length = strlen(estonian); + output_length = sizeof(buffer); + ret = tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_4, estonian, input_length, NULL, buffer, &output_length); + if (!ret && (output_length+1)==sizeof(estonianUTF8) && !strncmp(buffer, estonianUTF8, output_length)) { + pass++; + } else { + printf("Failed to encode estonian test\n"); + fail++; + } + + input_length = strlen(russian); + output_length = sizeof(buffer); + ret = tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_5, russian, input_length, NULL, buffer, &output_length); + if (!ret && (output_length+1)==sizeof(russianUTF8) && !strncmp(buffer, russianUTF8, output_length)) { + pass++; + } else { + printf("Failed to encode russian test\n"); + fail++; + } + + input_length = strlen(greek); + output_length = sizeof(buffer); + ret = tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_7, greek, input_length, NULL, buffer, &output_length); + if (!ret && (output_length+1)==sizeof(greekUTF8) && !strncmp(buffer, greekUTF8, output_length)) { + pass++; + } else { + printf("Failed to encode greek test\n"); + fail++; + } + + input_length = strlen(hebrew); + output_length = sizeof(buffer); + ret = tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_8, hebrew, input_length, NULL, buffer, &output_length); + if (!ret && (output_length+1)==sizeof(hebrewUTF8) && !strncmp(buffer, hebrewUTF8, output_length)) { + pass++; + } else { + printf("Failed to encode hebrew test\n"); + fail++; + } + + input_length = strlen(turkish2); + output_length = sizeof(buffer); + ret = tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_9, turkish2, input_length, NULL, buffer, &output_length); + if (!ret && (output_length+1)==sizeof(turkish2UTF8) && !strncmp(buffer, turkish2UTF8, output_length)) { + pass++; + } else { + printf("Failed to encode turkish2 test\n"); + fail++; + } + + input_length = strlen(swedish); + output_length = sizeof(buffer); + ret = tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_10, swedish, input_length, NULL, buffer, &output_length); + if (!ret && (output_length+1)==sizeof(swedishUTF8) && !strncmp(buffer, swedishUTF8, output_length)) { + pass++; + } else { + printf("Failed to encode swedish test\n"); + fail++; + } + + input_length = strlen(thai); + output_length = sizeof(buffer); + ret = tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_11, thai, input_length, NULL, buffer, &output_length); + if (!ret && (output_length+1)==sizeof(thaiUTF8) && !strncmp(buffer, thaiUTF8, output_length)) { + pass++; + } else { + printf("Failed to encode thai test\n"); + fail++; + } + + input_length = strlen(polish); + output_length = sizeof(buffer); + ret = tutf8e_encoder_buffer_encode(tutf8e_encoder_iso_8859_13, polish, input_length, NULL, buffer, &output_length); + if (!ret && (output_length+1)==sizeof(polishUTF8) && !strncmp(buffer, polishUTF8, output_length)) { + pass++; + } else { + printf("Failed to encode polish test\n"); + fail++; + } + + + /* string encode to UTF8, first input character invalid -> ? */ + output_length = sizeof(buffer); + copy = strdup(greek); + copy[0] = 255; + buffer[0] = 255; + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_7, copy, "?", buffer, &output_length); + if (!ret && buffer[0]=='?') { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode greek test\n"); + fail++; + } + free(copy); + + output_length = sizeof(buffer); + copy = strdup(thai); + copy[0] = 255; + buffer[0] = 255; + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_11, copy, "?", buffer, &output_length); + if (!ret && buffer[0]=='?') { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode thai test\n"); + fail++; + } + free(copy); + + + /* string encode to UTF8, first input character invalid -> [INVALID] */ + output_length = sizeof(buffer); + copy = strdup(greek); + copy[0] = 255; + buffer[0] = 255; + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_7, copy, "[INVALID]", buffer, &output_length); + if (!ret && !strncmp(buffer, "[INVALID]", 9)) { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode greek test\n"); + fail++; + } + free(copy); + + output_length = sizeof(buffer); + copy = strdup(thai); + copy[0] = 255; + buffer[0] = 255; + ret = tutf8e_encoder_string_encode(tutf8e_encoder_iso_8859_11, copy, "[INVALID]", buffer, &output_length); + if (!ret && !strncmp(buffer, "[INVALID]", 9)) { + printf("%s\n", buffer); + pass++; + } else { + printf("Failed to encode thai test\n"); + fail++; + } + free(copy); + + printf("%d passed, %d failed tests\n", pass, fail); +} |